diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 4cce788e7..0a2257d56 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -5,5 +5,4 @@ # the repo. Unless a later match takes precedence, # @global-owner1 and @global-owner2 will be requested for # review when someone opens a pull request. -* @devanshah2 @Tal-Daniel @ofer-haim @natalygmkibm @apurva-birajdar @itai-g-weather-com @RefaelAdi @PratikshaSonawane @mwnovak-ibm @hadarkorny @doryo @chirag-ibm @rasikashete3 @pankajkumaribm @taees-eimouri @pankajkumar @piyush-desai-ibm @ShalakaKulkarni15 @JingqiuDu - +* @devanshah2 @pankajkumaribm @taees-eimouri @JingqiuDu @Rose-Kaur @nida-bandukwala @laurel-hu @AndychenIBM @jsminem @GlennLee-IBM diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 51411bbb5..3c907a660 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -28,6 +28,9 @@ jobs: actions: read contents: read security-events: write + + env: + ARTIFACTORY_URL: https://repo.maven.apache.org/maven2/ strategy: fail-fast: false diff --git a/.whitesource b/.whitesource new file mode 100644 index 000000000..26e9c471e --- /dev/null +++ b/.whitesource @@ -0,0 +1,3 @@ +{ + "settingsInheritedFrom": "ibm-mend-config/mend-config@main" +} \ No newline at end of file diff --git a/build/defaultOfflinePackagePlugins.txt b/build/defaultOfflinePackagePlugins.txt index 3a906421f..d2f309c35 100644 --- a/build/defaultOfflinePackagePlugins.txt +++ b/build/defaultOfflinePackagePlugins.txt @@ -10,9 +10,11 @@ filter-plugin/logstash-filter-mysql-percona-guardium/logstash-filter-mysql_perco filter-plugin/logstash-filter-generic-guardium/logstash-filter-generic_guardium_filter filter-plugin/logstash-filter-saphana-guardium/logstash-filter-saphana_guardium_plugin_filter filter-plugin/logstash-filter-cassandra-guardium/logstash-filter-cassandra_guardium_plugin_filter +filter-plugin/logstash-filter-mysql-aws-guardium/logstash-filter-mysql_guardium_plugin_filter filter-plugin/logstash-filter-aurora-mysql-guardium/logstash-filter-auroramysqlguardiumpluginfilter filter-plugin/logstash-filter-dynamodb-guardium/logstash-filter-dynamodb_guardium_plugin_filter filter-plugin/logstash-filter-neptune-aws-guardium/logstash-filter-neptune_guardium_filter +filter-plugin/logstash-filter-cockroachdb-guardium/logstash-filter-cockroachdb_guardium_filter filter-plugin/logstash-filter-couchbasedb-guardium/logstash-filter-couchbasedb_guardium_plugin_filter filter-plugin/logstash-filter-couchdb-guardium/logstash-filter-couchdb_guardium_filter filter-plugin/logstash-filter-mariadb-guardium/logstash-filter-mariadb_guardium_filter @@ -25,6 +27,7 @@ filter-plugin/logstash-filter-neo4j-guardium/logstash-filter-neodb_guardium_filt filter-plugin/logstash-filter-snowflake-guardium/logstash-filter-guardium_snowflake_filter filter-plugin/logstash-filter-yugabyte-guardium/logstash-filter-yugabytedb_guardium_filter filter-plugin/logstash-filter-teradatadb-guardium/logstash-filter-teradatadb_guardium_plugin_filter +filter-plugin/logstash-filter-postgres-guardium/logstash-filter-s3sqs_postgresql_guardium_plugin_filter filter-plugin/logstash-filter-pubsub-apachesolr-guardium/logstash-filter-apache_solr_gcp_connector filter-plugin/logstash-filter-pubsub-bigquery-guardium/logstash-filter-big_query_guardium_filter filter-plugin/logstash-filter-pubsub-bigtable-guardium/logstash-filter-big_table_guardium_filter @@ -36,9 +39,15 @@ filter-plugin/logstash-filter-intersystems-iris-guardium/logstash-filter-intersy filter-plugin/logstash-filter-postgres-ibmcloud-guardium/logstash-filter-icd_postgresql_guardium_filter filter-plugin/logstash-filter-mysql-azure-guardium/logstash-filter-azure_mysql_guardium_filter filter-plugin/logstash-filter-scylldb-guardium/logstash-filter-scylladb_guardium_filter +filter-plugin/logstash-filter-databricks-guardium/logstash-filter-databricks_guardium_filter +filter-plugin/logstash-filter-trino-guardium/logstash-filter-trino_guardium_filter +filter-plugin/logstash-filter-capella-guardium/logstash-filter-capella_guardium_filter +filter-plugin/logstash-filter-opensearch-guardium/logstash-filter-opensearch_guardium_filter +filter-plugin/logstash-filter-milvus-guardium/logstash-filter-milvus_guardium_filter +filter-plugin/logstash-filter-singlestore-guardium/logstash-filter-singlestoredb_guardium_filter +filter-plugin/logstash-filter-alloydb-guardium/logstash-filter-alloydb_guardium_filter +input-plugin/logstash-input-couchbase-capella/logstash-input-couchbase_capella_input input-plugin/logstash-input-mongo-atlas/logstash-input-mongo_atlas_input -input-plugin/logstash-input-couchbase-capella/logstash-input-couchbase-capella_input -input-plugin/logstash-input-http/logstash-input-http_input -filter-plugin/logstash-filter-capella-guardium/logstash-filter-capella-guardium_filter -filter-plugin/logstash-filter-databricks-guardium/logstash-filter-databricks-guardium_filter -filter-plugin/logstash-filter-trino-guardium/logstash-filter-trino-guardium_filter \ No newline at end of file +input-plugin/logstash-input-s3sqs/logstash-input-s3_sqs +input-plugin/logstash-input-sqs-custom/logstash-input-custom_sqs + diff --git a/build/pluginsToBuild_GDP.txt b/build/pluginsToBuild_GDP.txt index 18a9481c5..bdd6c2ff2 100644 --- a/build/pluginsToBuild_GDP.txt +++ b/build/pluginsToBuild_GDP.txt @@ -11,9 +11,11 @@ filter-plugin/logstash-filter-mysql-percona-guardium filter-plugin/logstash-filter-generic-guardium filter-plugin/logstash-filter-saphana-guardium filter-plugin/logstash-filter-cassandra-guardium +filter-plugin/logstash-filter-mysql-aws-guardium filter-plugin/logstash-filter-aurora-mysql-guardium filter-plugin/logstash-filter-dynamodb-guardium filter-plugin/logstash-filter-neptune-aws-guardium +filter-plugin/logstash-filter-cockroachdb-guardium filter-plugin/logstash-filter-couchbasedb-guardium filter-plugin/logstash-filter-couchdb-guardium filter-plugin/logstash-filter-mariadb-guardium @@ -33,12 +35,18 @@ filter-plugin/logstash-filter-pubsub-firebase-realtime-guardium filter-plugin/logstash-filter-pubsub-firestore-guardium filter-plugin/logstash-filter-pubsub-spanner-guardium filter-plugin/logstash-filter-intersystems-iris-guardium +filter-plugin/logstash-filter-postgres-guardium filter-plugin/logstash-filter-postgres-ibmcloud-guardium filter-plugin/logstash-filter-mysql-azure-guardium filter-plugin/logstash-filter-scylldb-guardium +filter-plugin/logstash-filter-databricks-guardium +filter-plugin/logstash-filter-trino-guardium +filter-plugin/logstash-filter-capella-guardium +filter-plugin/logstash-filter-opensearch-guardium +filter-plugin/logstash-filter-milvus-guardium +filter-plugin/logstash-filter-singlestore-guardium +filter-plugin/logstash-filter-alloydb-guardium input-plugin/logstash-input-mongo-atlas +input-plugin/logstash-input-s3sqs input-plugin/logstash-input-couchbase-capella -input-plugin/logstash-input-http -filter-plugin/logstash-filter-capella-guardium -filter-plugin/logstash-filter-databricks-guardium -filter-plugin/logstash-filter-trino-guardium \ No newline at end of file +input-plugin/logstash-input-sqs-custom \ No newline at end of file diff --git a/build/verifiedUCPlugins_gdp.txt b/build/verifiedUCPlugins_gdp.txt index a50cc825a..06a5534b6 100644 --- a/build/verifiedUCPlugins_gdp.txt +++ b/build/verifiedUCPlugins_gdp.txt @@ -3,9 +3,11 @@ filter-plugin/logstash-filter-documentdb-aws-guardium/DocumentDBOverCloudwatchPa filter-plugin/logstash-filter-aurora-mysql-guardium/AuroraMysqlOverCloudwatchPackage filter-plugin/logstash-filter-dynamodb-guardium/DynamodbOverCloudwatch/DynamodbOverCloudwatchPackage #filter-plugin/logstash-filter-mariadb-aws-guardium/MariaDBOverCloudWatchPackage +filter-plugin/logstash-filter-mysql-aws-guardium/MySQLOverS3SQS filter-plugin/logstash-filter-mysql-aws-guardium/MysqlOverCloudwatchLogsPackage filter-plugin/logstash-filter-neptune-aws-guardium/NeptuneOverCloudWatchPackage filter-plugin/logstash-filter-postgres-guardium/PostgresOverCloudWatchPackageß +filter-plugin/logstash-filter-postgres-guardium/PostgresOverS3SQSPackage filter-plugin/logstash-filter-s3-guardium/S3OverCloudwatchLogsPackage filter-plugin/logstash-filter-s3-guardium/S3OverSQSPackage #Filebeat plug-ins @@ -16,12 +18,14 @@ filter-plugin/logstash-filter-couchdb-guardium/CouchdbOverFilebeatPackage filter-plugin/logstash-filter-hdfs-guardium/HdfsOverFilebeatPackage filter-plugin/logstash-filter-mariadb-guardium/MariaDBOverFilebeatPackage filter-plugin/logstash-filter-mongodb-guardium/MongodbOverFilebeatPackage +filter-plugin/logstash-filter-milvus-guardium/MilvusOverFilebeatPackage filter-plugin/logstash-filter-mysql-guardium/MysqlOverFilebeatPackage filter-plugin/logstash-filter-mysql-percona-guardium/MysqlPerconaOverFilebeatPackage filter-plugin/logstash-filter-neo4j-guardium/NeodbOverFilebeatPackage filter-plugin/logstash-filter-onPremGreenplumdb-guardium/GreenplumdbOverFilebeatPackage filter-plugin/logstash-filter-onPremPostgres-guardium/PostgresOverFilebeatPackage filter-plugin/logstash-filter-saphana-guardium/SaphanaOverFilebeatPackage +filter-plugin/logstash-filter-singlestore-guardium/SingleStoreOverFilebeatPackage filter-plugin/logstash-filter-yugabyte-guardium/YugabytedbOverFilebeatPackage #JDBC plug-ins filter-plugin/logstash-filter-snowflake-guardium/SnowflakeOverJbdcPackage @@ -37,18 +41,27 @@ filter-plugin/logstash-filter-pubsub-firestore-guardium/PubSubFireStorePackage filter-plugin/logstash-filter-pubsub-mysql-guardium/PubSubMySQLPackage filter-plugin/logstash-filter-pubsub-postgresql-guardium/PubSubPostgreSQLPackage filter-plugin/logstash-filter-pubsub-bigtable-guardium/gdp-pubsub-bigtable-package +filter-plugin/logstash-filter-alloydb-guardium/AlloyDBoverPubSubPackage #Syslog plug-ins +filter-plugin/logstash-filter-cockroachdb-guardium/CockroachDBOverSyslogPackage filter-plugin/logstash-filter-mongodb-guardium/MongoDBOverSyslogPackage filter-plugin/logstash-filter-mysql-guardium/MySQLOverSyslogPackage #Other filter-plugin/logstash-filter-mongodb-guardium/MongodbOverMongoAtlasPackage filter-plugin/logstash-filter-azure-postgresql-guardium/AzurePostgresqlOverAzureEventHub +filter-plugin/logstash-filter-databricks-guardium/AzureDatabricksOverAzureEventHub +filter-plugin/logstash-filter-trino-guardium/TrinoOverSyslogPackage +filter-plugin/logstash-filter-capella-guardium/CapellaCouchbaseOverCapellaPackage +filter-plugin/logstash-filter-opensearch-guardium/OpenSearchOverCloudwatchPackage #Input plug-ins input-plugin/logstash-input-azure-event-hubs/AzureEventHubsInputPackage input-plugin/logstash-input-beats/FilebeatInputPackage +input-plugin/logstash-input-couchbase-capella/InputCouchbaseCapellaPackage input-plugin/logstash-input-cloudwatch-logs/CloudwatchLogsInputPackage input-plugin/logstash-input-jdbc/JdbcInputPackage input-plugin/logstash-input-mongo-atlas/InputMongoAtlasPackage input-plugin/logstash-input-sqs/SQSInputPackage +input-plugin/logstash-input-sqs-custom/AWSSQSInputPackage +input-plugin/logstash-input-s3sqs/InputS3SQSPackage; input-plugin/logstash-input-tcp-syslog/SyslogInputPackage -input-plugin/logstash-input-google-pubsub/GooglePubSubPackage \ No newline at end of file +input-plugin/logstash-input-google-pubsub/GooglePubSubPackage diff --git a/build/verified_UC_plugins_full_list.txt b/build/verified_UC_plugins_full_list.txt index 8ce22a745..671c8fdac 100644 --- a/build/verified_UC_plugins_full_list.txt +++ b/build/verified_UC_plugins_full_list.txt @@ -4,9 +4,11 @@ filter-plugin/logstash-filter-documentdb-aws-guardium/DocumentDBOverCloudwatchPa filter-plugin/logstash-filter-aurora-mysql-guardium/AuroraMysqlOverCloudwatchPackage;logstash-filter-auroramysqlguardiumpluginfilter.zip filter-plugin/logstash-filter-dynamodb-guardium/DynamodbOverCloudwatch/DynamodbOverCloudwatchPackage;logstash-filter-dynamodb_guardium_plugin_filter.zip filter-plugin/logstash-filter-mariadb-aws-guardium/MariaDBOverCloudWatchPackage;logstash-filter-mariadb_guardium_filter.zip +filter-plugin/logstash-filter-mysql-aws-guardium/MySQLOverS3SQS;logstash-filter-mysql_guardium_plugin_filter.zip filter-plugin/logstash-filter-mysql-aws-guardium/MysqlOverCloudwatchLogsPackage; filter-plugin/logstash-filter-neptune-aws-guardium/NeptuneOverCloudWatchPackage;logstash-filter-neptune_guardium_filter.zip filter-plugin/logstash-filter-postgres-guardium/PostgresOverCloudWatchPackage; +filter-plugin/logstash-filter-postgres-guardium/PostgresOverS3SQSPackage;logstash-filter-s3sqs_postgresql_guardium_plugin_filter.zip filter-plugin/logstash-filter-s3-guardium/S3OverCloudwatchLogsPackage;logstash-filter-logstash_filter_s3_guardium.zip filter-plugin/logstash-filter-s3-guardium/S3OverSQSPackage;logstash-filter-logstash_filter_s3_guardium.zip #Filebeat plug-ins @@ -17,12 +19,14 @@ filter-plugin/logstash-filter-couchdb-guardium/CouchdbOverFilebeatPackage;logsta filter-plugin/logstash-filter-hdfs-guardium/HdfsOverFilebeatPackage;logstash-filter-hdfs_guardium_filter.zip filter-plugin/logstash-filter-mariadb-guardium/MariaDBOverFilebeatPackage;logstash-filter-mariadb_guardium_filter.zip filter-plugin/logstash-filter-mongodb-guardium/MongodbOverFilebeatPackage;logstash-filter-mongodb_guardium_filter.zip +filter-plugin/logstash-filter-milvus-guardium/MilvusOverFilebeatPackage;logstash-filter-milvus_guardium_filter.zip filter-plugin/logstash-filter-mysql-guardium/MysqlOverFilebeatPackage;logstash-filter-mysql_filter_guardium.zip filter-plugin/logstash-filter-mysql-percona-guardium/MysqlPerconaOverFilebeatPackage;logstash-filter-mysql_percona_filter.zip filter-plugin/logstash-filter-neo4j-guardium/NeodbOverFilebeatPackage;logstash-filter-neodb_guardium_filter.zip filter-plugin/logstash-filter-onPremGreenplumdb-guardium/GreenplumdbOverFilebeatPackage;logstash-filter-greenplumdb_guardium_filter.zip filter-plugin/logstash-filter-onPremPostgres-guardium/PostgresOverFilebeatPackage; filter-plugin/logstash-filter-saphana-guardium/SaphanaOverFilebeatPackage;logstash-filter-saphana_guardium_plugin_filter.zip +filter-plugin/logstash-filter-singlestore-guardium/SingleStoreOverFilebeatPackage;logstash-filter-singlestoredb_guardium_filter.zip filter-plugin/logstash-filter-yugabyte-guardium/YugabytedbOverFilebeatPackage;logstash-filter-yugabytedb_guardium_filter.zip #JDBC plug-ins filter-plugin/logstash-filter-snowflake-guardium/SnowflakeOverJbdcPackage; @@ -32,6 +36,7 @@ filter-plugin/logstash-filter-mssql-guardium/MssqlOnPremOverJdbcPackage; filter-plugin/logstash-filter-saphana-guardium/SaphanaOverJdbcPackage;logstash-filter-saphana_guardium_plugin_filter.zip filter-plugin/logstash-filter-intersystems-iris-guardium/gi-filter-intersystems-iris-package;gi-filter-intersystems-iris-package.zip #GCP plug-ins +filter-plugin/logstash-filter-alloydb-guardium/AlloyDBoverPubSubPackage;logstash-filter-alloydb_guardium_filter.zip filter-plugin/logstash-filter-pubsub-apachesolr-guardium/PubSubApacheSolrPackage;logstash-filter-apache_solr_gcp_connector.zip filter-plugin/logstash-filter-pubsub-bigquery-guardium/BigQueryOverPubSubPackage;logstash-filter-big_query_guardium_filter.zip filter-plugin/logstash-filter-pubsub-firebase-realtime-guardium/PubSubFirebasePackage;logstash-filter-fire_base_guardium_filter.zip @@ -47,6 +52,7 @@ filter-plugin/logstash-filter-pubsub-bigquery-guardium/gi-pubsub-bigquery-packag filter-plugin/logstash-filter-pubsub-apachesolr-guardium/gi-pubsub-apachsolr-package;logstash-filter-apache_solr_gcp_connector.zip filter-plugin/logstash-filter-pubsub-bigtable-guardium/gi-pubsub-bigtable-package;logstash-filter-big_table_guardium_filter.zip #Syslog plug-ins +filter-plugin/logstash-filter-cockroachdb-guardium/CockroachDBOverSyslogPackage;logstash-filter-cockroachdb_guardium_filter.zip filter-plugin/logstash-filter-onPremPostgres-guardium/PostgresOverSyslogPackage; filter-plugin/logstash-filter-yugabyte-guardium/YugabyteOverSyslogPackage; #Other @@ -59,6 +65,8 @@ input-plugin/logstash-input-cloudwatch-logs/CloudwatchLogsInputPackage input-plugin/logstash-input-jdbc/JdbcInputPackage; input-plugin/logstash-input-mongo-atlas/InputMongoAtlasPackage;logstash-input-mongo_atlas_input.zip input-plugin/logstash-input-sqs/SQSInputPackage; +input-plugin/logstash-input-sqs-custom/AWSSQSInputPackage;logstash-input-custom_sqs.zip +input-plugin/logstash-input-s3sqs/InputS3SQSPackage; input-plugin/logstash-input-tcp-syslog/TCPInputPackage; input-plugin/logstash-input-google-pubsub/GooglePubSubPackage; -input-plugin/logstash-input-google-pubsub/gi-pubsub-package; \ No newline at end of file +input-plugin/logstash-input-google-pubsub/gi-pubsub-package; diff --git a/common/build.gradle b/common/build.gradle index faae36a69..6000399b4 100644 --- a/common/build.gradle +++ b/common/build.gradle @@ -12,24 +12,38 @@ version "${file("VERSION").text.trim()}" // =========================================================================== -sourceCompatibility = 1.8 -targetCompatibility = 1.8 +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 buildscript { repositories { - mavenCentral() - jcenter() - } + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } } repositories { - mavenCentral() + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } } dependencies { implementation group: 'commons-validator', name: 'commons-validator', version: '1.7' + implementation group: 'commons-beanutils', name: 'commons-beanutils', version: '1.11.0' implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.23.1' implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.17.2' implementation group: 'com.google.guava', name: 'guava', version: '33.3.1-jre' @@ -52,15 +66,23 @@ tasks.withType(JavaCompile) { task javadocJar(type: Jar) { from javadoc - classifier = 'javadoc' + archiveClassifier = 'javadoc' } task sourcesJar(type: Jar) { from sourceSets.main.allJava - classifier = 'sources' + archiveClassifier = 'sources' } artifacts { archives sourcesJar archives javadocJar } + +task copyDependencies(type: Copy) { + description 'Copies all runtime dependencies into build/libs directory' + from configurations.runtimeClasspath + into "${buildDir}/libs" +} + +jar.finalizedBy(copyDependencies) \ No newline at end of file diff --git a/common/gradle/wrapper/gradle-wrapper.properties b/common/gradle/wrapper/gradle-wrapper.properties index aa991fcea..e2847c820 100644 --- a/common/gradle/wrapper/gradle-wrapper.properties +++ b/common/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip +networkTimeout=10000 +validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/common/src/main/java/com/ibm/guardium/universalconnector/commons/custom_parsing/CustomParser.java b/common/src/main/java/com/ibm/guardium/universalconnector/commons/custom_parsing/CustomParser.java index a666f7817..c43fe2bb6 100644 --- a/common/src/main/java/com/ibm/guardium/universalconnector/commons/custom_parsing/CustomParser.java +++ b/common/src/main/java/com/ibm/guardium/universalconnector/commons/custom_parsing/CustomParser.java @@ -5,7 +5,15 @@ import com.ibm.guardium.universalconnector.commons.custom_parsing.excepton.InvalidConfigurationException; import com.ibm.guardium.universalconnector.commons.custom_parsing.parsers.IParser; import com.ibm.guardium.universalconnector.commons.structures.Record; -import com.ibm.guardium.universalconnector.commons.structures.*; +import com.ibm.guardium.universalconnector.commons.structures.Accessor; +import com.ibm.guardium.universalconnector.commons.structures.Construct; +import com.ibm.guardium.universalconnector.commons.structures.Data; +import com.ibm.guardium.universalconnector.commons.structures.ExceptionRecord; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import com.ibm.guardium.universalconnector.commons.structures.Sentence; +import com.ibm.guardium.universalconnector.commons.structures.SentenceObject; +import com.ibm.guardium.universalconnector.commons.structures.SessionLocator; +import com.ibm.guardium.universalconnector.commons.structures.Time; import org.apache.commons.validator.routines.InetAddressValidator; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -216,7 +224,7 @@ protected SessionLocator getSessionLocator(String payload) { String clientIp = getClientIp(payload); String clientIpv6 = getClientIpv6(payload); - if (clientIpv6 != null && inetAddressValidator.isValidInet6Address(clientIpv6)) { + if (clientIpv6 != null && !clientIpv6.equals(DEFAULT_IPV6) && inetAddressValidator.isValidInet6Address(clientIpv6)) { // If client IP is IPv6, set both client and server to IPv6 sessionLocator.setIpv6(true); sessionLocator.setClientIpv6(clientIpv6); diff --git a/common/src/main/java/com/ibm/guardium/universalconnector/commons/structures/Record.java b/common/src/main/java/com/ibm/guardium/universalconnector/commons/structures/Record.java index 87696aba7..0b541a1ae 100644 --- a/common/src/main/java/com/ibm/guardium/universalconnector/commons/structures/Record.java +++ b/common/src/main/java/com/ibm/guardium/universalconnector/commons/structures/Record.java @@ -230,6 +230,20 @@ public void setRecordsAffected(Integer recordsAffected) { this.recordsAffected = recordsAffected; } + /** + * Execution time in microseconds for query execution + * Added for query duration tracking support + */ + private Integer executionTime; + + public Integer getExecutionTime() { + return executionTime; + } + + public void setExecutionTime(Integer executionTime) { + this.executionTime = executionTime; + } + @Override public String toString() { @@ -245,6 +259,7 @@ public String toString() { ", data=" + data + ", exception=" + exception + ", recordsAffected=" + recordsAffected + + ", executionTime=" + executionTime + '}'; } } \ No newline at end of file diff --git a/common/src/test/java/com/ibm/guardium/universalconnector/commons/custom_parsing/CustomParserTest.java b/common/src/test/java/com/ibm/guardium/universalconnector/commons/custom_parsing/CustomParserTest.java index fd0505c66..6dfadb9ce 100644 --- a/common/src/test/java/com/ibm/guardium/universalconnector/commons/custom_parsing/CustomParserTest.java +++ b/common/src/test/java/com/ibm/guardium/universalconnector/commons/custom_parsing/CustomParserTest.java @@ -3,7 +3,15 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.ibm.guardium.universalconnector.commons.custom_parsing.excepton.InvalidConfigurationException; import com.ibm.guardium.universalconnector.commons.structures.Record; -import com.ibm.guardium.universalconnector.commons.structures.*; +import com.ibm.guardium.universalconnector.commons.structures.Accessor; +import com.ibm.guardium.universalconnector.commons.structures.Construct; +import com.ibm.guardium.universalconnector.commons.structures.Data; +import com.ibm.guardium.universalconnector.commons.structures.ExceptionRecord; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import com.ibm.guardium.universalconnector.commons.structures.Sentence; +import com.ibm.guardium.universalconnector.commons.structures.SentenceObject; +import com.ibm.guardium.universalconnector.commons.structures.SessionLocator; +import com.ibm.guardium.universalconnector.commons.structures.Time; import org.junit.BeforeClass; import org.junit.Test; diff --git a/docs/FilterPluginCreationGuideline.md b/docs/FilterPluginCreationGuideline.md new file mode 100644 index 000000000..02438559d --- /dev/null +++ b/docs/FilterPluginCreationGuideline.md @@ -0,0 +1,281 @@ +# Setting up Logstash Development Environment: A Complete Guide + +## Introduction + +Setting up a Logstash development environment can be complex, especially when working with custom plugins and multiple +versions. This guide walks you through the complete process of setting up Logstash from source, building custom filter +and input plugins, and installing them for use. + +## Version Compatibility + +Before starting, it's crucial to understand the version dependencies: + +- **GDP 11.x** → Logstash 7.x → Java 8 +- **GDP 12.x** → Logstash 8.x → Java 11 + +Make sure you're using the correct Java version for your target Logstash version. + +## Prerequisites + +### 1. Java SDK Installation + +Download and install the IBM Semeru Runtime from: + +``` +https://developer.ibm.com/languages/java/semeru-runtimes/downloads/?license=IBM +``` + +### 2. Clone Required Repositories + +You'll need to clone three repositories: + +**Logstash Core:** + +```bash +git clone https://github.com/elastic/logstash.git +``` + +For a specific older version: + +```bash +git clone --branch 7.5 --single-branch https://github.com/elastic/logstash.git +``` + +**Universal Connectors (Public):** + +```bash +git clone https://github.com/IBM/universal-connectors +``` + +**Universal Connectors (Internal):** +Note: Only for IBM's developers + +```bash +git clone https://github.ibm.com/Activity-Insights/universal-connectors +``` + +### 3. Build Common Project + +Navigate to the common project inside the public universal-connectors repo and build it: + +1. Clean the project +2. Run assemble +3. Run build + + +## Building Logstash from Source + +### Step 1: Install Ruby Version Manager (RVM) + +Follow the official Logstash development guide, then execute these commands: + +```bash +# Install specific JRuby version +rvm install "jruby-9.3.10.0" + +# Install required gems +gem install rake +gem install bundler -v 2.3.27 + +# Import GPG keys +gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 + +# Install RVM with stable Ruby +\curl -sSL https://get.rvm.io | bash -s stable --ruby=$(cat .ruby-version) +``` + +### Step 2: Verify Ruby Version + +Ensure your Ruby version matches the required version: + +```bash +ruby -v +cat .ruby-version +``` + +These two commands should output the same version. + +### Step 3: Set Environment Variables + +Set these critical environment variables: + +```bash +export OSS=false +export LOGSTASH_SOURCE=1 +export LOGSTASH_PATH=pathToLogstashDirectory/logstash +``` + +**Important Note:** The `OSS=false` export is crucial. Without it, you may encounter this error when testing: + +``` +Logstash stopped processing because of an error: (SystemExit) exit +``` + +### Step 4: Test Your Logstash Installation + +Run a simple test to verify Logstash is working: + +```bash +bin/logstash -e 'input { stdin { } } output { stdout {} }' +``` + +## Building a Filter Plugin + +### Step 1: Create gradle.properties + +In your filter plugin project root, create a `gradle.properties` file with these paths (adjust to your local setup): + +```properties +LOGSTASH_CORE_PATH=pathToLogstashDirectory/logstash/logstash-core +GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH=pathToPublicUniversalConnectorDirectory/guardium-universalconnector-commons/build/libs +``` + +### Step 2: Modify build.gradle + +Add to the build script section: + +```gradle +ext { + snakeYamlVersion = '2.2' +} +``` + +Change the implementation line from: + +```gradle +implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "build/libs/logstash-core-*.*.*.jar") +``` + +To: + +```gradle +implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "build/libs/logstash-core.jar") +``` + +**Note:** You may need to re-import the Record type after these changes. + +### Step 3: Build the Plugin + +```bash +chmod +x gradlew +./gradlew clean build gem +``` + +Run tests to verify everything works correctly. + +## Building an Input Plugin + +The process for input plugins is similar to filter plugins with slight variations: + +### Step 1: Configure gradle.properties + +```properties +LOGSTASH_CORE_PATH=pathToLogstashDirectory/logstash/logstash-core +``` + +If there are conflicting paths, comment them out: + +```properties +#LOGSTASH_CORE_PATH=pathToLogstashDirectory/universal-connector-master/logstash-master/logstash-core +``` + +### Step 2: Modify build.gradle + +Add the same build script modifications as the filter plugin: + +```gradle +ext { + snakeYamlVersion = '2.2' +} +``` + +Update the implementation line: + +```gradle +implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "build/libs/logstash-core.jar") +``` + +### Step 3: Build + +```bash +./gradlew clean build +``` + +## Installing a Filter Plugin + +### Step 1: Prepare Offline Pack + +From your plugin project directory: + +```bash +bin/logstash-plugin prepare-offline-pack \ + --output logstash-filter-auroramysqlguardiumpluginfilter.zip \ + --overwrite logstash-filter-auroramysqlguardiumpluginfilter +``` + +### Step 2: Install the Plugin + +Navigate to your Logstash project directory and install: + +```bash +bin/logstash-plugin install \ + --no-verify \ + --local logstash-filter-auroramysqlguardiumpluginfilter-1.0.1.gem +``` + +## Common Issues and Solutions + +### Issue 1: SystemExit Error + +**Problem:** `Logstash stopped processing because of an error: (SystemExit) exit` + +**Solution:** Ensure you've set `export OSS=false` before running Logstash. + +### Issue 2: Ruby Version Mismatch + +**Problem:** Ruby version doesn't match requirements + +**Solution:** Use RVM to install the exact version specified in `.ruby-version`: + +```bash +rvm install "jruby-9.3.10.0" +rvm use jruby-9.3.10.0 +``` + +### Issue 3: Build Failures + +**Problem:** Gradle build fails with dependency issues + +**Solution:** + +- Verify all paths in `gradle.properties` are correct +- Ensure the common project is built first +- Check that `snakeYamlVersion` is set to `2.2` + +## Best Practices + +1. **Version Control**: Keep track of which Logstash version you're building against +2. **Path Management**: Use absolute paths in `gradle.properties` to avoid confusion +3. **Test Incrementally**: Test after each major step rather than waiting until the end +4. **Documentation**: Keep notes on any custom modifications you make + +## Conclusion + +Setting up a Logstash development environment requires careful attention to version compatibility, proper configuration +of build files, and correct environment variables. By following this guide, you should be able to: + +- Build Logstash from source +- Create custom filter and input plugins +- Install and test your plugins locally + +This setup enables you to develop, test, and deploy custom Logstash plugins for your specific data processing needs. + +## Additional Resources + +- [Logstash Official Documentation](https://github.com/elastic/logstash?tab=readme-ov-file#developing-logstash-core) +- [IBM Universal Connectors](https://github.com/IBM/universal-connectors) +- [IBM Semeru Runtimes](https://developer.ibm.com/languages/java/semeru-runtimes/downloads/?license=IBM) + +--- + +*Last Updated: January 2026* \ No newline at end of file diff --git a/docs/Guardium Data Protection/developing_plugins_gdp.md b/docs/Guardium Data Protection/developing_plugins_gdp.md index 06d67341b..e64107f47 100644 --- a/docs/Guardium Data Protection/developing_plugins_gdp.md +++ b/docs/Guardium Data Protection/developing_plugins_gdp.md @@ -79,7 +79,8 @@ filebeat.inputs: # Each -is an input. Most options can be set at the input level, so # you can use different inputs for various configurations. # Below are the input specific configurations. --type: log +-type: filestream +- id: # Change to true to enable this input configuration. enabled: true # Paths that should be crawled and fetched. Glob based paths.paths: diff --git a/docs/Guardium Data Protection/kafka_uc_integration.md b/docs/Guardium Data Protection/kafka_uc_integration.md index ea630e210..fd8e3d07d 100644 --- a/docs/Guardium Data Protection/kafka_uc_integration.md +++ b/docs/Guardium Data Protection/kafka_uc_integration.md @@ -223,22 +223,25 @@ To configure audit logs for Yugabyte DB, see [Enabling the audit logs](https://g key = "UcSessionID" queue.filename="omkafkaq" queue.spoolDirectory="/var/lib/rsyslog" - queue.size="300000" + queue.size="500000" queue.maxdiskspace="536870912" queue.lowwatermark="20000" - queue.highwatermark="200000" + queue.highwatermark="400000" queue.discardmark="250000" queue.type="LinkedList" - queue.discardseverity="4" + queue.discardseverity="8" queue.saveonshutdown="on" - queue.dequeuebatchsize="4" + queue.dequeuebatchsize="1024" partitions.auto="on" + action.resumeRetryCount="-1" + action.resumeInterval="10" + action.reportSuspension="on" errorFile="/var/log/rsyslog.err" confParam=[ "compression.codec=snappy", - "socket.timeout.ms=1000", - "socket.keepalive.enable=true", - "security.protocol=ssl", - "debug=all", + "queue.buffering.max.messages=10000000", + "socket.timeout.ms=1000", + "socket.keepalive.enable=true","security.protocol=ssl", + "debug=all", "ssl.ca.location=<_ENTER_CERTIFICATE>" ] ) diff --git a/docs/KafkaBasedUCs/AWSMsSQLJDBCKafkaConnect.md b/docs/KafkaBasedUCs/AWSMsSQLJDBCKafkaConnect.md new file mode 100644 index 000000000..b836c31d9 --- /dev/null +++ b/docs/KafkaBasedUCs/AWSMsSQLJDBCKafkaConnect.md @@ -0,0 +1,263 @@ +# Configuring AWS MSSQL datasource profile for JDBC Kafka Connect plug-ins + +You can create and configure datasource profiles through central manager for **AWS MSSQL JDBC Kafka Connect** plug-ins. + +## Meet MSSQL over JDBC Connect + +* Environments: AWS +* Supported inputs: Kafka connect JDBC 2.0 Customized (pull) +* Supported Guardium versions: + * Guardium Data Protection: Appliance bundle 12.2.2 or later + +Kafka-connect is a framework for streaming data between Apache Kafka and other systems. + +## Configuring AWS MSSQL RDS server +Create and configure an AWS RDS Microsoft SQL Server Enterprise Edition instance with external access, audit logging, and S3 integration for storing audit trails. + +### Procedure + +1. Create a database instance.
+ a. Go to https://console.aws.amazon.com/.
+ b. Click on **Services**.
+ c. In the **Database** section, click on **RDS**.
+ d. From the **Region** dropdown menu, select your region where you want to create the databse instance.
+ e. From the **Amazon RDS Dashboard**, click **Create database**.
+ f. In **Choose a database creation method**, select **Standard Create**.
+ g. In the **Engine options** field, select **Microsoft SQL Server and ‘SQL Server Enterprise Edition**.
+ h. In the **Version** field, select **SQL Server 2017 14.00.3281.6.v1**.
+ i. Select the **Dev/Test** template.
+ j. Enter the **Database name**, **Master username** and **Password**. This username and password is used as an input in jdbc connection details for universal connector.
+ k. Optional: In the **Export logs** field, select error logs.
+ l. To access the database from outside, go to the **Connectivity** section and set **Public access** to **Yes**.
+ m. Select **Create database**.
+ +2. To access the database instance from outside, you must add an inbound rule to the database.
+ a. Click the database that you created in the previous step.
+ b. Go to the **Connectivity & security** tab.
+ c. Under the **Security** section, select **VPC security group**, which is the default option you select when creating the database.
+ d. Go to selected default security group.
+ e. Under the **Inbound rule** section, click **Edit inbound rules**.
+ f. Click on the **Add rule** button and add the following rules for MSSQL.
+       i. From the **type** dropdown, select **MSSQL**. In the **Source** column, keep the custom as default. Then click the **Search** icon to select the **0.0.0.0/0** rule.
+       ii. From the **type** dropdown, select **MSSQL**. In the **Source** column, keep the custom as default. Then click the **Search** icon to select the **::/0** rule.
+ g. **Microsoft MSSQL Management Studio** is required to connect with the database and perform database operations. To connect with the database, use the endpoint and port values from the **Connectivity & security** tab in the RDS instance.
+ +3. Assign a parameter group to the database instance.
+ a. You can assign a default parameter group to your database. Use the parameter group family ``sqlserver-ee-14.0`` and set the **rds.sqlserver_audit** parameter to ``true``.
+ b. In the **Navigation** panel, choose **Databases**.
+ c. Select the **mssql** database that you created, then click **Modify**.
+ d. Go to **Advance configurations**.
+ e. Under **Database options**, select the parameter group from drop down.
+ f. Click **Continue**. On next window, select **Apply Immediately** and click **Modify DB Instance**.
+ +4. Create an S3 bucket.
+ a. Click **Services**.
+ b. Select **S3** from the services.
+ c. Choose **Create bucket**.
+ d. Enter the **Bucket name** and **AWS region**, then click **Create bucket**.
+ +5. Create a custom option group for your database instance.
+ a. Click **Services**.
+ b. In the **Database** section, click **RDS**.
+ c. In the **Navigation** panel, choose **Option groups**. Then select **Create group**.
+ d. In the **Create option group** window, complete the following steps.
+       i. In the **Name** field, enter a unique name for the option group within your AWS account. The name can contain only letters, digits, and hyphens.
+       ii. In the **Description** field, enter a brief description of the option group.
+       iii. For the **Engine** field, select **sqlserver-ee**.
+       iv. For the **Major engine version** field, select the major version of the DB engine that you want to use.
+ e. Select the created option group and click **Add option**.
+ f. Select **SQLSERVER_AUDIT** as an option name.
+ g. Set the S3 bucket that you created in the previous step.
+ h. Create a new IAM role. Then create a policy to attach to the IAM role. Use the following JSON:
+ + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": "s3:ListAllMyBuckets", + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": [ + "s3:ListBucket", + "s3:GetBucketACL", + "s3:GetBucketLocation" + ], + "Resource": "arn:aws:s3:::" + }, + { + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:ListMultipartUploadParts", + "s3:AbortMultipartUpload" + ], + "Resource": "arn:aws:s3:::/*" + } + ] + }
+ + i. In the **Scheduling** field, select **Apply Immediately**, then click **Add option**.
+ + +7. Associate the option group with the DB instance.
+ a. In the **Navigation** panel, choose **Databases**.
+ b. Select the **mssql** database that you created, then click **Modify**.
+ c. Go to **Advance configurations**.
+ d. Under **Database options**, select the custom option group that you created in the previous stes.
+ e. Click **Continue**. On the next window, select **Apply Immediately** and then click **Modify DB Instance**.
+ + +## Enabling auditing + +1. Connect to the database.
+ a. Launch SQL Server Management Studio and provide the following connection details:
+       i. In the **Server Name** field, enter the endpoint (available in the AWS RDS console).
+       ii. Enter the username and master password that you set when creating the database.
+ b. Create database.
+ +2. Audit specifications.
+ a. SQL Server audit allows you to create server audits that can contain the following specifications:
+       i. Server audit specifications for server-level events.
+       ii. Database audit specifications for database-level events.
+ b. When you define an audit, you specify the output location for the results (the audit destination). The audit is created in a disabled state and does not automatically audit any actions. After the audit is enabled, the audit destination receives data from the audit.
+ +3. Create an audit.
+ a. Create an audit in Management Studio by going to **Security > Audits > New Audit**.
+ b. Enter the ``D:\rdsdbdata\SQLAudit\`` filepath, which is the default path for AWS RDS instances.
+ c. In the **Maximum file size** field, deselect the **Unlimited** checkbox and enter a specific value.
+ d. Keep the remaining configurations as default.
+ e. Click **OK**.
+ f. Right-click the audit you created and select **Enable**.
+ +4. Create a server audit specification.
+ a. In **Management**, navigate to **Security** and expand it.
+ b. Right-click **Server Audit Specifications** and select **New Audit Specification**.
+ c. Select the audit that you created in the previous step.
+ d. Configure the audit log groups based on your requirements. For more information on audit log groups, see the Microsoft documentation website.
+ e. Click **OK**.
+ f. Right-click the database audit specification you created and select **Enable**.
+ +5. Create a database audit specification.
+ a. In **Management**, navigate to **Databases** and expand it. Then expand **Security** under the database.
+ b. Right-click **Database Audit Specifications** and select **New Audit Specification**.
+ c. Select the audit that you created in the previous step.
+ d. Configure the audit log groups based on your requirements. For more information on audit log groups, see the Microsoft documentation website.
+ e. Click **OK**.
+ f. Right-click the database audit specification you created and select **Enable**.
+ +6. Create a non-admin user to access the audit table without exposing admin credentials.
+ a. Log in to the database using admin credentials and run the following queries:
+ + ```sql + CREATE LOGIN WITH PASSWORD = ''; + USE msdb; + CREATE USER FOR LOGIN ; + GRANT SELECT ON msdb.dbo.rds_fn_get_audit_file TO ; + ``` + + b. In the **Input** section, set the database name as **msdb**.
+ + ```properties + jdbc_connection_string => "jdbc:sqlserver://:;databaseName=msdb;" + ``` + + c. Use the login credentials created in the previous step for the JDBC connection.
+ + ```properties + jdbc_user => "" + jdbc_password => "" + ``` + +## Creating datasource profiles + +You can create a new datasource profile from the **Datasource Profile Management** page. + +### Procedure + +1. Go to **Manage > Universal Connector > Datasource Profile Management** +2. Click the **➕ (Add)** button. +3. You can create a profile by using one of the following methods: + + * To **Create a new profile manually**, go to the **"Add Profile"** tab and provide values for the following fields. + * **Name** and **Description**. + * Select a **Plug-in Type** from the dropdown. For example, `AWS MSSQL Over JDBC Connect 2.0`. + + * To **Upload from CSV**, go to the **"Upload from CSV"** tab and upload an exported or manually created CSV file + containing one or more profiles. You can also choose from the following options: + * **Update existing profiles on name match** — Updates profiles with the same name if they already exist. + * **Test connection for imported profiles** — Automatically tests connections after profiles are created. + * **Use ELB** — Enables ELB support for imported profiles. You must provide the number of MUs to be used in the + ELB process. + +**Note:** Configuration options vary based on the selected plug-in. + +## Configuration: JDBC Kafka Connect 2.0-based Plugins + +The following table describes the fields that are specific to JDBC Kafka Connect 2.0 and similar plugins. + +| Field | Description | +|-------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Name** | Unique name of the profile. | +| **Description** | Description of the profile. | +| **Plug-in** | Plug-in type for this profile. A full list of available plug-ins are available on the **Package Management** page. | +| **Credential** | The credential to authenticate with the datasource. Must be created in **Credential Management**, or click **➕** to create one. | +| **Kafka Cluster** | Kafka cluster to deploy the universal connector. | +| **Label** | Grouping label. For example, customer name or ID. | +| **JDBC Driver Library** | JDBC driver for the database. | +| **Port** | Port that is used to connect to the database. | +| **Hostname** | Hostname of the database. | +| **Query** | SQL query that is used to extract audit logs. | +| **Service Name / SID** | The database **service name** or **SID**. | +| **Initial Time** | Initial polling time for audit logs. | +| **No Traffic Threshold** | Threshold setting for inactivity detection. | +| **Connection URL** | Full JDBC connection string. Format varies by database type.
For example, `jdbc:postgresql://mydb.abc123.us-east-1.rds.amazonaws.com:5432/mydb?ssl=true`. | +| **Use Enterprise Load Balancing (ELB)** | Enable this if ELB support is required. | +| **Managed Unit Count** | Number of Managed Units (MUs) to allocate for ELB. | + +**Note:** +- Depending on the plugin type, the configuration may require either: + - A **Connection URL**, or + - Separate fields for **Hostname**, **Port**, and **Service Name / SID** +- Ensure that the **profile name** is unique. +- Required credentials must be created before or during profile creation. + +--- + +## Testing a Connection + +After creating a profile, you must test the connection to ensure the provided configuration is valid. + +### Procedure + +1. Select the new profile. +2. From the top menu, click **Test Connection**. +3. If the test is successful, you can proceed to installing the profile. + +--- + +## Installing a Profile + +Once the connection test is successful, you can install the profile on **Managed Units (MUs)** or **Edges**. The parsed +audit logs are sent to the selected Managed Unit or Edge to be consumed by the **Sniffer**. + +### Procedure + +1. Select the profile. +2. From the **Install** menu, click **Install**. +3. From the list of available MUs and Edges that is displayed, select the ones that you want to deploy the profile to. + +--- + +## Uninstalling or reinstalling profiles + +An installed profile can be uninstalled or reinstalled if needed. + +### Procedure + +1. Select the profile. +2. From the list of available actions, select the desired option: **Uninstall** or **Reinstall**. + +--- diff --git a/docs/KafkaBasedUCs/AlloyDBPubsubKafkaConnect.md b/docs/KafkaBasedUCs/AlloyDBPubsubKafkaConnect.md new file mode 100644 index 000000000..d9afd2fa0 --- /dev/null +++ b/docs/KafkaBasedUCs/AlloyDBPubsubKafkaConnect.md @@ -0,0 +1,215 @@ +# AlloyDB over Pub/Sub Source Connector + +This connector enables IBM Guardium Data Protection (GDP) to monitor and collect audit logs from AlloyDB databases +through Google Cloud Pub/Sub by using Kafka Connect. + +## Meet AlloyDB over Pub/Sub Connect + +* Environments: On-prem +* Supported inputs: Kafka connect Pubsub 2.0 (pull) +* Supported Guardium versions: + * Guardium Data Protection: Appliance bundle 12.2.2 or later + +Kafka-connect is a framework for streaming data between Apache Kafka and other systems. + +## Configuring AlloyDB on GCP + +1. [Create a cluster and its + primary instance](https://cloud.google.com/alloydb/docs/quickstart/create-and-connect?hl=en#create-cluster). +2. [Connect to your instance and create + a database](https://cloud.google.com/alloydb/docs/quickstart/create-and-connect?hl=en#run). +3. [Connect to the database that you created](https://cloud.google.com/alloydb/docs/quickstart/create-and-connect?hl=en#connect-to-guestbook). +4. [Verify your database + connection](https://cloud.google.com/alloydb/docs/quickstart/create-and-connect?hl=en#verify-connection). +5. [Create a log sink in Pub/Sub](https://cloud.google.com/logging/docs/export/configure_export_v2#creating_sink). + * Use the following inclusion filter for ```Choose logs to include in sink``` during log sink creation to specify + which logs to route. The following filter captures relevant logs based on data access and activity logs: + + ((resource.type="alloydb.googleapis.com/Instance" logName="projects//logs/alloydb.googleapis.com%2Fpostgres.log" )) + +## Configuring GCP for the input plug-in + +1. [Create a topic in Pub/Sub](https://cloud.google.com/pubsub/docs/create-topic#create_a_topic_2). +2. [Create a subscription in Pub/Sub](https://cloud.google.com/pubsub/docs/create-subscription#create_a_pull_subscription). +3. [Create service account credentials](https://developers.google.com/workspace/guides/create-credentials#create_a_service_account): + - To grant subscription access to the service account, select **Pub/Sub Subscriber** from the role + selection list during the service account creation process. + - You do not need to grant users access to this service account. +4. [Create credentials for a service account](https://developers.google.com/workspace/guides/create-credentials#create_credentials_for_a_service_account). + This key is used in the Kafka Connect connector configuration. + +## Enabling audit logs + +The inclusion filter that is used during log sink creation makes sure that only relevant logs are routed. + +### Viewing or downloading logs + +To view or download the generated logs, make sure that the appropriate Identity and Access Management (IAM) roles are +assigned. +These roles control access to logs in GCP. + +* **View logs**: + - roles/logging.viewer (Logs Viewer) + - roles/logging.privateLogViewer (Private Logs Viewer) +* **Download logs**: + - roles/logging.admin (Logging Admin) + - roles/logging.viewAccessor (Logs View Accessor) + +For more information on IAM roles and access control, +see [Access Control with IAM](https://cloud.google.com/logging/docs/access-control). + +### Setting destination permissions + +Route audit logs to a specific destination, such as Pub/Sub topic and subscription. + +1. [Get sink writer's identity](https://cloud.google.com/logging/docs/export/configure_export_v2#dest-auth). +2. If you have owner access to the + destination, [set access controls](https://cloud.google.com/pubsub/docs/access-control#console). Copy the sink + writer's identity and enter it in the **New Principals** field when you configure access policies for topics and + subscriptions.
+ * For **topics**, assign the **Pub/Sub Publisher** and **Pub/Sub Subscriber** role.
+ * For **subscriptions**, assign the **Pub/Sub Publisher** role. + +## Limitations +- Audit logs that contain SQL queries do not contain port and host information, so they are mapped to the default values. +- When you use GCP, duplicate entries can appear in both the reports and audit logs. + +## Configuring Guardium + +The Guardium universal connector is the Guardium entry point for native audit and data access logs. The Guardium +universal connector identifies and parses the received events, and converts them to a standard Guardium format. The +output of the Guardium universal connector is forwarded to the Guardium sniffer on the collector for policy and auditing +enforcements. You can configure Guardium to read the native audit and data access logs by customizing the AlloyDB +template. + +### Before you begin + +* Configure the policies that you need. For more information, see [Policies](/docs/#policies). +* You must have permissions for the S-Tap Management role. By default, the admin user is assigned the S-Tap Management + role. + +## Creating datasource profiles + +You can create a new datasource profile from the **Datasource Profile Management** page. + +### Procedure + +1. Go to **Manage > Universal Connector > Datasource Profile Management** +2. Click the **➕ (Add)** button. +3. You can create a profile by using one of the following methods: + + * To create a new profile manually, go to the **"Add Profile"** tab and provide values for the following fields. + * **Name** and **Description**. + * Select a **Plug-in Type** from the dropdown. For example, **AlloyDB Over PubSub Kafka Connect 2.0**. + + * To upload from CSV, go to the **Upload from CSV** tab and upload an exported or manually created CSV file + containing one or more profiles. You can also choose from the following options: + * **Update existing profiles on name match** — Updates profiles with the same name if they already exist. + * **Test connection for imported profiles** — Automatically tests connections after profiles are created. + * **Use ELB** — Enables ELB support for imported profiles. You must provide the number of MUs to be used in the + ELB process. + +**Note:** Configuration options vary based on the selected plug-in. + +## Configuration: Pub/Sub Kafka Connect 2.0-based Plugins + +The following table describes the fields that are specific to Pub/Sub Kafka Connect 2.0 and similar plugins. + +| Field | Description | +|-------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Name** | Unique name of the profile. | +| **Description** | Description of the profile. | +| **Plug-in** | Plug-in type for this profile. A full list of available plug-ins are available on the **Package Management** page. | +| **Credential** | The credential to authenticate with the datasource. Must be created in **Credential Management**, or click **➕** to create one. | +| **Kafka Cluster** | Kafka cluster to deploy the universal connector. | +| **Label** | Grouping label. For example, **customer name** or **ID**. | +| **GCP project id** | Google Cloud project ID that contains the Pub/Sub subscription. | +| **Pub/Sub Subscription ID** | Pub/Sub subscription ID from which messages are consumed. | +| **GCP Topic** | Pub/Sub topic name. | +| **Maximum poll records** | The maximum number of records returned in a single poll | +| **Expected events per second** | Expected events per second. This value is used to automatically calculate the **parallel.pull.count** parameter when it not set. Calculation formula: ceil(expected.eps / 1000). | | +| **Number of parallel pull streams** | Number of parallel pull streams to use. If not specified, this value is automatically calculated based on **expected.eps** (1 subscriber per 1000 EPS). | +| **No traffic threshold (minutes)** | The time period after which the system detects inactivity. | + +## Testing a Connection + +After creating a profile, you must test the connection to ensure that the provided configuration is valid. + +### Procedure + +1. Select the new profile. +2. From the top menu, click **Test Connection**. +3. If the test is successful, you can proceed to installing the profile. + +--- + +## Installing a Profile + +Once the connection test is successful, you can install the profile on **Managed Units (MUs)** or **Edges**. The parsed +audit logs are sent to the selected Managed Unit or Edge to be consumed by the Sniffer. + +### Procedure + +1. Select the profile. +2. From the **Install** menu, click **Install**. +3. From the list of available MUs and Edges that is displayed, select the ones that you want to deploy the profile to. + +--- + +## Uninstalling or reinstalling profiles + +An installed profile can be uninstalled or reinstalled if needed. + +### Procedure + +1. Select the profile. +2. From the list of available actions, select the desired option: **Uninstall** or **Reinstall**. + +--- + +## Required Configuration Changes for Production + +## Configuring GCP for production + +You must configure **exactly-once** delivery on your Pub/Sub subscription to prevent duplicate audit log entries at the +Pub/Sub level before the messages reach the connector. + +### Procedure + +1. To enable **exactly-once** delivery on your Pub/Sub subscription, use only one of the following commands based on + your scenario. + * For an existing subscription: + ```bash + gcloud pubsub subscriptions update \ + --enable-exactly-once-delivery + ``` + * For a new subscription: + ```bash + gcloud pubsub subscriptions create \ + --topic= \ + --enable-exactly-once-delivery \ + --ack-deadline=600 + ``` + +## Troubleshooting + +#### Messages are not being processed + +1. Verify that all Kafka worker settings are added. +2. Make sure the worker is restarted after configuration changes. +3. Restart the connector after the worker restart. + +#### "Quota exceeded" errors in GCP + +1. Check the current quota usage in the GCP console. +2. Request a quota increase. +3. Temporarily reduce the number of tasks until the quota is increased. + +#### High number of unacknowledged messages in Pub/Sub + +1. Verify that the connector is running by using the following command.
+ ``` + curl http://localhost:8083/connectors//status + ``` +2. Check for errors in logs. +3. Verify that exactly-once delivery is enabled on the subscription. diff --git a/docs/KafkaBasedUCs/ApachesolrPubsubkafkaConnect.md b/docs/KafkaBasedUCs/ApachesolrPubsubkafkaConnect.md new file mode 100644 index 000000000..78d87ebc4 --- /dev/null +++ b/docs/KafkaBasedUCs/ApachesolrPubsubkafkaConnect.md @@ -0,0 +1,302 @@ +# Apache Solr over Pub/Sub Source Connector + +This connector enables IBM Guardium Data Protection (GDP) to monitor and collect audit logs from Apache Solr databases +through Google Cloud Pub/Sub using Kafka Connect. + +## Meet Apache Solr over Pub/Sub Connect + +* Environments: On-prem +* Supported inputs: Kafka connect Pubsub 2.0 (pull) +* Supported Guardium versions: + * Guardium Data Protection: Appliance bundle 12.2.2 or later + +Kafka-connect is a framework for streaming data between Apache Kafka and other systems. + +## Configuring Apache Solr on GCP + +### Creating a GCP instance + +1. [Create a GCP VM Instance](https://cloud.google.com/compute/docs/instances/create-start-instance#create-instance-methods). +2. Optional: Change the **Zone** for this VM. Compute Engine randomizes the list of zones within each region to encourage + use across multiple zones. +3. Select a Machine configuration for your VM. +4. In the **Firewall** section, select **Allow HTTP traffic** or **Allow HTTPS + traffic** to permit HTTP or HTTPS traffic to the VM. Add the port on which Solr is running (default is 8983). + +### Configuring the instance + +1. Install Java by running the following command. + ``` + $ sudo apt install default-jre + ``` +2. Verify the Java version by running the following command. + ``` + $ java –version + ``` +3. For Solr to work as expected, you need to install the ``lsof`` command by running the following command. The ``lsof`` command stands for list open files. + ``` + $ sudo apt install lsof + ``` +4. Download the Solr installation files by running the following commands. + ``` + $ cd /usr/src + $ sudo apt install wget + $ sudo apt-get install wget + $ sudo wget https://archive.apache.org/dist/lucene/solr/8.6.0/solr-8.6.0.tgz + $ sudo tar -xzvf solr-8.6.0.tgz + ``` +5. Run the Solr installation script. + ``` + $ cd solr-8.6.0/bin + $ sudo ./install_solr_service.sh ../../solr-8.6.0.tgz + ``` + +### Launching and logging Apache Solr + +1. When the script completes, Solr is installed as a service and runs in the background on your server (on port 8983). To verify the status of Solr, run the following command. + ``` + $ sudo service solr status + ``` +2. Solr can run in the following two modes: + + - **Standalone mode**: An index is stored on a single computer, and the setup is called a core. Multiple cores or indexes can exist. To launch Solr in **Standalone** mode, run the following command. + ``` + $ cd /opt/solr-8.6.0 + $ sudo bin/solr start -force + ``` + - **SolrCloud mode**: An index is distributed across multiple computers or multiple server instances on one computer. Groups of documents are called collections. To launch Solr in **SolrCloud** mode, run the following command. + ``` + $ cd /opt/solr-8.6.0 + $ sudo bin/solr start -e cloud -force + ``` + +For more information about Apache Solr, see [Installing Solr](https://solr.apache.org/guide/8_8/installing-solr.html). + +**Note:** When the Apache Solr setup is complete, install and configure the Ops Agent on the system. +To access the Solr admin panel, visit the hostname or IP address (external IP address of your VM instance) on the ``http://ip_address:port/solr/`` port where Solr is running. + +### Creating a Solr core in standalone mode + +1. Create a new Solr core by using the following command. + ``` + $ sudo bin/solr create -c core_name -force + ``` + For example, to create a core named ``new_core``, use the following command. + + ``` + $ sudo bin/solr create -c new_core -force + ``` + +2. The created core appears in the **Core** drop-down menu on the Solr admin console. + +### Creating a Solr collection in SolrCloud mode + +1. Create a new Solr collection by using the following command (with default shard and replica count). + ``` + $ sudo bin/solr create -c collection_name -force + ``` + To create a collection with a specified shard and replica count, use the following command. + ``` + $ sudo bin/solr create -c collection_name -s -rf -force + ``` + For example, to create a collection named ``new_collection``, use the following command. + ``` + $ sudo bin/solr create -c new_collection -force + $ sudo bin/solr create -c new_collection -s 1 -rf 2 -force + ``` +2. The created collection appears in the **Collection** drop-down menu on the Solr admin console. + +## Configuring GCP for the input plug-in + +1. [Create a topic in Pub/Sub](https://cloud.google.com/pubsub/docs/create-topic#create_a_topic_2). +2. [Create a subscription in Pub/Sub](https://cloud.google.com/pubsub/docs/create-subscription#create_a_pull_subscription). +3. [Create service account credentials](https://developers.google.com/workspace/guides/create-credentials#create_a_service_account). + - To grant subscription access to the service account, select the **Pub/Sub Subscriber** role from the role + selection list during the service account creation process. + - You do not need to grant users access to this service account. +4. [Create credentials for a service account](https://developers.google.com/workspace/guides/create-credentials#create_credentials_for_a_service_account). + This key is used in the Kafka Connect connector configuration. + +## Enabling audit logs + +The inclusion filter that is used during log sink creation makes sure that only relevant logs are routed. + +### Viewing or downloading logs + +To view or download the generated logs, make sure that the appropriate Identity and Access Management (IAM) roles are +assigned. These roles control access to logs in GCP. + +* **View logs**: + - roles/logging.viewer (Logs Viewer) + - roles/logging.privateLogViewer (Private Logs Viewer) +* **Download logs**: + - roles/logging.admin (Logging Admin) + - roles/logging.viewAccessor (Logs View Accessor) + +For more information on IAM roles and access control, see [Access Control with IAM](https://cloud.google.com/logging/docs/access-control). + +### Setting destination permissions + +To route audit logs to a specific destination, such as Pub/Sub topic and subscription, complete the following steps. + +1. [Get sink writer's identity](https://cloud.google.com/logging/docs/export/configure_export_v2#dest-auth). +2. If you have owner access to the destination, [set access controls](https://cloud.google.com/pubsub/docs/access-control#console). Copy the sink + writer's identity and enter it in the **New Principals** field when you configure access policies for topics and + subscriptions.
+ * For **topics**, assign the **Pub/Sub Publisher** and **Pub/Sub Subscriber** role.
+ * For **subscriptions**, assign the **Pub/Sub Publisher** role. + +## Limitations +1. The following important fields can not be mapped with ApacheSolr qtp logs. + - SourceProgram : This field is left blank since this information is not embedded in the messages pulled from Google Cloud. + - clientIP and serverIP : Both fields are populated with `0.0.0.0`, as this information is not embedded in the messages pulled from Google Cloud. + - OS User : Not available with logs + - Client HostName : Not available with logs + - dbUser : Not available with logs + - LOGIN_FAILED : Not available with logs + +2. While launching Solr in SolrCloud mode, multiple logs are generated for single query execution as a call to shard (in SolrCloud, a logical partition of a single Collection) and replica (A Core that acts as a physical copy of a Shard in a SolrCloud Collection). + +3. On executing error queries in multiline from third-party tool, GCP is not capturing log as a single event. So, partial query will be displayed in the SQL Error Report. + +4. When you use GCP, duplicate entries can appear in both the reports and audit logs. + +## Configuring Guardium + +The Guardium universal connector is the Guardium entry point for native audit and data access logs. The Guardium +universal connector identifies and parses the received events, and converts them to a standard Guardium format. The +output of the Guardium universal connector is forwarded to the Guardium sniffer on the collector for policy and auditing +enforcements. You can configure Guardium to read the native audit and data access logs by customizing the Apache Solr +template. + +### Before you begin + +* Configure the policies that you need. For more information, see [Policies](/docs/#policies). +* You must have permissions for the S-Tap Management role. By default, the admin user is assigned the S-Tap Management + role. + +## Creating datasource profiles + +You can create a new datasource profile from the **Datasource Profile Management** page. + +### Procedure + +1. Go to **Manage > Universal Connector > Datasource Profile Management** +2. Click the **➕ (Add)** button. +3. You can create a profile by using one of the following methods: + + * To create a new profile manually, go to the **"Add Profile"** tab and provide values for the following fields. + * **Name** and **Description**. + * Select a **Plug-in Type** from the dropdown. For example, **Apache Solr Over PubSub Kafka Connect 2.0**. + + * To upload from CSV, go to the **Upload from CSV** tab and upload an exported or manually created CSV file + containing one or more profiles. You can also choose from the following options: + * **Update existing profiles on name match** — Updates profiles with the same name if they already exist. + * **Test connection for imported profiles** — Automatically tests connections after profiles are created. + * **Use ELB** — Enables ELB support for imported profiles. You must provide the number of MUs to be used in the + ELB process. + +**Note:** Configuration options vary based on the selected plug-in. + +## Configuration: Pub/Sub Kafka Connect 2.0-based Plugins + +The following table describes the fields that are specific to Pub/Sub Kafka Connect 2.0 and similar plugins. + +| Field | Description | +|-------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Name** | Unique name of the profile. | +| **Description** | Description of the profile. | +| **Plug-in** | Plug-in type for this profile. A full list of available plug-ins are available on the **Package Management** page. | +| **Credential** | The credential to authenticate with the datasource. Must be created in **Credential Management**, or click **➕** to create one. | +| **Kafka Cluster** | Kafka cluster to deploy the universal connector. | +| **Label** | Grouping label. For example, **customer name** or **ID**. | +| **GCP project id** | Google Cloud project ID that contains the Pub/Sub subscription. | +| **Pub/Sub Subscription ID** | Pub/Sub subscription ID from which messages are consumed. | +| **GCP Topic** | Pub/Sub topic name. | +| **Maximum poll records** | The maximum number of records returned in a single poll. | +| **Expected events per second** | Expected events per second. This value is used to automatically calculate the **parallel.pull.count** parameter when it not set. Calculation formula: ceil(expected.eps / 1000). | | +| **Number of parallel pull streams** | Number of parallel pull streams to use. If not specified, this value is automatically calculated based on **expected.eps** (1 subscriber per 1000 EPS). | +| **No traffic threshold (minutes)** | The time period after which the system detects inactivity. | + +## Testing a Connection + +After creating a profile, you must test the connection to ensure that the provided configuration is valid. + +### Procedure + +1. Select the new profile. +2. From the top menu, click **Test Connection**. +3. If the test is successful, you can proceed to installing the profile. + +--- + +## Installing a Profile + +Once the connection test is successful, you can install the profile on **Managed Units (MUs)** or **Edges**. The parsed +audit logs are sent to the selected Managed Unit or Edge to be consumed by the Sniffer. + +### Procedure + +1. Select the profile. +2. From the **Install** menu, click **Install**. +3. From the list of available MUs and Edges that is displayed, select the ones that you want to deploy the profile to. + +--- + +## Uninstalling or reinstalling profiles + +An installed profile can be uninstalled or reinstalled if needed. + +### Procedure + +1. Select the profile. +2. From the list of available actions, select the desired option: **Uninstall** or **Reinstall**. + +--- + +## Required Configuration Changes for Production + +## Configuring GCP for production + +You must configure **exactly-once** delivery on your Pub/Sub subscription to prevent duplicate audit log entries at the +Pub/Sub level before the messages reach the connector. + +### Procedure + +1. To enable **exactly-once** delivery on your Pub/Sub subscription, use only one of the following commands based on + your scenario. + * For an existing subscription: + ```bash + gcloud pubsub subscriptions update \ + --enable-exactly-once-delivery + ``` + * For a new subscription: + ```bash + gcloud pubsub subscriptions create \ + --topic= \ + --enable-exactly-once-delivery \ + --ack-deadline=600 + ``` + +## Troubleshooting + +#### Messages are not being processed + +1. Verify that all Kafka worker settings are added. +2. Make sure the worker is restarted after configuration changes. +3. Restart the connector after the worker restart. + +#### "Quota exceeded" errors in GCP + +1. Check the current quota usage in the GCP console. +2. Request a quota increase. +3. Temporarily reduce the number of tasks until the quota is increased. + +#### High number of unacknowledged messages in Pub/Sub + +1. Verify that the connector is running by using the following command.
+ ``` + curl http://localhost:8083/connectors//status + ``` +2. Check for errors in logs. +3. Verify that exactly-once delivery is enabled on the subscription. diff --git a/docs/KafkaBasedUCs/AuroraMySqlCloudwatchKafkaConnect.md b/docs/KafkaBasedUCs/AuroraMySqlCloudwatchKafkaConnect.md new file mode 100644 index 000000000..af0231404 --- /dev/null +++ b/docs/KafkaBasedUCs/AuroraMySqlCloudwatchKafkaConnect.md @@ -0,0 +1,192 @@ +# Configuring Aurora-MySQL datasource profile for Kafka Connect plug-ins + +Create and configure datasource profiles through central manager for **Aurora-MySQL +over CloudWatch Kafka Connect** plug-ins. + +### Meet Aurora-MySQL over CloudWatch Connect + +* Environments: AWS +* Supported inputs: Kafka connect Cloudwatch 2.0 (pull) +* Supported Guardium versions: + * Guardium Data Protection: Appliance bundle 12.2.2 or later + +Kafka-connect is a framework for streaming data between Apache Kafka and other systems. This connector enables +monitoring of Aurora-MySQL audit logs through CloudWatch. + +## Enabling auditing for Aurora-MySQL + +### Creating a database + +1. Go to https://console.aws.amazon.com/. +2. Click **Services**. +3. In the **Database** section, click **RDS**. +4. From the **Region** dropdown menu, select your region where you want to create the databse instance. +5. In the central panel of the Amazon RDS dashboard, click **Create database**. +6. Choose a database creation method. +7. In the **Engine** field, select **Amazon Aurora**, then select **Amazon Aurora MySQL-Compatible Edition**. +8. Select a capacity type (Provisioned). +9. Select a template (Production or Dev/Test). +10. In the **Settings** section, enter the database instance name and create the master account with a username and password to log in to the database. +11. Select the database instance size according to your requirements. +12. Select appropriate storage options. For example, you can enable auto scaling. +13. Select the **Availability** and **Durability** options. +14. Select the connectivity settings that are appropriate for your environment. To make the database accessible, set the **Public access** option to **Publicly Accessible within Additional Configuration**. +15. Select the **Authentication type** for the database (choose from Password Authentication, Password and IAM database authentication, or Password and Kerberos authentication). +16. Expand the **Additional Configuration** options and complete the following steps.
+ a. Configure the database options.
+ b. Select a DB cluster parameter group.
+ c. Select options for Backup.
+ d. If desired, enable **Encryption** on the database instances.
+ e. In **Log exports**, select the log types to publish to Amazon CloudWatch (Audit log).
+ f. Select the options for **Deletion protection**.
+17. Click **Create Database**. +18. To view the database, click **Databases** under Amazon RDS in the left panel. +19. To authorize inbound traffic, edit the security group.
+ a. In the database summary page, select the **Connectivity and Security** tab. Under **Security**, click **VPC security group**.
+ b. Click the group name you selected while creating the database (each database has one active group).
+ c. In the **Inbound rules** section, choose to edit the inbound rules and set the following rule.
+       - **Type**: MYSQL/Aurora
+       - **Protocol**: TCP
+       - **Port Range**: 3306 (depending on your requirements, the source can be set to a specific IP address or opened to all hosts)
+ d. Click **Add Rule** and then click **Save changes**. You may need to restart the database.
+ +### Creating a parameter group + +1. Click **Parameter Groups**, then click **Create Parameter Groups**. +2. Provide the following details.
+ a. **Parameter group family**: Select the Aurora MySQL version
+ b. **Type**: DB cluster parameter group
+ c. **Group name**: Enter a name for the group
+ d. **Description**: Enter a description
+ +3. Click **Create**. +4. Click **DB Parameter > Parameter group actions > Edit**. +5. Change the parameter values by adding the following settings.
+ a. **server_audit_events** = ``CONNECT,QUERY_DCL,QUERY_DDL,QUERY_DML``
+ b. **server_audit_excl_users** = ``rdsadmin``
+ c. **server_audit_logging** = ``1``
+ d. **server_audit_logs_upload** = ``1``
+ e. **log_output** = ``FILE``
+6. Click **Save changes**. +7. Go to **Database Clusters**, then click **Modify**. +8. Go to **Additional Configurations > Database options**. +9. Change the DB clustor parameter group. +10. Click **Continue**, then select **Apply immediately**. +11. Click **Modify Cluster**. +12. Reboot the DB cluster for the changes to take effect. + +## Viewing the logs entries on Cloudwatch + +By default, each database instance has an associated log group with a name in this format: ``/aws/rds/instance//aurora-mysqlql``. You can use this log group, or you can create a new one and associate it with the database instance. + +### Procedure + +1. On the AWS Console page, open the **Services** menu. +2. Enter the CloudWatch string in the search box. +3. Click **CloudWatch** to redirect to the CloudWatch dashboard. +4. In the left panel, select **Logs**. +5. Click **Log Groups**. + +Go to Cloudwatch from the search box and find the details of the generated logs (UserActivity/Connection) in the following log groups: + - `/aws/rds/cluster//audit` + - `/aws/rds/cluster//error` + +## Limitations + +1. The aurora-mysql plug-in does not support IPV6. +2. The aurora-mysql auditing does not audit **Procedure**, **Function**, and **Show** table operations. +3. **Source Program** appears blank in report. +4. Syntactically incorrect queries are not captured in audit logs. + + +## Creating datasource profiles + +You can create a new datasource profile from the **Datasource Profile Management** page. + +### Procedure + +1. Go to **Manage > Universal Connector > Datasource Profile Management** +2. Click the **➕ (Add)** button. +3. You can create a profile by using one of the following methods: + + - To **Create a new profile manually**, go to the **"Add Profile"** tab and provide values for the following fields. + - **Name** and **Description**. + - Select a **Plug-in Type** from the dropdown. For example, `AWS Aurora MySQL Over Cloudwatch Connect 2.0`. + + - To **Upload from CSV**, go to the **"Upload from CSV"** tab and upload an exported or manually created CSV file + containing one or more profiles. + You can also choose from the following options: + - **Update existing profiles on name match** — Updates profiles with the same name if they already exist. + - **Test connection for imported profiles** — Automatically tests connections after profiles are created. + - **Use ELB** — Enables ELB support for imported profiles. You must provide the number of MUs to be used in the + ELB process. + +**Note:** Configuration options vary based on the selected plug-in. + +## Configuring Aurora-MySQL Over CloudWatch Kafka Connect 2.0 + +The following table describes the fields that are specific to Aurora-MySQL over CloudWatch Kafka Connect 2.0 plugin. + +| Field | Description | +|-----------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Name** | Unique name of the profile. | +| **Description** | Description of the profile. | +| **Plug-in** | Plug-in type for this profile. Select **AWS Aurora MySQL Over Cloudwatch Connect 2.0**. A full list of available plug-ins are available on the **Package Management** page. | +| **Credential** | Select AWS Credentials or AWS Role ARN. The credential to authenticate with AWS. Must be created in **Credential Management**, or click **➕** to create one. For more information, see [Creating Credentials](https://www.ibm.com/docs/en/SSMPHH_12.x/com.ibm.guardium.doc.stap/guc/guc_credential_management.html). | +| **Kafka Cluster** | Select the appropriate Kafka cluster from the available Kafka cluster list or create a new Kafka cluster. For more information, see [Managing Kafka clusters](https://www.ibm.com/docs/en/SSMPHH_12.x/com.ibm.guardium.doc.stap/guc/guc_kafka_cluster_management.html). | +| **Label** | Grouping label. For example, customer name or ID. | +| **AWS account region** | Specifies the AWS region where your Aurora-MySQL instance is located (e.g., us-east-1, eu-west-1). | +| **Log groups** | List of CloudWatch log groups to monitor. These are the log groups where Aurora-MySQL audit logs are exported. | +| **Filter pattern** | CloudWatch Logs filter pattern to apply. Use "None" to retrieve all logs, or specify a pattern to filter specific log events. | +| **Account ID** | Your AWS account ID (12-digit number). This identifies your AWS account. | +| **Cluster name** | The name of your Aurora-MySQL cluster or instance identifier. | +| **Ingestion delay (seconds)** | Default value is 900 seconds (15 minutes). This delay accounts for the time it takes for logs to be available in CloudWatch after being generated. | +| **No-traffic threshold (minutes)** | Default value is 60. If there is no incoming traffic for an hour, S-TAP displays a red status. Once incoming traffic resumes, the status returns to green. | +| **Unmask sensitive value** | Optional boolean flag. When enabled, sensitive values in the audit logs will not be masked. | +| **Use Enterprise Load Balancing (ELB)** | Enable this if ELB support is required. | +| **Managed Unit Count** | Number of Managed Units (MUs) to allocate for ELB. | + +**Note:** + +- Ensure that the **profile name** is unique. +- Required credentials must be created before or during profile creation. +- The AWS credentials must have appropriate permissions to read CloudWatch logs. + +--- + +## Testing a Connection + +After creating a profile, you must test the connection to ensure the provided configuration is valid. + +### Procedure + +1. Select the new profile. +2. From the top menu, click **Test Connection**. +3. If the test is successful, you can proceed to installing the profile. + +--- + +## Installing a Profile + +Once the connection test is successful, you can install the profile on **Managed Units (MUs)** or **Edges**. The parsed +audit logs are sent to the selected Managed Unit or Edge to be consumed by the **Sniffer**. + +### Procedure + +1. Select the profile. +2. From the **Install** menu, click **Install**. +3. From the list of available MUs and Edges that is displayed, select the ones that you want to deploy the profile to. + +--- + +## Uninstalling or reinstalling profiles + +An installed profile can be uninstalled or reinstalled if needed. + +### Procedure + +1. Select the profile. +2. From the list of available actions, select the desired option: **Uninstall** or **Reinstall**. + +--- + diff --git a/docs/KafkaBasedUCs/AuroraPostgresCloudwatchKafkaConnect.md b/docs/KafkaBasedUCs/AuroraPostgresCloudwatchKafkaConnect.md new file mode 100644 index 000000000..ec2cac53d --- /dev/null +++ b/docs/KafkaBasedUCs/AuroraPostgresCloudwatchKafkaConnect.md @@ -0,0 +1,211 @@ +# Configuring Aurora Postgres datasource profile for Kafka Connect plug-ins + +Create and configure datasource profiles through Central Manager for **Aurora Postgres +over CloudWatch Kafka Connect** plug-ins. + +### Meet Aurora Postgres over CloudWatch Connect + +* Environments: AWS +* Supported inputs: Kafka connect Cloudwatch 2.0 (pull) +* Supported Guardium versions: + * Guardium Data Protection: Appliance bundle 12.2.1 or later + +Kafka-connect is a framework for streaming data between Apache Kafka and other systems. This connector enables +monitoring of Aurora Postgres audit logs through CloudWatch. + +## Enabling auditing for Aurora Postgres + +### Creating a database + +1. Go to https://console.aws.amazon.com/. +2. Click **Services**. +3. In the **Database** section, click **RDS**. +4. From the **Region** dropdown menu, select your region where you want to create the databse instance. +5. In the central panel of the Amazon RDS dashboard, click **Create database**. +6. Choose a database creation method. +7. In the **Engine** field, select **PostgreSQL**, and then select the appropriate version. +8. Select a template (Production, Dev/Test, or Free Tier). +9. In the **Settings** section, enter the database instance name and create the master account with a username and password to log in to the database. +10. Select the database instance size according to your requirements. +11. Select appropriate storage options. For example, you can enable auto scaling. +12. Select the **Availability** and **Durability** options. +13. Select the connectivity settings that are appropriate for your environment. To make the database accessible, set the **Public access** option to **Publicly Accessible within Additional Configuration**. +14. Select the **Authentication type** for the database (choose from Password Authentication, Password and IAM database authentication, or Password and Kerberos authentication). +15. Expand the **Additional Configuration** options and complete the following steps.
+ a. Configure the database options.
+ b. Select a DB cluster parameter group.
+ c. Select options for Backup.
+ d. Optional: Enable **Encryption** on the database instances.
+ e. In **Log exports**, select the **Postgresql** log type to publish to Amazon CloudWatch.
+ f. Select the options for **Deletion protection**.
+16. Click **Create Database**. +17. To view the database, click **Databases** under Amazon RDS in the left panel. +18. To authorize inbound traffic, edit the security group.
+ a. In the database summary page, select the **Connectivity and Security** tab. Under **Security**, click **VPC security group**.
+ b. Click the group name you selected while creating the database (each database has one active group).
+ c. In the **Inbound rules** section, choose to edit the inbound rules and set the following rule.
+       - **Type**: PostgreSQL
+       - **Protocol**: TCP
+       - **Port Range**: 5432
+       **Note:** Depending on your requirements, the source can be set to a specific IP address or it can be opened to all hosts.
+ d. Click **Add Rule** and then click **Save changes**. You may need to restart the database.
+ + +## Enabling the PGAudit extension + +There are different ways to audit and log in PostgreSQL. In this procedure, we will use **PGAudit**, the open-source audit logging extension for PostgreSQL 9.5+. This extension supports logging for sessions or objects. + +**Note:** Configure either **Session Auditing** or **Object Auditing**. You cannot enable both at the same time. + +### 1. Create a database parameter group. + +When you create a database instance, it is associated with the default parameter group. To create a new database parameter group, complete the following steps. + +1. Go to **Services** > **Database** > **Parameter groups** +2. From the left panel, click **Create Parameter Group**. +3. Enter the parameter group details.
+ a. Select the parameter group family. For example, **aurora-postgres12**. This version should match the version of the database you created and with which this parameter group will be associated.
+ b. Enter the **DB parameter group name**.
+ c. Enter the **DB parameter group description**.
+4. Click **Save**. The new group appears in the **Parameter Groups** section. + +### 2a. Enabling PGAudit Session auditing + +Session Auditing allows you to log activities that are selected in the **pgaudit.log** parameter for logging. Be cautious when selecting which activities to log, as logged activities can affect database instance performance. + +1. From the Amazon RDS left panel, select **Parameter Groups**. +2. Select the parameter group you created. +3. Click **Edit parameters** and add the following settings.
+ a. **pgaudit.log** = ``all, -misc`` (Select options from the **Allowed values** list. You can specify multiple values separated by commas. Values that are marked with "**-**" are excluded from logging.)
+ b. **pgaudit.log_catalog** = ``0``
+ c. **pgaudit.log_parameter** = ``0``
+ d. **shared_preload_libraries** = ``pgaudit``
+ e. **log_error_verbosity** = ``default``
+ +### 2b. Enabling PGAudit Object Auditing + +Object auditing affects performance less than session auditing due to the fine-grained criteria of tables and columns that you can select for auditing. + +1. Set the following parameters.
+ a. **pgaudit.log** = ``none`` (since this is not needed for extensive SESSION logging)
+ b. **pgaudit.role** = ``rds_pgaudit``
+ c. **pgaudit.log_catalog** = ``0``
+ d. **pgaudit.log_parameter** = ``0``
+ e. **shared_preload_libraries** = ``pgaudit``
+ f. **log_error_verbosity** = ``default``
+ +2. Provide the required permissions to the **rds_pgaudit** role when associating it with the table to be audited. For example, ```GRANT ALL ON TO rds_pgaudit```. + This grant enables full **SELECT**, **INSERT**, **UPDATE**, and **DELETE** logging on the relation. + +### 3. Associating the DB parameter group with the database instance + +1. Go to **Services** > **Database** > **RDS** > **Databases**. +2. Click the Aurora Postgres database instance that you want to update. +3. Click **Modify**. +4. Go to **Additional Configurations** > **Database Options** > **DB Parameter Group menu**, and select the newly-created group. +5. Click **Continue**. +6. Select the database instance in its configuration section. The state of the DB Parameter Group is pending-reboot. +7. Reboot the database instance for the changes to take effect. + +## Viewing the logs entries on Cloudwatch + +1. On the AWS Console page, open the **Services** menu. +2. Enter the CloudWatch string in the search box. +3. Click **CloudWatch** to redirect to the CloudWatch dashboard. +4. In the left panel, select **Logs**. +5. Click **Log Groups**. + +Go to Cloudwatch from the search box and find the details of the generated logs (UserActivity/Connection) in the `/aws/rds/cluster//postgresql` log group. + + +## Creating datasource profiles + +You can create a new datasource profile from the **Datasource Profile Management** page. + +### Procedure + +1. Go to **Manage > Universal Connector > Datasource Profile Management** +2. Click the **➕ (Add)** button. +3. You can create a profile by using one of the following methods: + + - To **Create a new profile manually**, go to the **"Add Profile"** tab and provide values for the following fields. + - **Name** and **Description**. + - Select a **Plug-in Type** from the dropdown. For example, `AWS Postgres Over Cloudwatch Connect 2.0`. + + - To **Upload from CSV**, go to the **"Upload from CSV"** tab and upload an exported or manually created CSV file + containing one or more profiles. + You can also choose from the following options: + - **Update existing profiles on name match** — Updates profiles with the same name if they already exist. + - **Test connection for imported profiles** — Automatically tests connections after profiles are created. + - **Use ELB** — Enables ELB support for imported profiles. You must provide the number of MUs to be used in the + ELB process. + +**Note:** Configuration options vary based on the selected plug-in. + +## Configuring Aurora Postgres Over CloudWatch Kafka Connect 2.0 + +The following table describes the fields that are specific to Aurora Postgres over CloudWatch Kafka Connect 2.0 plugin. + +| Field | Description | +|-----------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Name** | Unique name of the profile. | +| **Description** | Description of the profile. | +| **Plug-in** | Plug-in type for this profile. Select `AWS Postgres Over Cloudwatch Connect 2.0`. A full list of available plug-ins are available on the **Package Management** page. | +| **Credential** | Select AWS Credentials or AWS Role ARN. The credential to authenticate with AWS. Must be created in **Credential Management**, or click **➕** to create one. For more information, see [Creating Credentials](https://www.ibm.com/docs/en/SSMPHH_12.x/com.ibm.guardium.doc.stap/guc/guc_credential_management.html). | +| **Kafka Cluster** | Select the appropriate Kafka cluster from the available Kafka cluster list or create a new Kafka cluster. For more information, see [Managing Kafka clusters](https://www.ibm.com/docs/en/SSMPHH_12.x/com.ibm.guardium.doc.stap/guc/guc_kafka_cluster_management.html). | +| **Label** | Grouping label. For example, customer name or ID. | +| **AWS account region** | Specifies the AWS region where your Aurora Postgres instance is located (e.g., us-east-1, eu-west-1). | +| **Log groups** | List of CloudWatch log groups to monitor. These are the log groups where Aurora Postgres audit logs are exported. | +| **Filter pattern** | CloudWatch Logs filter pattern to apply. Use "None" to retrieve all logs, or specify a pattern to filter specific log events. | +| **Account ID** | Your AWS account ID (12-digit number). This identifies your AWS account. | +| **Cluster name** | The name of your Aurora Postgres cluster or instance identifier. | +| **Ingestion delay (seconds)** | Default value is 900 seconds (15 minutes). This delay accounts for the time it takes for logs to be available in CloudWatch after being generated. | +| **No-traffic threshold (minutes)** | Default value is 60. If there is no incoming traffic for an hour, S-TAP displays a red status. Once incoming traffic resumes, the status returns to green. | +| **Unmask sensitive value** | Optional boolean flag. When enabled, sensitive values in the audit logs are not masked. | +| **Use Enterprise Load Balancing (ELB)** | Enable this if ELB support is required. | +| **Managed Unit Count** | Number of Managed Units (MUs) to allocate for ELB. | + +**Note:** + +- Ensure that the **profile name** is unique. +- Required credentials must be created before or during profile creation. +- The AWS credentials must have appropriate permissions to read CloudWatch logs. + +--- + +## Testing a Connection + +After creating a profile, you must test the connection to make sure that the provided configuration is valid. + +### Procedure + +1. Select the new profile. +2. From the top menu, click **Test Connection**. +3. If the test is successful, you can proceed to installing the profile. + +--- + +## Installing a Profile + +Once the connection test is successful, you can install the profile on **Managed Units (MUs)** or **Edges**. The parsed +audit logs are sent to the selected Managed Unit or Edge to be consumed by the **Sniffer**. + +### Procedure + +1. Select the profile. +2. From the **Install** menu, click **Install**. +3. From the list of available MUs and Edges that is displayed, select the ones that you want to deploy the profile to. + +--- + +## Uninstalling or reinstalling profiles + +An installed profile can be uninstalled or reinstalled if needed. + +### Procedure + +1. Select the profile. +2. From the list of available actions, select the desired option: **Uninstall** or **Reinstall**. + +--- + diff --git a/docs/KafkaBasedUCs/AwsS3OverCloudwatchKafkaConnectProfile.md b/docs/KafkaBasedUCs/AwsS3OverCloudwatchKafkaConnectProfile.md new file mode 100644 index 000000000..a52682ced --- /dev/null +++ b/docs/KafkaBasedUCs/AwsS3OverCloudwatchKafkaConnectProfile.md @@ -0,0 +1,142 @@ +# Configuring S3 datasource profile for Kafka Connect plug-ins + +Create and configure datasource profiles through central manager for **AWS S3 +over CloudWatch Kafka Connect** plug-ins. + +### Meet S3 over CloudWatch Connect + +* Environments: AWS +* Supported inputs: Kafka connect Cloudwatch 2.0 (pull) +* Supported Guardium versions: + * Guardium Data Protection: Appliance bundle 12.2.2 or later + +Kafka-connect is a framework for streaming data between Apache Kafka and other systems. This connector enables +monitoring of S3 audit logs through CloudWatch. + +## Configuring Amazon AWS CloudTrail to send S3 log files to CloudWatch + +There are different methods for auditing and logging. CloudTrail is used for this example as it supports all required parameters. + +### Procedure + +1. Go to https://console.aws.amazon.com/cloudtrail. +2. Click **Trails** in the left menu. +3. Click **Create trail** and enter the trail name. +4. Under **Storage location**, verify that **Create new S3 bucket** is selected. +5. Under **Log file SSE-KMS encryption**, clear the **Enabled** box. +6. Under **CloudWatch Logs**, check the **Enabled** box. +7. Verify **New** is selected for **Log group**. +8. Under **Log group name**, provide a new log group name. +9. Verify **New** is selected for **IAM Role**. +10. For **Role name**, provide a new role name. +11. Click **Next**. +12. For **Event type**, select **Management events** and **Data events**. +13. Verify that **Read** and **Write** are selected for **API Activity**. +14. In the **Data Events** section, click **Switch to basic event selectors**. +15. Click **Add data event type** > **Data event source** and then select **S3**. +16. Select the S3 buckets that you want to monitor. +17. Click **Next**. +18. Verify that all parameters shown are correct. Then click **Create trail**. + +### Viewing S3 log entries on CloudWatch + +By default, each CloudTrail trail has an associated log group with a name in the format specified during trail creation. You can use this log group, or you can create a new one and associate it with the trail. + +1. On the AWS Console page, open the **Services** menu. +2. Enter `CloudWatch` in the search box. +3. Click **CloudWatch** to redirect to the CloudWatch dashboard. +4. In the left panel, select **Logs**. +5. Click **Log Groups**. + + +## Creating datasource profiles + +You can create a new datasource profile from the **Datasource Profile Management** page. + +### Procedure + +1. Go to **Manage > Universal Connector > Datasource Profile Management** +2. Click the **➕ (Add)** button. +3. You can create a profile by using one of the following methods: + + - To **Create a new profile manually**, go to the **"Add Profile"** tab and provide values for the following fields. + - **Name** and **Description**. + - Select a **Plug-in Type** from the dropdown. For example, `AWS S3 Over Cloudwatch Connect 2.0`. + + - To **Upload from CSV**, go to the **"Upload from CSV"** tab and upload an exported or manually created CSV file + containing one or more profiles. + You can also choose from the following options: + - **Update existing profiles on name match** — Updates profiles with the same name if they already exist. + - **Test connection for imported profiles** — Automatically tests connections after profiles are created. + - **Use ELB** — Enables ELB support for imported profiles. You must provide the number of MUs to be used in the + ELB process. + +**Note:** Configuration options vary based on the selected plug-in. + +## Configuring S3 Over CloudWatch Kafka Connect 2.0 + +The following table describes the fields that are specific to S3 over CloudWatch Kafka Connect 2.0 plugin. + +| Field | Description | +|-----------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Name** | Unique name of the profile. | +| **Description** | Description of the profile. | +| **Plug-in** | Plug-in type for this profile. Select **AWS S3 Over Cloudwatch Connect 2.0**. A full list of available plug-ins are available on the **Package Management** page. | +| **Credential** | Select AWS Credentials or AWS Role ARN. The credential to authenticate with AWS. Must be created in **Credential Management**, or click **➕** to create one. For more information, see [Creating Credentials](https://www.ibm.com/docs/en/SSMPHH_12.x/com.ibm.guardium.doc.stap/guc/guc_credential_management.html). | +| **Kafka Cluster** | Select the appropriate Kafka cluster from the available Kafka cluster list or create a new Kafka cluster. For more information, see [Managing Kafka clusters](https://www.ibm.com/docs/en/SSMPHH_12.x/com.ibm.guardium.doc.stap/guc/guc_kafka_cluster_management.html). | +| **Label** | Grouping label. For example, customer name or ID. | +| **AWS account region** | Specifies the AWS region where your S3 buckets are located (e.g., us-east-1, eu-west-1). | +| **Log groups** | List of CloudWatch log groups to monitor. These are the log groups where S3 audit logs (via CloudTrail) are exported. Format: `/aws/cloudtrail/` | +| **Filter pattern** | CloudWatch Logs filter pattern to apply. Use "None" to retrieve all logs, or specify a pattern to filter specific log events. | +| **Account ID** | Your AWS account ID (12-digit number). This identifies your AWS account. | +| **Cluster name** | The name of your S3 bucket or identifier used to distinguish this data source. | +| **Ingestion delay (seconds)** | Default value is 900 seconds (15 minutes). This delay accounts for the time it takes for logs to be available in CloudWatch after being generated. | +| **No-traffic threshold (minutes)** | Default value is 60. If there is no incoming traffic for an hour, S-TAP displays a red status. Once incoming traffic resumes, the status returns to green. | +| **Unmask sensitive value** | Optional boolean flag. When enabled, sensitive values in the audit logs will not be masked. | +| **Use Enterprise Load Balancing (ELB)** | Enable this if ELB support is required. | +| **Managed Unit Count** | Number of Managed Units (MUs) to allocate for ELB. | + +**Note:** + +- Ensure that the **profile name** is unique. +- Required credentials must be created before or during profile creation. +- The AWS credentials must have appropriate permissions to read CloudWatch logs. + +--- + +## Testing a Connection + +After creating a profile, you must test the connection to ensure the provided configuration is valid. + +### Procedure + +1. Select the new profile. +2. From the top menu, click **Test Connection**. +3. If the test is successful, you can proceed to installing the profile. + +--- + +## Installing a Profile + +Once the connection test is successful, you can install the profile on **Managed Units (MUs)** or **Edges**. The parsed +audit logs are sent to the selected Managed Unit or Edge to be consumed by the **Sniffer**. + +### Procedure + +1. Select the profile. +2. From the **Install** menu, click **Install**. +3. From the list of available MUs and Edges that is displayed, select the ones that you want to deploy the profile to. + +--- + +## Uninstalling or reinstalling profiles + +An installed profile can be uninstalled or reinstalled if needed. + +### Procedure + +1. Select the profile. +2. From the list of available actions, select the desired option: **Uninstall** or **Reinstall**. + +--- + diff --git a/docs/KafkaBasedUCs/AzureDatabricksEventHubKafkaConnect.md b/docs/KafkaBasedUCs/AzureDatabricksEventHubKafkaConnect.md new file mode 100644 index 000000000..0d8cc569b --- /dev/null +++ b/docs/KafkaBasedUCs/AzureDatabricksEventHubKafkaConnect.md @@ -0,0 +1,202 @@ +# Configuring Azure Databricks datasource profiles for JDBC Kafka Connect Plug-ins + +Create and configure datasource profiles through Central Manager for **Azure Databricks JDBC Kafka Connect** plug-ins. + +## Meet Azure Databricks Over JDBC Connect + +* Environments: Azure +* Supported inputs: Kafka connect Azure EventHub 2.0 +* Supported Guardium versions: + * Guardium Data Protection: Appliance bundle 12.2.2 or later. + +Kafka-connect is a framework for streaming data between Apache Kafka and other systems. + +## Configuring the Azure Databricks + +1. Login to https://portal.azure.com. +2. From the **Azure portal** menu or the home page, create an Azure Databricks DB account. + - If you are on the home page, select **Create a resource**. + - On the new page, search for and select **Azure Databricks**. +3. Select **Subscription** and **Resource group**. Keep the other tab values as is. Then click **Review + Create**. +4. Review the settings, and then select **Create**. It takes a few minutes to create the account. Wait for the portal page to display ``Your deployment is complete`` before proceeding to the next step. +5. Go to the Azure Databricks page by clicking **Go to resource**. + +## Azure Event Hub connection + +1. In the search bar, enter ``Event hub``. +2. Select **Create event hubs namespace**. +3. To create a namespace, complete the following steps.
+ a. Select the subscription in which you want to create the namespace.
+ b. Select the resource group that you created in the previous step.
+ c. Enter a unique name for the namespace.
+ d. Select a location for the namespace.
+ e. Choose the appropriate pricing tier. (In this example, we selected **basic**).
+ f. Leave the throughput units (or processing units for standard and premium tier) settings as is.
+ g. Select **Review + Create**. Review the settings and select **Create**.
+ h. Your recently created namespace appears in **resource group**.
+4. To create an event hub, complete the following steps.
+ a. Go to the Event Hubs Namespace page.
+ b. Click **+ Event Hub**.
+ c. Enter a unique name for the event hub.
+ d. Choose the maximum number of partitions that you expect to require during peak usage for this event hub.
+ For example, if you want to generate traffic from 2 DB instances, then choose at least 2 partitions if not more.
+ e. Click **Review + create**.
+ f. Review the settings and click **Create**.
+5. Connection string for an event hub.
+ a. In the list of event hubs, select your event hub.
+ b. On the Event Hubs instance page, go to **Settings** > **Shared access policies** > **Add**.
+ c. Name the policy, click **Manage** to provide permissions, and create the policy.
+ d. Select **Connection string–primary key** from policy (this string is required in the input plugin).
+ +6. Azure Storage Accounts Creation:
+ a. Login to https://portal.azure.com.
+ b. Search Storage accounts in search bar.
+ c. Click **Create**.
+ d. **Basic** Tab:
+     - Select the subscription in which you want to create the storage account.
+     - Select an existing resource group or create a new one.
+     - Enter a unique name for the storage account.
+     - Select the same region for the storage account that you selected for the server.
+      - Choose any performance type.
+     - Select **Geo-redundant(GRS) Redundancy configuration**.
+     - Select **Make read access to data**.
+     - Click **Next:Advance**.
+ e. **Advanced** tab:
+     - **Require secure transfer** should already be selected.
+     - **Allow enabling public access** should already be selected.
+     - **Enable storage account key access** should already be selected.
+     - Select the latest TLS version.
+     - Permitted scope should display the default value (from any storage account).
+     - The remaining parameters (Hierarchical Namespace, Access protocols, Blob storage, and Azure Files) should display the default values provided by Azure.
+     - Click **Next:Networking**.
+ f. **Networking** tab:
+     - Enable public access from all networks for **Network access**.
+     - Select **Microsoft network routing** for **Routing preference**.
+     - Click **Next:Data protection**.
+ g. **Data protection** tab:
+     - Keep the default values provided by Azure.
+     - Click **Next:Encryption**.
+ h. **Encryption** tab:
+     - **Encryption type** should already be set to **Microsoft-managed key(MMK)**.
+     - **Enable support for customer-managed keys** should be set to the default value (**blobs and files**).
+     - By default, **Infrastructure encryption** should not be enabled.
+     - Click **Next:Tags**.
+ i. For the **Tags** tab, make no changes and click **Next:Review**.
+ j. Click **Create** after you review all the parameters.
+ + + +## Link event hub to Databricks + +1. Login to https://portal.azure.com. +2. Navigate to your Azure Databricks. Open the **Diagnostic settings** pane under the **Monitoring** section to create a new diagnostic setting. +3. In the **Diagnostic settings** pane, fill in the form with your preferred categories. +4. Select your categories details, and then send your logs to your preferred destination. In this example, **Stream to an event hub** is selected. Then put your prefered event hub information in. +5. Launch your Databricks Workspace and go to **Profile** at top right corner. +6. Click **Settings** > **Advanced**, then search for **Verbose Audit Logs** and turn it on. + +## Connecting to the Azure Databricks + +### Insert/Update data through Data Explorer + +1. Login to https://portal.azure.com. +2. Navigate to your Azure Databricks. Launch Workspace. +3. Under **SQL Editor**, you can run sql command by creating new quert scripts. + + +#### Limitations +1. The following fields are not available in the audit logs from Azure Databricks: **Database name**, **ProtocolVersion**, **AppUserName**, **Client mac**, **Common Protocol**, **Os User**, **ClientOs**, **ServerOs**. +2. The log with sql execution does not display **Client IP**, but it can be found in another log with the **commandFinish** action. +3. Eventhub takes 10~30 minutes to receive raw logs from Databricks. The same delay time for Guardium is expected. +4. The Databricks auditing does not audit authentication failure (Login Failed) operations. + +## Creating datasource profiles + +You can create a new datasource profile from the **Datasource Profile Management** page. + +### Procedure + +1. Go to **Manage > Universal Connector > Datasource Profile Management** +2. Click the **➕ (Add)** button. +3. You can create a profile by using one of the following methods: + + * To **Create a new profile manually**, go to the **"Add Profile"** tab and provide values for the following fields. + * **Name** and **Description**. + * Select a **Plug-in Type** from the dropdown. For example, **Azure Databricks Over JDBC Kafka Connect 2.0**. + + * To **Upload from CSV**, go to the **"Upload from CSV"** tab and upload an exported or manually created CSV file + containing one or more profiles. You can also choose from the following options: + * **Update existing profiles on name match** — Updates profiles with the same name if they already exist. + * **Test connection for imported profiles** — Automatically tests connections after profiles are created. + * **Use ELB** — Enables ELB support for imported profiles. You must provide the number of MUs to be used in the + ELB process. + +**Note:** Configuration options vary based on the selected plug-in. + +## Configuration: JDBC Kafka Connect 2.0-based Plugins + +The following table describes the fields that are specific to JDBC Kafka Connect 2.0 and similar plugins. + +| Field | Description | +|-------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Name** | Unique name of the profile. | +| **Description** | Description of the profile. | +| **Plug-in** | Plug-in type for this profile. A full list of available plug-ins are available on the **Package Management** page. | +| **Credential** | The credential to authenticate with the datasource. Must be created in **Credential Management**, or click **➕** to create one. | +| **Kafka Cluster** | Kafka cluster to deploy the universal connector. | +| **Label** | Grouping label. For example, customer name or ID. | +| **JDBC Driver Library** | JDBC driver for the database. | +| **Port** | Port that is used to connect to the database. | +| **Hostname** | Hostname of the database. | +| **Query** | SQL query that is used to extract audit logs. | +| **Service Name / SID** | The database **service name** or **SID**. | +| **Initial Time** | Initial polling time for audit logs. | +| **No Traffic Threshold** | Threshold setting for inactivity detection. | +| **Connection URL** | Full JDBC connection string. Format varies by database type.
For example, `jdbc:sqlserver://myserver.database.windows.net:1433;database=mydb;encrypt=true;`. | +| **Use Enterprise Load Balancing (ELB)** | Enable this if ELB support is required. | +| **Managed Unit Count** | Number of Managed Units (MUs) to allocate for ELB. | + +**Note:** +- Depending on the plugin type, the configuration may require either: + - A **Connection URL**, or + - Separate fields for **Hostname**, **Port**, and **Service Name / SID** +- Ensure that the **profile name** is unique. +- Required credentials must be created before or during profile creation. + +--- + +## Testing a Connection + +After creating a profile, you must test the connection to ensure the provided configuration is valid. + +### Procedure + +1. Select the new profile. +2. From the top menu, click **Test Connection**. +3. If the test is successful, you can proceed to installing the profile. + +--- + +## Installing a Profile + +Once the connection test is successful, you can install the profile on **Managed Units (MUs)** or **Edges**. The parsed +audit logs are sent to the selected Managed Unit or Edge to be consumed by the **Sniffer**. + +### Procedure + +1. Select the profile. +2. From the **Install** menu, click **Install**. +3. From the list of available MUs and Edges that is displayed, select the ones that you want to deploy the profile to. + +--- + +## Uninstalling or reinstalling profiles + +An installed profile can be uninstalled or reinstalled if needed. + +### Procedure + +1. Select the profile. +2. From the list of available actions, select the desired option: **Uninstall** or **Reinstall**. + +--- diff --git a/docs/KafkaBasedUCs/AzureMsSQLJDBCKafkaConnect.md b/docs/KafkaBasedUCs/AzureMsSQLJDBCKafkaConnect.md new file mode 100644 index 000000000..fdac2b289 --- /dev/null +++ b/docs/KafkaBasedUCs/AzureMsSQLJDBCKafkaConnect.md @@ -0,0 +1,221 @@ +# Configuring Azure SQL datasource profiles for JDBC Kafka Connect Plug-ins + +Create and configure datasource profiles through central manager for **Azure MSSQL JDBC Kafka Connect** plug-ins. + +## Meet Azure SQL Over JDBC Connect + +* Environments: Azure +* Supported inputs: Kafka connect JDBC 2.0 (pull) +* Supported Guardium versions: + * Guardium Data Protection: Appliance bundle 12.2.2 or later + +Kafka-connect is a framework for streaming data between Apache Kafka and other systems. + +## Configuring the AzureSQL service + +There are two ways to get Azure SQL audit data: + +1. Universal Connector using Azure object storage and a JDBC feed +2. Guardium Streams using an Azure Event Hub +This plugin uses object storage. + +If you want more insights into these options, contact the Guardium Offering Managers. + +### Procedure +1. Go to https://portal.azure.com/. +2. In the search bar, enter **AzureSQL**. +3. Click **Create**. +4. Select the **SQL databases** option. +5. Choose the **Resource Type** depending on your requirements (**Single database** or **Elastic pool**). Then click **Create**. +6. Select **Existing Resource group** or **Create New one**. +7. Provide a **Database Name**. +8. Click the **Create New** option under the **Server** field. +9. Provide a **Server Name** and select the appropriate **Location**. +10. Select **Authentication method** based on your requirement. +11. Enter a **Server admin login name** and **Password**. Then click **OK**. +12. If needed, select the **Compute + storage** configuration. +13. Click **Review + Create**. +14. Verify the configuration and then click **Create**. +15. From the search bar, navigate to **Storage Account** and create a new storage account.
+ a. Select the resource group that you created.
+ b. Enter a **Storage Account Name**.
+ c. Select the appropriate location.
+ d. Choose additional configurations, if needed.
+ e. Click **Review + create** > **Create**.
+ +## Enabling Auditing + +1. Click on the **Menu** button and go to the **Resource groups** tab. +2. In the **Resource groups** tab, click **Resource Group**.
+ a. Select the SQL Server that you created.
+ b. Expand the **Show Firewall setting** options and add **Client IPaddress** and **Public IPaddress** of the gmachine that is required to capture traffic.
+ c. Add **Client IPaddress** by clicking ```Add Client IP``` Button.
+ Use the `curl ipinfo.io/ip` command to obtain the public IPaddress of your gmachine.
+ d. Click **Save**.
+5. To enable auditing, complete the following steps.
+ a. Select your SQL Database.
+ b. From the search bar, navigate to **Auditing**.
+ c. In **Auditing**, click **Enable Azure SQL Auditing**.
+ d. Select the **Storage Audit log destination** check box.
+ e. Select the **Storage Account** that you created.
+ f. Go to **Advanced properties**, and select **Retention (Days)**.
+ g. Click **Save**.
+ + +## Connecting to AzureSQL Database + +1. Start the SQL Server Management Studio and provide connection details. Enter the **server name**, **username**, and the **master password** that you set while creating the database. +2. Click **Connect**. + + +## Viewing audit logs + +Use the following query and enter your **storage-account-name**, **server_instance_name**, **DB-NAME** values. + + ``` + SELECT event_time,succeeded,session_id,database_name,client_ip,server_principal_name,application_name,statement,server_instance_name,host_name,DATEDIFF_BIG(ns, '1970-01-01 00:00:00.00000', event_time) AS updatedeventtime,additional_information FROM sys.fn_get_audit_file('https://.blob.core.windows.net/sqldbauditlogs//', DEFAULT, DEFAULT) where action_id='BCM' and statement not like '%xproc%' and statement not like '%SPID%' and statement not like '%DEADLOCK_PRIORITY%' and application_name not like '%Microsoft SQL Server Management Studio - Transact-SQL IntelliSense%' and DATEDIFF_BIG(ns, '1970-01-01 00:00:00.00000', event_time) > 0;``` + +**Note:** Create a non-admin user to access the audit table without exposing admin credentials. Create the non-admin user in the Azure Portal directly. + +1. Log in to the database using admin credentials and run the following queries. + + +``` +sql +CREATE USER WITH PASSWORD = ''; +ALTER ROLE db_datareader ADD MEMBER ; +GRANT VIEW DATABASE STATE TO ; +GRANT VIEW DATABASE SECURITY AUDIT TO ; + ``` + +2. Use the user credentials that you created in the previous step for the JDBC connection. + +``` +properties +jdbc_user => "" +jdbc_password => "" + ``` + + +## Finding the Enrollment ID + +**Note:** You need to be an Enterprise Administrator to access the Enrollment ID. + +1. Log in to your Azure Enterprise account at https://ea.azure.com. + +2. In the top left corner, you can see your **Enrollment ID**. + +3. Copy the Enrollment ID and save it for later use. + +## Getting the JDBC Connection string + +1. Click on **Database**. + +2. In the search bar, enter **connection string**. + +3. Select **JDBC** and copy the string. + +4. Enter this connection string value to the JDBC Input plugin **jdbc_connection_string** parameter. + + +## Limitations +• The azureSQL plug-in does not support IPV6. + +• The azureSQL auditing does not audit authentication failure (Login Failed) operations. + +• AzureSQL audit-records does not audit **serverIP**. The **serverIp** value is hardcoded to ``0.0.0.0``. + +• The following important fields are not mapped with AzureSQL audit logs: **create user with password** operation, **OS USER** field. + +• AzureSQL auditing does not audit operations perform by **Beekeeper Studio Tools**. + +## Creating datasource profiles + +You can create a new datasource profile from the **Datasource Profile Management** page. + +### Procedure + +1. Go to **Manage > Universal Connector > Datasource Profile Management** +2. Click the **➕ (Add)** button. +3. You can create a profile by using one of the following methods: + + * To **Create a new profile manually**, go to the **"Add Profile"** tab and provide values for the following fields. + * **Name** and **Description**. + * Select a **Plug-in Type** from the dropdown. For example, `Azure SQL Over JDBC Kafka Connect 2.0`. + + * To **Upload from CSV**, go to the **"Upload from CSV"** tab and upload an exported or manually created CSV file + containing one or more profiles. You can also choose from the following options: + * **Update existing profiles on name match** — Updates profiles with the same name if they already exist. + * **Test connection for imported profiles** — Automatically tests connections after profiles are created. + * **Use ELB** — Enables ELB support for imported profiles. You must provide the number of MUs to be used in the + ELB process. + +**Note:** Configuration options vary based on the selected plug-in. + +## Configuration: JDBC Kafka Connect 2.0-based Plugins + +The following table describes the fields that are specific to JDBC Kafka Connect 2.0 and similar plugins. + +| Field | Description | +|-------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Name** | Unique name of the profile. | +| **Description** | Description of the profile. | +| **Plug-in** | Plug-in type for this profile. A full list of available plug-ins are available on the **Package Management** page. | +| **Credential** | The credential to authenticate with the datasource. Must be created in **Credential Management**, or click **➕** to create one. | +| **Kafka Cluster** | Kafka cluster to deploy the universal connector. | +| **Label** | Grouping label. For example, customer name or ID. | +| **JDBC Driver Library** | JDBC driver for the database. | +| **Port** | Port that is used to connect to the database. | +| **Hostname** | Hostname of the database. | +| **Query** | SQL query that is used to extract audit logs. | +| **Service Name / SID** | The database **service name** or **SID**. | +| **Initial Time** | Initial polling time for audit logs. | +| **No Traffic Threshold** | Threshold setting for inactivity detection. | +| **Connection URL** | Full JDBC connection string. Format varies by database type.
For example, `jdbc:sqlserver://myserver.database.windows.net:1433;database=mydb;encrypt=true;`. | +| **Use Enterprise Load Balancing (ELB)** | Enable this if ELB support is required. | +| **Managed Unit Count** | Number of Managed Units (MUs) to allocate for ELB. | + +**Note:** +- Depending on the plugin type, the configuration may require either: + - A **Connection URL**, or + - Separate fields for **Hostname**, **Port**, and **Service Name / SID** +- Ensure that the **profile name** is unique. +- Required credentials must be created before or during profile creation. + +--- + +## Testing a Connection + +After creating a profile, you must test the connection to ensure the provided configuration is valid. + +### Procedure + +1. Select the new profile. +2. From the top menu, click **Test Connection**. +3. If the test is successful, you can proceed to installing the profile. + +--- + +## Installing a Profile + +Once the connection test is successful, you can install the profile on **Managed Units (MUs)** or **Edges**. The parsed +audit logs are sent to the selected Managed Unit or Edge to be consumed by the **Sniffer**. + +### Procedure + +1. Select the profile. +2. From the **Install** menu, click **Install**. +3. From the list of available MUs and Edges that is displayed, select the ones that you want to deploy the profile to. + +--- + +## Uninstalling or reinstalling profiles + +An installed profile can be uninstalled or reinstalled if needed. + +### Procedure + +1. Select the profile. +2. From the list of available actions, select the desired option: **Uninstall** or **Reinstall**. + +--- diff --git a/docs/KafkaBasedUCs/AzureMySQLEventHubKafkaConnect.md b/docs/KafkaBasedUCs/AzureMySQLEventHubKafkaConnect.md new file mode 100644 index 000000000..536288f07 --- /dev/null +++ b/docs/KafkaBasedUCs/AzureMySQLEventHubKafkaConnect.md @@ -0,0 +1,298 @@ +# Configuring Azure MySQL datasource profile for Kafka Connect Plug-ins + +Create and configure datasource profiles through central manager for **Azure +MySQL over Event Hub Kafka Connect** plug-ins. + +### Meet Azure MySQL over Event Hub Connect + +* Tested versions: 8.0.42 (Flexible Server) +* Environment: Azure +* Supported inputs: Kafka connect Azure EventHub 2.0 +* Supported Guardium versions: + * Guardium Data Protection: Appliance bundle 12.2.2 or later + +Kafka-connect is a framework for streaming data between Apache Kafka and other systems. This connector enables +monitoring of Azure MySQL audit logs through Azure Event Hub. + +## Configuring the Azure MySQL Service + +You can use the following methods to obtain Azure MySQL audit data: + +1. Azure Event Hub +2. Azure Storage +3. Log Analytics Workspace + +For this procedure, we are using Azure Event Hub. + +### Procedure + +1. Login to https://portal.azure.com. +2. In the search bar, search for and select **Azure Database for MySQL servers**. +3. Click **Create**. +4. On the Select Azure Database for MySQL deployment option page, select **Flexible server**. +5. On the **Basics** tab, provide the following details. + - **Subscription**: Select your subscription name. + - **Resource group**: Select existing resource group or create new one. + - **Server name**: User need to provide a unique name that identifies your flexible server. + - **Region**: Select the region or location. + - **MySQL version**: Choose latest Version (here 8.0). + - **Workload type**: User can choose size as per the requirement. + - **Compute + storage**: Server configuration can be changed as per the requirement. + - **Availability zone**: No preference, can be specified as per your requirement. + - **Authentication method**: MySQL authentication only. + - **Admin username**: Provide Username. + - **Password**: Provide Password. +6. Under the **Networking** tab, for **Connectivity method**, select **Public access**. +7. For configuring Firewall rules, select **Add current client IP address** and check the checkbox to allow public access + from any azure service. +8. Click **Review + create** to review your flexible server configuration. +9. Verify the configuration and then click **Create**. +10. When the deployment is completed, the server is ready for use. + +## Enabling Audit Logs + +1. Select your Azure Database for MySQL server. +2. Under **Settings**, select the **Server** parameter. +3. Update the **audit_log_enabled** parameter to ``ON``. +4. Select the event types to be logged by updating the **audit_log_events** parameter as shown below. + - **CONNECTION** includes: + - Connection initiation (successful or unsuccessful). + - User reauthentication with different user/password during session. + - Connection termination. + - **GENERAL** includes: + - DML_SELECT, DML_NONSELECT, DML, DDL, DCL, and ADMIN. + +## Creating and connecting the Azure Event Hub namespace + +### Creating Azure Event Hub namespace + +1. Login to https://portal.azure.com. +2. Search for ``event hub`` in search bar. +3. Click **Create event hubs namespace**. +4. To create a namespace, complete the following steps.
+ a. Select the **Subscription** in which you want to create the namespace.
+ b. Select the **Resource group** that you created in the previous step.
+ c. Enter a unique name for the namespace.
+ d. Select the same location for the namespace that you selected for the server.
+ e. Choose the pricing tier based on your requirement.
+ f. Set the throughput units (or processing units for standard and premium tiers) as ``10`` to prevent data loss. You can + update the throughput units as per your requirement (some pricing tiers provide Auto-inflate feature). In Azure, 1 + throughput unit handles incoming data of up to 1 MB/second/1000 events and outgoing data of up to 2MB/second/4096 + events.
+ g. Click **Review + Create** at the bottom of the page.
+ h. Review the settings and select **Create**.
+ i. After the namespace is created, it appears in **Resource group**.
+ +### Azure Event Hub Connection + +1. To create an event hub, complete the following steps.
+ a. Go to the Event Hubs Namespace page.
+ b. To add an event hub, click **+ Event Hub**.
+ c. Enter unique name for event hub.
+ d. Choose at least as many partitions as required during the peak load of your application for that particular event hub. For example, if you want to generate traffic from 2 database instances, the partition count should be at least 2 or more.
+ e. Click **Review+create**.
+ f. Review the settings and click **Create**.
+ +## Creating Azure Storage Accounts + +1. Login to https://portal.azure.com. +2. Search for ``Storage accounts`` in the search bar. +3. Click **Create**. +4. **Basic** Tab: + - Select the Subscription in which you want to create the Storage account. + - Select or create new Resource group. + - Enter a unique name for Storage account. + - Select same region for the storage account which you selected for server. + - Choose any Performance type. + - Select Geo-redundant(GRS) Redundancy configuration. + - Select Make read access to data option. + - Click **Next:Advance**. +5. **Advanced** tab: + - Require secure transfer option should be selected. + - Allow enabling public access option should be selected. + - Enable storage account key access option should be selected. + - Select latest TLS version. + - Permitted scope should be the default value(From any storage account). + - Other parameters Hierarchical Namespace, Access protocols, Blob storage and Azure Files should be default value + provided by azure. + - Click **Next:Networking**. +6. **Networking** tab: + - Enable public access from all networks for Network access. + - Select the **Microsoft network routing** option for Routing preference. + - Click **Next:Data protection**. +7. **Data protection** tab: + - Use the default values provided by Azure. + - Click **Next:Encryption**. +8. **Encryption** tab: + - **Encryption type** is the Microsoft-managed key(MMK). + - Enable support for customer-managed keys option should be by default value (blobs and files). + - By default, infrastructure encryption should not be enabled. + - Click **Next:Tags**. +9. On the **Tags** tab no need to select anything and click **Next:Review**. +10. Click on Create button after review all the parameters. + +## Stream logs to an Event Hub + +1. Login to https://portal.azure.com. +2. Go to server in your Azure Portal. +3. From **Monitoring**, select **Diagnostics settings**. +4. To change existing settings, select **Edit setting**. +5. To add new settings, select **Add diagnostics setting**.
+ a. Enter a name for the setting.
+ b. Select **MySQL Audit Logs** from categories.
+ c. In **Destination details**, select **Archive to a storage account**.
+ d. In **Archive to a storage account**, select **Storage account as created above**.
+ e. In **Destination details**, choose **Stream to an event hub**.
+ f. In **Stream to event hub**, select **namespace name and event hub name as created above**. Keep the event hub policy name as is.
+ g. Click **Save** to save the setting.
+6. After about 15 minutes, verify that the events are displayed in your event hub. + +### Configuring multiple collectors + +When UCs are configured on two separate Collectors, additional configurations are needed to monitor traffic from a single Event Hub. + +1. Create a namespace in Azure Event Hub following the standard procedure, but select the **Standard pricing tier** instead of Basic in the pricing tier configuration. +2. After the namespace is created, create an Event Hub and generate a connection string as described in the previous steps. +3. After successfully creating the Event Hub, add a consumer group to the event hub.
+ a. From the list of Event Hubs, select your Event Hub.
+ b. On the Event Hub instance page, select **Consumer Groups** from the **Entities** section. Then click **Add**.
+ c. Enter a name for the consumer group and create it.
+4. Configure log streaming to the Event Hub as described in the previous steps. +5. For gmachine, use the following consumer group names in the connector configurations.
+ a. Machine 1: Use the consumer group name ``$Default``.
+ b. Machine 2: Use the name of the consumer group created in step 3.
+ c. Keep all other configuration settings as is.
+ +## Connecting to Azure MySQL Database + +1. Go to the server and click **Connect** in the Overview page. +2. Enter the password that you have set while creating the server and click **Enter**. +3. You can successfully connect Mysql and execute queries. + +## Limitations + +- The audit log does not contain a server IP. The **Server IP** is set to default value `0.0.0.0`. +- Error events will cause a duplicated success events in Guardium due to duplicate events in the Azure EventHub audit + log. +- Azure EventHub is not capturing **Syntactical error** queries logs, **Login Failed** logs, and logs (i.e., when using + **az commands** (example below) and **REST API**). + - Example: + `az mysql flexible-server db create --resource-group --server-name --database-name ` +- There are certain [limited privileges](https://learn.microsoft.com/en-us/azure/mysql/how-to-create-users) given by + Azure MYSQL to users. +- We are getting below extra logs while executing `USE` command: + - show tables + - show databases +- The following important fields cannot be mapped with Azure MySQL logs: + - Source Program + - Client Host Name + - Server Port +- Database name is not available in **General logs**, it only available at the time of **Disconnect** and **Connect** +(We must use **database name** at the time of connection to get the database name). +- Eventhub capturing identical duplicate logs for each query and same has been carrying to Guardium reports. +- **Database name** and **Service name** are not identical when user execute queries using Third party tool (DB + Visualizer/ MySQL Workbench). + +## Creating datasource profiles + +You can create a new datasource profile from the **Datasource Profile Management** page. + +### Procedure + +1. Go to **Manage > Universal Connector > Datasource Profile Management** +2. Click the **➕ (Add)** button. +3. You can create a profile by using one of the following methods: + + * To **Create a new profile manually**, go to the **"Add Profile"** tab and provide values for the following fields. + * **Name** and **Description**. + * Select a **Plug-in Type** from the dropdown. For example, `Azure MySQL Over Event Hub Connect 2.0`. + + * To **Upload from CSV**, go to the **"Upload from CSV"** tab and upload an exported or manually created CSV file + containing one or more profiles. You can also choose from the following options: + * **Update existing profiles on name match** — Updates profiles with the same name if they already exist. + * **Test connection for imported profiles** — Automatically tests connections after profiles are created. + * **Use ELB** — Enables ELB support for imported profiles. You must provide the number of MUs to be used in the + ELB process. + +**Note:** Configuration options vary based on the selected plug-in. + +## Configuring Azure MySQL Over Event Hub Kafka Connect 2.0 + +The following table describes the fields that are specific to Azure Event Hub Kafka Connect 2.0 plugin. + +| Field | Description | +|------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Name** | Unique name of the profile. | +| **Description** | Description of the profile. | +| **Plug-in** | Plug-in type for this profile. **Select Azure MySQL Over Event Hub Connect 2.0**. A full list of available plug-ins are available on the **Package Management** page. | +| **Credential** | The credential to authenticate with Azure Event Hub. Must be created in **Credential Management**, or click **➕** to create one. For more information, see [Creating Credentials](https://www.ibm.com/docs/en/SSMPHH_12.x/com.ibm.guardium.doc.stap/guc/guc_credential_management.html). | +| **Kafka Cluster** | Select the appropriate Kafka cluster from the available Kafka cluster list or create a new Kafka cluster. For more information, see [Managing Kafka clusters](https://www.ibm.com/docs/en/SSMPHH_12.x/com.ibm.guardium.doc.stap/guc/guc_kafka_cluster_management.html). | +| **Label** | Grouping label. For example, customer name or ID. | +| **Event Hub Connection String** | Primary connection string from the Event Hub shared access policy. This connection string is obtained from the Event Hub namespace. | +| **Storage Connection String** | Connection string for the Azure Storage Account. Required when reading from multiple Event Hubs to track offsets. | +| **Consumer Group** | Consumer group name for the Event Hub. Use `$Default` for single collector or create custom consumer groups for multiple collectors. | +| **Event Hub Partition Count** | Number of partitions in the Event Hub. This value should match or exceed the number of database instances generating traffic. | +| **Enrollment ID** | Unique identifier for the Azure resource enrollment. | +| **No-traffic threshold (minutes)** | Default value is 60. If there is no incoming traffic for an hour, S-TAP displays a red status. Once incoming traffic resumes, the status returns to green. | +| **Use Enterprise Load Balancing (ELB)** | Enable this if ELB support is required. | +| **Managed Unit Count** | Number of Managed Units (MUs) to allocate for ELB. | + + **Note:** + +- Ensure that the **profile name** is unique. +- Required credentials must be created before or during profile creation. +- The Azure credentials must have appropriate permissions to read from Event Hub. +- The partition count must be configured based on expected traffic volume. + +--- + +## Azure Credential Configuration + +When creating credentials for Azure Event Hub, provide the following information: + +| Field name | Description | +|-----------------------|-----------------------------------------------------------------| +| **Name** | A unique credential name | +| **Description** | A description for your credential | +| **Credential Type** | Azure Event Hub Credentials `Azure Event Hub Connection String` | +| **Connection String** | Event Hub connection string with manage permissions | + +--- + +--- + +## Testing a Connection + +After creating a profile, you must test the connection to ensure the provided configuration is valid. + +### Procedure + +1. Select the new profile. +2. From the top menu, click **Test Connection**. +3. If the test is successful, you can proceed to installing the profile. + +--- + +## Installing a Profile + +Once the connection test is successful, you can install the profile on **Managed Units (MUs)** or **Edges**. The parsed +audit logs are sent to the selected Managed Unit or Edge to be consumed by the **Sniffer**. + +### Procedure + +1. Select the profile. +2. From the **Install** menu, click **Install**. +3. From the list of available MUs and Edges that is displayed, select the ones that you want to deploy the profile to. + +--- + +## Uninstalling or reinstalling profiles + +An installed profile can be uninstalled or reinstalled if needed. + +### Procedure + +1. Select the profile. +2. From the list of available actions, select the desired option: **Uninstall** or **Reinstall**. + +--- diff --git a/docs/KafkaBasedUCs/AzurePostgreSQLEventHubKafkaConnect.md b/docs/KafkaBasedUCs/AzurePostgreSQLEventHubKafkaConnect.md new file mode 100644 index 000000000..b106d91ce --- /dev/null +++ b/docs/KafkaBasedUCs/AzurePostgreSQLEventHubKafkaConnect.md @@ -0,0 +1,268 @@ +# Configuring Azure PostgreSQL datasource profiles for Kafka Connect Plug-ins + +Create and configure datasource profiles through central manager for **Azure +PostgreSQL over Event Hub Kafka Connect** plug-ins. + +### Meet Azure PostgreSQL over Event Hub Connect + +* Tested versions: 17.9 (Flexible Server) +* Environment: Azure +* Supported inputs: Kafka connect Azure EventHub 2.0 +* Supported Guardium versions: + * Guardium Data Protection: Appliance bundle 12.2.2 or later + +Kafka-connect is a framework for streaming data between Apache Kafka and other systems. This connector enables +monitoring of Azure PostgreSQL audit logs through Azure Event Hub. + +## Configuring the Azure PostgreSQL service + +You can retrieve Azure PostgreSQL audit data in the following ways: + +1. Azure Event Hub +2. Azure Storage +3. Log Analytics Workspace +4. Azure Partner Solution + +This plug-in uses Azure Event Hub as the data streaming service. + +### Procedure + +There are multiple ways to install a Postgres server. For this example, we assume that you already have a working Azure PostgreSQL setup. For more information about the Azure Postgres setup, see [quickstart-create-server-portal](https://learn.microsoft.com/en-us/azure/postgresql/flexible-server/quickstart-create-server-portal). + +**Note:** You can choose between a Single Server or a PostgreSQL Flexible Server deployment. The Single Server option is on the retirement path and will be deprecated in the future. For more information about the retirement schedule, see [Microsoft documentation](https://learn.microsoft.com/en-us/azure/postgresql/migrate/whats-happening-to-postgresql-single-server?wt.mc_id=searchAPI_azureportal_inproduct_rmskilling&sessionId=d1a1e6c6a39842e1bc0191329167d1c3). + +## Enabling Auditing + +1. On the Database auditing page, go to **Settings** and select **server** parameter. +2. Search for **shared_preload_libraries** in server parameter. +3. Select **shared_preload_libraries** as PGAUDIT and save. +4. Go to overview and restart the server to apply the changes. +5. After installation of pgAudit, you can configure its parameters to start logging. +6. On the Database auditing page, go to **Settings** > **server parameters** and set the following parameters. + a. **log_checkpoints** = ``off`` + b. **log_error_verbosity** = ``VERBOSE`` + c. **log_line_prefix** = Specify as based on your requirement but should include timestamp, client ip, client port, database + username, database name, process id, application name, sql state. For more information, + see [Error Reporting and Logging](https://www.postgresql.org/docs/current/runtime-config-logging.html#GUC-LOG-LINE-PREFIX). + d. **pgaudit.log** = ``DDL,FUNCTION,READ,WRITE,ROLE`` + e. **pgaudit.log_catalog** = ``off`` + f. **pgaudit.log_client** = ``off`` + g. **pgaudit.log_parameter** = ``off`` +7. Click **Save**. + +## Configuring Azure Event Hub + +### Azure Event Hub Connection + +1. Search for Storage Accounts in the search bar of the Azure portal.
+ (If you need your plug-in to read events from multiple Event Hubs, you need a storage account.)
+ a. Click **Create**.
+ b. Enter the required details in the presented form.
+ c. Review the details, and then create the storage account.
+ d. Go into the newly created storage account.
+ e. In the menu, go to **Security + networking** > **Access keys**.
+ f. Choose any key (though key1 is preferable), and click **Show** in **Connection String**.
+ g. Copy the presented connection string and save it somewhere.
+ +2. In the search bar, enter and select ``event hub``. +3. Select **Create** to create event hubs namespace. +4. To create a namespace, complete the following steps.
+ a. Select the **Subscription** in which you want to create the namespace.
+ b. Select the **Resource group** that you created in the previous step.
+ c. Enter a unique name for the namespace.
+ d. Select a location for the namespace.
+ e. Choose the pricing tier based on your requirement.
+ f. Set the throughput units (or processing units for standard and premium tiers) as ``10`` to prevent data loss. You can + update the throughput units as per your requirement (some pricing tiers provide Auto-inflate feature). In Azure, 1 + throughput unit handles incoming data of up to 1 MB/second/1000 events and outgoing data of up to 2MB/second/4096 + events.
+ g. Select **Review + Create**.
+ h. Review the settings and select **create**.
+ i. After successful creation, the recently created namespace appears in the resource group.
+ +5. To create an event hub, complete the following steps. + 1. Go to the Event Hubs Namespace page. + 2. Click **+ Event Hub** to add event hub. + 3. Enter a unique name for the event hub, then select **create**. + +6. Connection string for an eventhub, complete the following stpes. + a. From the list of event hubs, select your event hub.
+ b. On the Event Hubs instance page, go to **Settings** > **Shared access policies** > **Add**.
+ c. In shared access policies, click **Add**.
+ d. Enter a policy name and select **Manage** from the permission. Then create the policy.
+ e. Take the value of the primary key - connection string from the policy (this is required for the connector + configuration).
+ f. The value of the primary key - connection string will be used in the connector configuration.
+ +7 Stream logs to an Event Hub
+ a. Login to https://portal.azure.com.
+ b. Go to server in your Azure Portal.
+ c. From **Monitoring**, select **Diagnostics settings**.
+ d. To change existing settings, select **Edit setting**.
+ e. To add new settings, select **Add diagnostics setting**.
+       i. Enter a name for the setting.
+       ii. Select **PostgreSQL Server Logs** from categories.
+       iii. In **Destination details**, choose **Stream to an event hub**.
+       iv. In **Stream to event hub**, select the **namespace name** and **event hub name** as created above*. Keep the event hub policy name as is.
+       v. Click **Save**.
+ +8. After about 15 minutes, verify that the events are displayed in your event hub. + + +### Configuring multiple collectors + +When UCs are configured on two separate Collectors, additional configurations are needed to monitor traffic from a single Event Hub. + +1. Create a namespace in Azure Event Hub following the standard procedure, and select the **Standard pricing tier** instead of Basic in the pricing tier configuration. +2. After the namespace is created, create an Event Hub and generate a connection string as described in the previous steps. +3. After successfully creating the Event Hub, add a consumer group to the event hub.
+ a. From the list of Event Hubs, select your Event Hub.
+ b. On the Event Hub instance page, go to **Entities** > **Consumer group**. Then click **Add**.
+ c. Enter a name for the consumer group and create it.
+4. Configure log streaming to the Event Hub as described in the previous steps. +5. For gmachine, use the following consumer group names in the connector configurations.
+ a. Machine 1: Use the consumer group name ``$Default``.
+ b. Machine 2: Use the name of the consumer group created in step 3.
+ c. Keep all other configuration settings as is.
+ +**Note:** + +The following recommendations apply when configuring the plug-in to read from multiple event hubs. These guidelines are relevant in two scenarios: +- When multiple connection strings are specified in event_hub_connections. +- When multiple universal connector configurations exist on a single Guardium machine. + +* When there is more than one connection string provided in the **event_hub_connections** parameter, define both + **storage_connection** and **consumer_group** parameters. This helps to differentiate the files where timestamp offsets are + written. The **azure_event_hubs** input does not store the offset locally. + +Not following this recommendation may result in data loss. + +## Connecting to Azure PostgreSQL Database + +1. Start the psql and provide connection details. +2. Enter the **server name** from the overview window of the server in the Azure portal. +3. Enter the database name as ``postgres``. +4. In the **Port** field, enter ``5432``. +5. Provide the **username**. From the Azure portal, click **overview** and copy the admin username and ' + password that you set when you created the database. +6. Click **Enter**. +7. Run the following command to give access to pgaudit. + ```sql + GRANT pg_read_all_settings TO ; + -- (use admin-username which is given at the time of creation of database) + -- eg: GRANT pg_read_all_settings TO postgres; + ``` + +## Limitations + +- The Azure PostgreSQL plug-in does not support IPV6. +- The audit log does not contain a server IP. The **Server IP** is set to default value `0.0.0.0`. +- For failed logins, **SQL string** and **Source Program** is not available. +- Multiple records related to session timeout with `SESSION_ERROR` are seen in the exception log report. +- For primary or foreign key constraints violations, entries are added to both the SQL error report and the full SQL + report. +- The source program appears blank in the report for some clients (in this case, values appear for psql and + pgadmin but visual studio is blank). +- **Client Host Name** and **Server Port** is not available in the audit logs. +- SQL string that caused the Exception is not available in exception log report for sql errors. + +## Creating datasource profiles + +You can create a new datasource profile from the **Datasource Profile Management** page. + +### Procedure + +1. Go to **Manage > Universal Connector > Datasource Profile Management** +2. Click the **➕ (Add)** button. +3. You can create a profile by using one of the following methods: + + * To **Create a new profile manually**, go to the **"Add Profile"** tab and provide values for the following fields. + * **Name** and **Description**. + * Select a **Plug-in Type** from the dropdown. For example, `Azure PostgreSQL Over Event Hub Connect 2.0`. + + * To **Upload from CSV**, go to the **"Upload from CSV"** tab and upload an exported or manually created CSV file + containing one or more profiles. You can also choose from the following options: + * **Update existing profiles on name match** — Updates profiles with the same name if they already exist. + * **Test connection for imported profiles** — Automatically tests connections after profiles are created. + * **Use ELB** — Enables ELB support for imported profiles. You must provide the number of MUs to be used in the + ELB process. + +**Note:** Configuration options vary based on the selected plug-in. + +## Configuring Azure PostgreSQL Over Event Hub Kafka Connect 2.0 + +The following table describes the fields that are specific to Azure Event Hub Kafka Connect 2.0 plugin. + +| Field | Description | +|-----------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Name** | Unique name of the profile. | +| **Description** | Description of the profile. | +| **Plug-in** | Plug-in type for this profile. Select `Azure PostgreSQL Over Event Hub Connect 2.0`. A full list of available plug-ins are available on the **Package Management** page. | +| **Credential** | The credential to authenticate with Azure Event Hub. Must be created in **Credential Management**, or click **➕** to create one. For more information, see [Creating Credentials](https://www.ibm.com/docs/en/SSMPHH_12.x/com.ibm.guardium.doc.stap/guc/guc_credential_management.html). | +| **Kafka Cluster** | Select the appropriate Kafka cluster from the available Kafka cluster list or create a new Kafka cluster. For more information, see [Managing Kafka clusters](https://www.ibm.com/docs/en/SSMPHH_12.x/com.ibm.guardium.doc.stap/guc/guc_kafka_cluster_management.html). | +| **Label** | Grouping label. For example, customer name or ID. | +| **Event Hub Connection String** | Primary connection string from the Event Hub shared access policy. This connection string is obtained from the Azure Event Hub configuration. | +| **Storage Connection String** | Connection string for the Azure Storage Account. Required when reading from multiple Event Hubs to track offsets. | +| **Consumer Group** | Consumer group name for the Event Hub. Use `$Default` for single collector or create custom consumer groups for multiple collectors. | +| **Event Hub Partition Count** | Number of partitions in the Event Hub. This value must match or exceed the number of database instances generating traffic. | +| **Enrollment ID** | Unique identifier for the Azure resource enrollment. | +| **No-traffic threshold (minutes)** | Default value is 60. If there is no incoming traffic for an hour, S-TAP displays a red status. Once incoming traffic resumes, the status returns to green. | +| **Use Enterprise Load Balancing (ELB)** | Enable this if ELB support is required. | +| **Managed Unit Count** | Number of Managed Units (MUs) to allocate for ELB. | + +**Note:** + +- Ensure that the **profile name** is unique. +- Required credentials must be created before or during profile creation. +- The Azure credentials must have appropriate permissions to read from Event Hub. + +--- + +## Azure Credential Configuration + +When creating credentials for Azure Event Hub, provide the following information. + +| Field name | Description | +|-----------------------|-----------------------------------------------------------------| +| **Name** | A unique credential name | +| **Description** | A description for your credential | +| **Credential Type** | Azure Event Hub Credentials `Azure Event Hub Connection String` | +| **Connection String** | Event Hub connection string with manage permissions | + +--- + +## Testing a Connection + +After creating a profile, you must test the connection to ensure the provided configuration is valid. + +### Procedure + +1. Select the new profile. +2. From the top menu, click **Test Connection**. +3. If the test is successful, you can proceed to installing the profile. + +--- + +## Installing a Profile + +Once the connection test is successful, you can install the profile on **Managed Units (MUs)** or **Edges**. The parsed +audit logs are sent to the selected Managed Unit or Edge to be consumed by the **Sniffer**. + +### Procedure + +1. Select the profile. +2. From the **Install** menu, click **Install**. +3. From the list of available MUs and Edges that is displayed, select the ones that you want to deploy the profile to. + +--- + +## Uninstalling or reinstalling profiles + +An installed profile can be uninstalled or reinstalled if needed. + +### Procedure + +1. Select the profile. +2. From the list of available actions, select the desired option: **Uninstall** or **Reinstall**. + +--- diff --git a/docs/KafkaBasedUCs/BigqueryPubsubKafkaConnect.md b/docs/KafkaBasedUCs/BigqueryPubsubKafkaConnect.md new file mode 100644 index 000000000..68d3405fe --- /dev/null +++ b/docs/KafkaBasedUCs/BigqueryPubsubKafkaConnect.md @@ -0,0 +1,270 @@ +# Bigquery over Pub/Sub Source Connector + +This connector enables IBM Guardium Data Protection (GDP) to monitor and collect audit logs from Bigquery databases +through Google Cloud Pub/Sub using Kafka Connect. + +## Meet BigQuery over Pub/Sub Connect + +* Environments: On-prem +* Supported inputs: Kafka connect Pubsub 2.0 (pull) +* Supported Guardium versions: + * Guardium Data Protection: Appliance bundle 12.2.2 or later + +Kafka-connect is a framework for streaming data between Apache Kafka and other systems. + +## Configuring BigQuery on GCP + +### BigQuery Setup +1. Complete the BigQuery [Prerequisites](https://cloud.google.com/bigquery/docs/quickstarts/quickstart-cloud-console). + +2. Enable the BigQuery API.
+BigQuery is automatically enabled in new projects. For existing projects, you must enable the BigQuery API. For more information, see [Enable the BigQuery API](https://cloud.google.com/bigquery-transfer/docs/enable-transfer-service#creating_a_project_and_enabling_the_api). +3. Create a BigQuery dataset to store your data. For more information, see [Create a Dataset](https://cloud.google.com/bigquery/docs/datasets). + +4. Create a table:
+ a. Expand the **View actions** menu and click **Open**.
+ b. In the details panel, click **Create table (+)**.
+ c. On the Create table page, enter a **Table name** (for example, user).
+ e. Optional: Add more fields by clicking **Add Field (+)**.
+ f. Click **Create Table**. For more information, see [Create Table](https://cloud.google.com/bigquery/docs/tables).
+ +5. Query your table data. For more information, see [Query Table Data](https://cloud.google.com/bigquery/docs/quickstarts/quickstart-cloud-console#query_table_data). + +### Permissions and Roles for log viewing and downloading + +To view or download the generated logs, make sure that the appropriate Identity and Access Management (IAM) roles are +assigned. + +* To View logs: + - roles/logging.viewer (Logs Viewer) + - roles/logging.privateLogViewer (Private Logs Viewer) +* To download logs: + - roles/logging.admin (Logging Admin) + - roles/logging.viewAccessor (Logs View Accessor) + +### Create a topic in Pub/Sub + +1. Go to the Pub/Sub topics page in the Cloud Console. +2. Click **Create a topic**. +3. In the **Topic ID** field, provide a unique topic name. For example, ``MyTopic``. +4. Click **Create Topic**. + +### Create a subscription in Pub/Sub + +1. Display the menu for the topic created in the previous step and click **New subscription**. +2. Type a name for the subscription, such as MySub. +3. Leave the delivery type as **Pull**. +4. Click **Create**. + +### Create a log sink in Pub/Sub + +1. In the Cloud Console, go to the **Logging** > **Log Router page**. +2. Click **Create sink**. +3. In the **Sink details** panel, enter the following details:
+ a. Sink name: Provide an identifier for the sink. Once you create the sink, you cannot rename it. However, you can delete a sink and create a new one.
+ b. Sink description (optional): Describe the purpose or use case for the sink.
+4. In the **Sink destination** panel, select the Cloud Pub/Sub topic as sink service and select the topic that you created in the previous steps. +5. Choose logs to include in the sink in the Build inclusion filter panel. +6. You can filter the logs by log name, resource, and severity. +7. In cases of multiple regions, you need to do the same set of configurations for each region. + Based on the region, different configuration files are used for the input plug-in. + +### Set destination (TOPIC & SUBSCRIPTION) permissions + +1. Obtain the sink's writer identity from the new sink. For example, an email address.
+ a. Go to the Log Router page, and select **menu** > **View sink details**.
+ b. The writer identity appears in the Sink details panel.
+2. If you have owner access to the destination:
+ a. Add the sink's writer identity to topic.
+ - Navigate to the Topic you created.
+ - Click on the **SHOW INFO** panel.
+ - Click **ADD PRINCIPAL**.
+ - Paste writer identity in the New Principals.
+ - Give it the Pub/Sub Publisher role and subscriber role.
+ + b. Add the sink's writer identity to subscription.
+ - Navigate to the Subscription.
+ - Click **SHOW INFO** panel.
+ - Click **ADD PRINCIPAL**.
+ - Paste writer identity in the New Principals<./br> + - Give it the subscriber role.
+ +### Creating service account credentials + +1. Go to the **Service accounts** section of the IAM & Admin console. +2. Select **project** and click **Create Service Account**. +3. Enter a **Service account name**, such as Bigquery-Pub/Sub. +4. Click **Create**. +5. The owner role is required for the service account. Select the owner role from the drop-down menu. +6. Click **Continue**. You do not need to grant users access to this service account. +7. Click **Create Key**. This key is used by the Logstash input plug-in configuration file. +8. Select JSON and click **Create**. + +### Inclusion Filter + +To edit the Sink, go to **Logs Router** > **Sink Inclusion Filter**. + +This inclusion filter excludes unnecessary logs and includes required logs with resource types and metadata reason as **DELETE**, **TABLE_INSERT_REQUEST**, **TABLE_DELETE_REQUEST** or **CREATE** and **metadata jobStatus**. + +``` +(resource.type=("bigquery_project") AND protoPayload.authenticationInfo.principalEmail:* AND +(protoPayload.metadata.jobChange.job.jobStatus.jobState = DONE AND -protoPayload.metadata.jobChange.job.jobConfig.queryConfig.statementType = "SCRIPT")) +OR +(protoPayload.metadata.datasetDeletion.reason = "DELETE") OR (protoPayload.metadata.tableCreation.reason = "TABLE_INSERT_REQUEST") OR (protoPayload.metadata.tableDeletion.reason = "TABLE_DELETE_REQUEST") OR (protoPayload.metadata.datasetCreation.reason = "CREATE") +``` + +## Viewing the Audit logs + +The inclusion filter is used to view the Audit logs in the GCP Logs Explorer. + +### Supported audit logs +1. BigQueryAudit - `ACTIVITY`, `DATA_ACCESS` logs +2. BigQuery Log - `EMERGENCY`, `ALERT`, `CRITICAL`, `ERROR`, `WARNING`, `NOTICE`, `DEBUG`, `DEFAULT` + +## Limitations +1. If no information regarding certain fields is available in the logs, those fields are not mapped. +2. Exception object is prepared based on severity of the logs. +3. The data model size is limited to 10 GB per table. If you have a 100 GB reservation per project per location, BigQuery BI Engine limits the reservation per table to 10 GB. The rest of the available reservation is used for other tables in the project. +4. BigQuery cannot read the data in parallel if you use gzip compression. Loading compressed JSON data into BigQuery is slower than loading uncompressed data. +5. You cannot include both compressed and uncompressed files in the same load job. +6. JSON data must be newline delimited. Each JSON object must be on a separate line in the file. +7. The maximum size for a gzip file is 4 GB. +8. Log messages have a size limit of 100K bytes. +9. The Audit/Data access log doesn't contain a server IP. The default value of the **server IP** is set to `0.0.0.0`. +10. The following important fields cannot be mapped, as there is no information regarding these fields in the logs: + - Source program + - OS User + - Client HostName +11. `serverHostName` pattern for BigQuery GCP: roject-id_bigquery.googleapis.com. +12. When you try to create or delete a data set or table using BigQuery UI options, fields like the FULL SQL & Objects and Verbs column appear blank, because these actions don't receive any query from GCP logs. You can ignore these actions, by updating the inclusion filter: + "(resource.type=("bigquery_project") AND protoPayload.authenticationInfo.principalEmail:* AND + (protoPayload.metadata.jobChange.job.jobStatus.jobState = DONE AND -protoPayload.metadata.jobChange.job.jobConfig.queryConfig.statementType = "SCRIPT"))" +13. The parser does not support queries in which a keyword is used as a table name or column name, or in scenarios of nested parameters inside functions. +14. The BigQuery audit log doesn’t include login failed logs, so these do not appear in the guardium LOGIN_FAILED report. +15. Syntactically correct SQL queries that fail on Database are captured only in the SQL_Error report. + +## Creating datasource profiles + +You can create a new datasource profile from the **Datasource Profile Management** page. + +### Procedure + +1. Go to **Manage > Universal Connector > Datasource Profile Management** +2. Click the **➕ (Add)** button. +3. You can create a profile by using one of the following methods: + + * To create a new profile manually, go to the **"Add Profile"** tab and provide values for the following fields. + * **Name** and **Description**. + * Select a **Plug-in Type** from the dropdown. For example, **BigQuery Over PubSub Kafka Connect 2.0**. + + * To upload from CSV, go to the **Upload from CSV** tab and upload an exported or manually created CSV file + containing one or more profiles. You can also choose from the following options: + * **Update existing profiles on name match** — Updates profiles with the same name if they already exist. + * **Test connection for imported profiles** — Automatically tests connections after profiles are created. + * **Use ELB** — Enables ELB support for imported profiles. You must provide the number of MUs to be used in the + ELB process. + +**Note:** Configuration options vary based on the selected plug-in. + +## Configuration: Pub/Sub Kafka Connect 2.0-based Plugins + +The following table describes the fields that are specific to Pub/Sub Kafka Connect 2.0 and similar plugins. + +| Field | Description | +|-------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Name** | Unique name of the profile. | +| **Description** | Description of the profile. | +| **Plug-in** | Plug-in type for this profile. A full list of available plug-ins are available on the **Package Management** page. | +| **Credential** | The credential to authenticate with the datasource. Must be created in **Credential Management**, or click **➕** to create one. | +| **Kafka Cluster** | Kafka cluster to deploy the universal connector. | +| **Label** | Grouping label. For example, **customer name** or **ID**. | +| **GCP project id** | Google Cloud project ID that contains the Pub/Sub subscription. | +| **Pub/Sub Subscription ID** | Pub/Sub subscription ID from which messages are consumed. | +| **GCP Topic** | Pub/Sub topic name. | +| **Maximum poll records** | The maximum number of records returned in a single poll | +| **Expected events per second** | Expected events per second. This value is used to automatically calculate the **parallel.pull.count** parameter when it not set. Calculation formula: ceil(expected.eps / 1000). | | +| **Number of parallel pull streams** | Number of parallel pull streams to use. If not specified, this value is automatically calculated based on **expected.eps** (1 subscriber per 1000 EPS). | +| **No traffic threshold (minutes)** | The time period after which the system detects inactivity. | + +## Testing a Connection + +After creating a profile, you must test the connection to ensure that the provided configuration is valid. + +### Procedure + +1. Select the new profile. +2. From the top menu, click **Test Connection**. +3. If the test is successful, you can proceed to installing the profile. + +--- + +## Installing a Profile + +Once the connection test is successful, you can install the profile on **Managed Units (MUs)** or **Edges**. The parsed +audit logs are sent to the selected Managed Unit or Edge to be consumed by the Sniffer. + +### Procedure + +1. Select the profile. +2. From the **Install** menu, click **Install**. +3. From the list of available MUs and Edges that is displayed, select the ones that you want to deploy the profile to. + +--- + +## Uninstalling or reinstalling profiles + +An installed profile can be **uninstalled** or **reinstalled** if needed. + +### Procedure + +1. Select the profile. +2. From the list of available actions, select the desired option: **Uninstall** or **Reinstall**. + +--- + +## Required Configuration Changes for Production + +## Configuring GCP for production + +You must configure **exactly-once** delivery on your Pub/Sub subscription to prevent duplicate audit log entries at the +Pub/Sub level before the messages reach the connector. + +### Procedure + +1. To enable **exactly-once** delivery on your Pub/Sub subscription, use only one of the following commands based on + your scenario. + * For an existing subscription: + ```bash + gcloud pubsub subscriptions update \ + --enable-exactly-once-delivery + ``` + * For a new subscription: + ```bash + gcloud pubsub subscriptions create \ + --topic= \ + --enable-exactly-once-delivery \ + --ack-deadline=600 + ``` + +## Troubleshooting + +#### Messages are not being processed + +1. Verify that all Kafka worker settings are added. +2. Make sure the worker is restarted after configuration changes. +3. Restart the connector after the worker restart. + +#### "Quota exceeded" errors in GCP + +1. Check the current quota usage in the GCP console. +2. Request a quota increase. +3. Temporarily reduce the number of tasks until the quota is increased. + +#### High number of unacknowledged messages in Pub/Sub + +1. Verify that the connector is running by using the following command.
+ ``` + curl http://localhost:8083/connectors//status + ``` +2. Check for errors in logs. +3. Verify that exactly-once delivery is enabled on the subscription. diff --git a/docs/KafkaBasedUCs/BigtablePubsubKafkaConnect.md b/docs/KafkaBasedUCs/BigtablePubsubKafkaConnect.md new file mode 100644 index 000000000..5b76e70fd --- /dev/null +++ b/docs/KafkaBasedUCs/BigtablePubsubKafkaConnect.md @@ -0,0 +1,259 @@ +# Bigtable over Pub/Sub Source Connector + +This connector enables IBM Guardium Data Protection (GDP) to monitor and collect audit logs from Bigtable databases +through Google Cloud Pub/Sub using Kafka Connect. + +## Meet BigTable over Pub/Sub Connect + +* Environments: On-prem +* Supported inputs: Kafka connect Pubsub 2.0 (pull) +* Supported Guardium versions: + * Guardium Data Protection: Appliance bundle 12.2.2 or later + +Kafka-connect is a framework for streaming data between Apache Kafka and other systems. + +## Configuring BigTable on GCP + +### BigTable Setup +1. Complete the BigTable [Prerequisites](https://cloud.google.com/bigtable/docs/quickstarts/quickstart-cloud-console). + +2. Enable the BigTable API.
+BigQuery is automatically enabled in new projects. For existing projects, you must enable the BigTable API. For more information, see [Enable the BigTable API](https://console.cloud.google.com/marketplace/product/google/bigtable.googleapis.com). +3. Create a BigTable dataset to store your data. For more information, see [Create a Dataset](https://cloud.google.com/bigtable/docs/creating-instance). + +4. Create a table. For more information, see [Create Table](https://cloud.google.com/bigtable/docs/samples/bigtable-hw-create-table).
+ +5. Query table data. For more information, see see [Query Table Data](https://cloud.google.com/bigquery/docs/external-data-bigtable). + +### Permissions and Roles for log viewing and downloading + +To view or download the generated logs, make sure that the appropriate Identity and Access Management (IAM) roles are +assigned. + +* To View logs: + - roles/logging.viewer (Logs Viewer) + - roles/logging.privateLogViewer (Private Logs Viewer) +* To download logs: + - roles/logging.admin (Logging Admin) + - roles/logging.viewAccessor (Logs View Accessor) + +### Creating a topic in Pub/Sub + +1. Go to the Pub/Sub topics page in the Cloud Console. +2. Click **Create a topic**. +3. In the **Topic ID** field, provide a unique topic name. For example, ``MyTopic``. +4. Click **Create Topic**. + +### Creating a subscription in Pub/Sub + +1. Display the menu for the topic created in the previous step and click **New subscription**. +2. Type a name for the subscription, such as MySub. +3. Leave the delivery type as **Pull**. +4. Click **Create**. + +### Creating a log sink in Pub/Sub + +1. In the Cloud Console, go to the **Logging** > **Log Router page**. +2. Click **Create sink**. +3. In the **Sink details** panel, enter the following details:
+ a. Sink name: Provide an identifier for the sink. Once you create the sink, you cannot rename it. However, you can delete a sink and create a new one.
+ b. Sink description (optional): Describe the purpose or use case for the sink.
+4. In the **Sink destination** panel, select the Cloud Pub/Sub topic as sink service and select the topic that you created in the previous steps. +5. Choose logs to include in the sink in the Build inclusion filter panel. +6. You can filter the logs by log name, resource, and severity. +7. In cases of multiple regions, you need to do the same set of configurations for each region. + Based on the region, different configuration files are used for the input plug-in. + +### Setting permissions for the destination (TOPIC & SUBSCRIPTION) + +1. Obtain the sink's writer identity from the new sink. For example, an email address.
+ a. Go to the Log Router page, and select **menu** > **View sink details**.
+ b. The writer identity appears in the Sink details panel.
+2. If you have owner access to the destination:
+ a. Add the sink's writer identity to topic.
+ - Navigate to the Topic you created.
+ - Click on the **SHOW INFO** panel.
+ - Click **ADD PRINCIPAL**.
+ - Paste writer identity in the New Principals.
+ - Give it the Pub/Sub Publisher role and subscriber role.
+ + b. Add the sink's writer identity to subscription.
+ - Navigate to the Subscription.
+ - Click **SHOW INFO** panel.
+ - Click **ADD PRINCIPAL**.
+ - Paste writer identity in the New Principals<./br> + - Give it the subscriber role.
+ +### Creating service account credentials + +1. Go to the **Service accounts** section of the IAM & Admin console. +2. Select **project** and click **Create Service Account**. +3. Enter a **Service account name**, such as Bigtable-Pub/Sub. +4. Click **Create**. +5. The owner role is required for the service account. Select the owner role from the drop-down menu. +6. Click **Continue**. You do not need to grant users access to this service account. +7. Click **Create Key**. This key is used by the Logstash input plug-in configuration file. +8. Select **JSON** and click **Create**. + +### Inclusion Filter + +To edit the Sink, go to **Logs Router** > **Sink Inclusion Filter**. + +This inclusion filter excludes unnecessary logs and includes required logs with resource types and metadata only from BigTable. + +``` +protoPayload.serviceName="bigtableadmin.googleapis.com" OR protoPayload.serviceName="bigtable.googleapis.com" +``` + +## Viewing the Audit logs + +The above inclusion filter is used to view the audit logs in the GCP Logs Explorer. + +### Supported audit logs + +1. BigTableAudit - `ACTIVITY`, `DATA_ACCESS` logs +2. BigTable Log - `CREATEINSTANCE`, `DELETEINSTANCE`, `UPDATEINSTANCE`, `CREATECLUSTER`, `DELETECLUSTER`, `UPDATECLUSTER`, `CREATETABLE`, `DELETETABLE`, `MODIFYCOLUMNFAMILIES`, `EXECUTEQUERY`, `LISTCLUSTERS`, `LISTINSTANCES`. + +## Limitations +* Exception object is prepared based on severity of the logs. +* The data model size is limited to 10 GB per table. If you have a 100 GB reservation per project per location, BigTable BI Engine limits the reservation per table to 10 GB. The rest of the available reservation is used for other tables in the project. +* BigTable cannot read the data in parallel if you use gzip compression. Loading compressed JSON data into BigTable is slower than loading uncompressed data. +* You cannot include both compressed and uncompressed files in the same load job. +* JSON data must be newline delimited. Each JSON object must be on a separate line in the file. +* The maximum size for a gzip file is 4 GB. +* Log messages have a size limit of 100K bytes. +* The Audit/Data access log doesn't contain a server IP. The default value is set to `0.0.0.0` and can be in IPV4 or IPV6 format. +* The following important fields cannot be mapped, as there is no information regarding these fields in the logs: + - Source program + - OS User + - Client HostName +* While using GCP, duplicate entries may appear in both the reports and audit logs. +* Bigtable uses two different service names (`bigtable.googleapis.com` & `bigtableadmin.googleapis.com`) depending on the tasks being performed. This results in two distinct S-TAP host entries. +* Multiple sessions from the same Bigtable instance may result in multiple S-TAP entries. +* The BigTable audit log doesn’t include login failed logs. So, these logs do not appear in the guardium LOGIN_FAILED report. + +## Creating datasource profiles + +You can create a new datasource profile from the **Datasource Profile Management** page. + +### Procedure + +1. Go to **Manage > Universal Connector > Datasource Profile Management** +2. Click the **➕ (Add)** button. +3. You can create a profile by using one of the following methods: + + * To create a new profile manually, go to the **"Add Profile"** tab and provide values for the following fields. + * **Name** and **Description**. + * Select a **Plug-in Type** from the dropdown. For example, **BigTable Over PubSub Kafka Connect 2.0**. + + * To upload from CSV, go to the **Upload from CSV** tab and upload an exported or manually created CSV file + containing one or more profiles. You can also choose from the following options: + * **Update existing profiles on name match** — Updates profiles with the same name if they already exist. + * **Test connection for imported profiles** — Automatically tests connections after profiles are created. + * **Use ELB** — Enables ELB support for imported profiles. You must provide the number of MUs to be used in the + ELB process. + +**Note:** Configuration options vary based on the selected plug-in. + +## Configuration: Pub/Sub Kafka Connect 2.0-based Plugins + +The following table describes the fields that are specific to Pub/Sub Kafka Connect 2.0 and similar plugins. + +| Field | Description | +|-------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Name** | Unique name of the profile. | +| **Description** | Description of the profile. | +| **Plug-in** | Plug-in type for this profile. A full list of available plug-ins are available on the **Package Management** page. | +| **Credential** | The credential to authenticate with the datasource. Must be created in **Credential Management**, or click **➕** to create one. | +| **Kafka Cluster** | Kafka cluster to deploy the universal connector. | +| **Label** | Grouping label. For example, **customer name** or **ID**. | +| **GCP project id** | Google Cloud project ID that contains the Pub/Sub subscription. | +| **Pub/Sub Subscription ID** | Pub/Sub subscription ID from which messages are consumed. | +| **GCP Topic** | Pub/Sub topic name. | +| **Maximum poll records** | The maximum number of records returned in a single poll | +| **Expected events per second** | Expected events per second. This value is used to automatically calculate the **parallel.pull.count** parameter when it not set. Calculation formula: ceil(expected.eps / 1000). | | +| **Number of parallel pull streams** | Number of parallel pull streams to use. If not specified, this value is automatically calculated based on **expected.eps** (1 subscriber per 1000 EPS). | +| **No traffic threshold (minutes)** | The time period after which the system detects inactivity. | + +## Testing a Connection + +After creating a profile, you must test the connection to ensure that the provided configuration is valid. + +### Procedure + +1. Select the new profile. +2. From the top menu, click **Test Connection**. +3. If the test is successful, you can proceed to installing the profile. + +--- + +## Installing a Profile + +Once the connection test is successful, you can install the profile on **Managed Units (MUs)** or **Edges**. The parsed +audit logs are sent to the selected Managed Unit or Edge to be consumed by the Sniffer. + +### Procedure + +1. Select the profile. +2. From the **Install** menu, click **Install**. +3. From the list of available MUs and Edges that is displayed, select the ones that you want to deploy the profile to. + +--- + +## Uninstalling or reinstalling profiles + +An installed profile can be **uninstalled** or **reinstalled** if needed. + +### Procedure + +1. Select the profile. +2. From the list of available actions, select the desired option: **Uninstall** or **Reinstall**. + +--- + +## Required Configuration Changes for Production + +## Configuring GCP for production + +You must configure **exactly-once** delivery on your Pub/Sub subscription to prevent duplicate audit log entries at the +Pub/Sub level before the messages reach the connector. + +### Procedure + +1. To enable **exactly-once** delivery on your Pub/Sub subscription, use only one of the following commands based on + your scenario. + * For an existing subscription: + ```bash + gcloud pubsub subscriptions update \ + --enable-exactly-once-delivery + ``` + * For a new subscription: + ```bash + gcloud pubsub subscriptions create \ + --topic= \ + --enable-exactly-once-delivery \ + --ack-deadline=600 + ``` + +## Troubleshooting + +#### Messages are not being processed + +1. Verify that all Kafka worker settings are added. +2. Make sure the worker is restarted after configuration changes. +3. Restart the connector after the worker restart. + +#### "Quota exceeded" errors in GCP + +1. Check the current quota usage in the GCP console. +2. Request a quota increase. +3. Temporarily reduce the number of tasks until the quota is increased. + +#### High number of unacknowledged messages in Pub/Sub + +1. Verify that the connector is running by using the following command.
+ ``` + curl http://localhost:8083/connectors//status + ``` +2. Check for errors in logs. +3. Verify that exactly-once delivery is enabled on the subscription. diff --git a/docs/KafkaBasedUCs/DocumentDBCloudwatchKafkaConnect.md b/docs/KafkaBasedUCs/DocumentDBCloudwatchKafkaConnect.md new file mode 100644 index 000000000..ad9db29be --- /dev/null +++ b/docs/KafkaBasedUCs/DocumentDBCloudwatchKafkaConnect.md @@ -0,0 +1,240 @@ +# Configuring DocumentDB on AWS datasource profiles for Kafka Connect plug-ins + +Create and configure datasource profiles through Central Manager for DocumentDB over CloudWatch Kafka Connect plug-ins. + +## Meet DocumentDB over Cloudwatch Kafka Connect + +* **Environment:** AWS +* **Supported inputs:** Kafka connect Cloudwatch 2.0 (pull) +* **Supported Guardium versions:** + * Guardium Data Protection: Appliance bundle 12.2.2 or later + +Kafka-connect is a framework for streaming data between Apache Kafka and other systems. This connector enables monitoring of DocumentDB audit logs through CloudWatch. + +## Configuring Amazon DocumentDB + +In the AWS web interface, configure the service for DocumentDB. + +### Procedure + +1. Go to the [AWS Console](https://console.aws.amazon.com/). +2. From the navigation menu, click **Services**. +3. Click **All services** > **Database**. +4. On the right panel, click **Amazon DocumentDB**. +5. From the dropdown menu, select your region. +6. Click **Create**. +7. Enter a cluster identifier. Then click **Create cluster**. + +## Enabling Audit Logs + +You can use different methods for auditing and logging. This example uses CloudTrail because it supports all required parameters. The following events are supported for auditing in AWS. + +### Procedure + +1. From the AWS console, open the **Amazon DocumentDB** service. +2. In the left navigation pane, click **Parameter groups**. +3. Check which cluster parameter group is currently associated with the DocumentDB cluster. +4. If the cluster uses the default parameter group, click **Create** (or **Create parameter group**) to create a custom cluster parameter group. +5. For the new parameter group, select the family version that matches your DocumentDB cluster (for example, docdb4.0 for version 4.0.0) +6. Enter a name and description for the parameter group, then click **Create**. +7. Select the newly created parameter group and click **Edit** (or **Edit parameters**). +8. In the search box, enter `audit_logs` to locate the **audit_logs** parameter. +9. Change the **audit_logs** field value from disabled to one of the following options. + * **enabled** / **all**: Logs all events + * Specific event type: Enter a supported value to log only certain events (for example, `DDL` or `DML`). +10. Save the changes to the parameter group. +11. In the left navigation pane, click **Clusters**, then select the DocumentDB cluster. +12. Click **Modify**. +13. In the Cluster parameter group or Additional configuration section, select the custom parameter group that you edited. +14. Under the Scheduling of modifications section, select **Apply immediately** (or choose the appropriate maintenance window, if required). +15. Review your changes. Then click **Modify cluster** to apply the new parameter group with audit logging enabled. +16. After the cluster modification completes, Amazon DocumentDB automatically creates a CloudWatch Logs log group named: `/aws/docdb//audit`. + +### Viewing DocumentDB log entries on CloudWatch + +By default, each CloudTrail trail has an associated log group with a name in the format specified during trail creation. You can use this log group, or you can create a new one and associate it with the trail. + +1. On the AWS Console page, open the **Services** menu. +2. In the search box, enter `CloudWatch`. +3. Click **CloudWatch** to open the CloudWatch dashboard. +4. In the left panel, click **Logs**. +5. Click **Log Groups**. + +## Exporting CloudWatch Logs to SQS by using a Lambda function (Optional) + +To achieve load balancing of audit logs between different collectors, you can export the audit logs from CloudWatch to SQS. + +### Creating the SQS queue + +1. Go to the [AWS Console](https://console.aws.amazon.com/). +2. Click **Services**. +3. In the search box, enter `SQS` and click **Simple Queue Services**. +4. Click **Create Queue**. +5. In the **Type** field, select **Standard**. +6. Enter a name for the queue. +7. Keep the default settings for the remaining fields. + +### Creating a policy for the IAM User + +Complete the following steps for the IAM user that will access the SQS logs in Guardium. + +1. Go to [AWS Console](https://console.aws.amazon.com/). +2. Go to **IAM service** > **Policies** > **Create Policy**. +3. For the **Service** field, select **SQS**. +4. Select the following checkboxes: **ListQueues**, **DeleteMessage**, **DeleteMessageBatch**, **GetQueueAttributes**, **GetQueueUrl**, **ReceiveMessage**, **ChangeMessageVisibility**, and **ChangeMessageVisibilityBatch**. +5. In the **Resources** section, specify the ARN of the queue that you created in the previous step. +6. Click **Review policy** and enter the policy name. +7. Click **Create policy**. +8. Assign the policy to the user.
+ a. Log in to the [IAM console](https://console.aws.amazon.com/iam/).
+ b. Go to **Users** on the console, and select the IAM user that you want to grant permissions.
+ c. In the **Permissions** tab, click **Add permissions** > **Attach existing policies directly**.
+ d. Select the checkbox for the policy that you created.
+ e. Click **Next: Review** > **Add permissions**.
+ +### Creating an IAM Role + +Create an IAM role for the Lambda function. The AWS Lambda service requires permission to log events and write to the SQS queue. Create the IAM Role named **Export-DocumentDB-CloudWatch-to-SQS-Lambda** with the following policies: **AmazonSQSFullAccess**, **CloudWatchLogsFullAccess**, and **CloudWatchEventsFullAccess**. + +1. Go to the [AWS Console](https://console.aws.amazon.com/). +2. Go to **IAM** > **Roles** > **Create role**. +3. For **Use case**, select **Lambda** and click **Next**. +4. Search for ``AmazonSQSFullAccess`` and select it. +5. Search for ``CloudWatchLogsFullAccess`` and select it. +6. Search for ``CloudWatchEventsFullAccess`` and select it. +7. Enter a **Role Name**. For example, `Export-DocumentDB-CloudWatch-to-SQS-Lambda`. +8. Click **Create role**. + +### Creating the Lambda Function + +1. Go to the [AWS Console](https://console.aws.amazon.com/). +2. Go to **Services** and search for `Lambda function`. +3. Click **Functions** > **Create Function** +4. Keep **Author from Scratch** selected. +5. For **Function name**, enter a name. For example, **Export-DocumentDB-CloudWatch-Logs-To-SQS**. +6. For **Runtime**, select **Python 3.x**. +7. For **Permissions**, select **Use an existing role** and select the IAM role that you created in the previous step (`Export-DocumentDB-CloudWatch-to-SQS-Lambda`). +8. Click **Create function**. +9. In the **Code view**, add the function code from the DocumentDB Lambda function file (available in the plugin package). +10. Click **Configuration** > **Environment Variables**. +11. Create the following variables. + - **Key** = `GROUP_NAME`, **Value** = The name of the CloudWatch log group whose logs you want to export (for example, `/aws/cloudtrail/DocumentDB-trail`) + - **Key** = `QUEUE_NAME`, **Value** = The queue URL where logs are sent (for example, `https://sqs.us-east-1.amazonaws.com/1111111111/DocumentDB`) +12. Click **Save** > **Deploy**. + +### Automating the Lambda function + +1. Go to the CloudWatch dashboard. +2. In the navigation panel, click **Events** > **Rules**. +3. Click **Create Rule**. +4. For **Rule name**, enter a name (for example, `cloudwatchToSqs`). +5. For **Rule Type**, select **Schedule**. +6. Define the schedule. For **Schedule pattern**, select a schedule that runs at a regular rate, such as every 10 minutes. +7. Enter the rate expression (the rate at which the function is executed). This value must match the time that is specified in the lambda function code that calculates the time delta. For instance, if the function code is set to 2 minutes, set the rate to 2 minutes unless changed in the code. +8. Click **Next**.H +9. Select **Target1**. For **Target Type**, select **AWS Service**. +10. For **Target**, select **Lambda Function**. +11. Select the Lambda function that you created in the previous step (`Export-DocumentDB-CloudWatch-Logs-To-SQS`). +12. Optional: Add tags. +13. Click **Create Rule**. + +**Note:** Before you make any changes to the Lambda function code, you must disable the rule that you created. Deploy the changes and then re-enable the rule. + +## Limitations +- Due to native limitations in Amazon DocumentDB, audit logs may truncate query details (1 KB limit), and profiler logs only capture slow queries (based on a configurable threshold, around 50 ms). As a result, full query visibility cannot be guaranteed. + - https://docs.aws.amazon.com/documentdb/latest/developerguide/event-auditing.html + - https://docs.aws.amazon.com/documentdb/latest/developerguide/profiling.html +- DocumentDB Profiler logs capture database operations that take longer than some period of time (for example, 100 ms). If the threshold value is not configurable and the set value is too high, profiler logs may not be captured for every database operation. +- The following important fields cannot be mapped with DocumentDB audit or profiler logs: + - **Source program**: Available only for aggregate queries + - **OS User**: Not available with audit or profiler logs + - **Client HostName**: Not available with audit or profiler logs +- Server IPs are not reported because they are not part of the audit stream. However, the `add_field` clause in the configuration adds a user-defined server host name that can be used in reports and policies. +- The sniffer saves the database name when a new session is created, not with every event. The database name is updated and populated correctly in Guardium only when a new database connection is established with a database name. If a database connection is established without a database name, the database on which the first query for that session runs is retained in Guardium, even if the user switches between databases for the same session. +- SQL errors are not supported. +- For DocumentDB over CloudWatch Kafka, logs take 15-20 minutes to appear in Guardium after they are generated by the database. + +## Creating datasource profiles + +You can create a new datasource profile from the Datasource Profile Management page. + +### Procedure + +1. Go to **Manage > Universal Connector > Datasource Profile Management** +2. Click the **➕ (Add)** button. +3. Create a profile by using one of the following methods: + + - To **Create a new profile manually**, go to the **"Add Profile"** tab and provide values for the following fields: + - **Name** and **Description**. + - **Plug-in Type** — Select a plug-in type from the dropdown (for example, `DocumentDB over Cloudwatch Connect 2.0`). + + - To **Upload from CSV**, go to the **"Upload from CSV"** tab and upload an exported or manually created CSV file containing one or more profiles. + You can also choose from the following options: + - **Update existing profiles on name match** — Updates profiles with the same name if they already exist. + - **Test connection for imported profiles** — Automatically tests connections after profiles are created. + - **Use ELB** — Enables ELB support for imported profiles. You must provide the number of MUs to use in the ELB process. + +**Note:** Configuration options vary based on the selected plug-in. + +## Configuring DocumentDB Over CloudWatch Kafka Connect 2.0 + +The following table describes the fields that are specific to DocumentDB over CloudWatch Kafka Connect 2.0 plugin. + +| Field | Description | +|--------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Name** | Unique name of the profile. | +| **Description** | Description of the profile. | +| **Plug-in** | Plug-in type for this profile. Select `DocumentDB Over Cloudwatch Connect 2.0`. A full list of available plug-ins are available on the Package Management page. | +| **Credential** | Select AWS Credentials or AWS Role ARN. The credential to authenticate with AWS. Must be created in **Credential Management**, or click **➕** to create one. For more information, see [Creating Credentials](https://www.ibm.com/docs/en/SSMPHH_12.x/com.ibm.guardium.doc.stap/guc/guc_credential_management.html). | +| **Kafka Cluster** | Select the appropriate Kafka cluster from the available Kafka cluster list or create a new Kafka cluster. For more information, see [Managing Kafka clusters](https://www.ibm.com/docs/en/SSMPHH_12.x/com.ibm.guardium.doc.stap/guc/guc_kafka_cluster_management.html). | +| **Label** | Grouping label. For example, customer name or ID. | +| **AWS account region** | Specifies the AWS region where your DocumentDB tables are located (e.g., us-east-1, eu-west-1). | +| **Log groups** | List of CloudWatch log groups to monitor. These are the log groups where DocumentDB audit logs (via CloudTrail) are exported. Format: `/aws/cloudtrail/` | +| **Filter pattern** | CloudWatch Logs filter pattern to apply. Use "None" to retrieve all logs, or specify a pattern to filter specific log events. | +| **Account ID** | Your AWS account ID (12-digit number). This identifies your AWS account. | +| **Cluster name** | The name of your DocumentDB cluster or table identifier. | +| **Ingestion delay (seconds)** | Default value is 900 seconds (15 minutes). This delay accounts for the time it takes for logs to be available in CloudWatch after being generated. | +| **No-traffic threshold (minutes)** | Default value is 60. If there is no incoming traffic for an hour, S-TAP displays a red status. Once incoming traffic resumes, the status returns to green. | +| **Unmask sensitive value** | Optional boolean flag. When enabled, sensitive values in the audit logs will not be masked. | +| **Use Enterprise Load Balancing (ELB)** | Enable this if ELB support is required. | +| **Managed Unit Count** | Number of Managed Units (MUs) to allocate for ELB. | + +**Note:** +- Ensure that the **profile name** is unique. +- Required credentials must be created before or during profile creation. +- The AWS credentials must have appropriate permissions to read CloudWatch logs and CloudTrail events. + +## Testing a connection + +After you create a profile, test the connection to ensure that the configuration is valid. + +### Procedure + +1. Select the profile. +2. From the top menu, click **Test Connection**. +3. If the test is successful, you can install the profile. + +--- + +## Installing a profile + +After the connection test is successful, you can install the profile on Managed Units (MUs) or Edges. The parsed audit logs are sent to the selected Managed Unit or Edge to be consumed by the Sniffer. + +### Procedure + +1. Select the profile. +2. From the **Install** menu, click **Install**. +3. From the list of available MUs and Edges, select the ones where you want to deploy the profile. + +--- + +## Uninstalling or reinstalling profiles + +You can uninstall or reinstall an installed profile. + +### Procedure + +1. Select the profile. +2. Click **Uninstall** or **Reinstall**. + +--- diff --git a/docs/KafkaBasedUCs/DynamoDBCloudwatchKafkaConnect.md b/docs/KafkaBasedUCs/DynamoDBCloudwatchKafkaConnect.md new file mode 100644 index 000000000..24ce0df69 --- /dev/null +++ b/docs/KafkaBasedUCs/DynamoDBCloudwatchKafkaConnect.md @@ -0,0 +1,237 @@ +# Configuring DynamoDB on AWS datasource profiles for Kafka Connect Plug-ins + +Create and configure datasource profiles through Central Manager for **DynamoDB over Cloudwatch Kafka Connect** plug-ins. + +## Meet DynamoDB over Cloudwatch Kafka Connect + +* **Environment:** AWS +* **Supported inputs:** Kafka connect Cloudwatch 2.0 (pull) +* **Supported Guardium versions:** + * Guardium Data Protection: Appliance bundle 12.2.2 or later + +Kafka-connect is a framework for streaming data between Apache Kafka and other systems. This connector enables monitoring of DynamoDB audit logs through CloudWatch. + +## Configuring Amazon DynamoDB + +In the AWS web interface, configure the service for DynamoDB. + +### Procedure + +1. Go to https://console.aws.amazon.com/. +2. Click **Services** in the top left menu. +3. Underneath **All services**, click on **Database**. +4. On the right panel, click **DynamoDB**. +5. At the top right, click on the dropdown menu and select your region. +6. Click the orange **Create Table** button. +7. Enter a table name. +8. Enter a partition key. +9. Scroll down and click **Create table**. + +## Enabling Audit Logs + +There are different methods for auditing and logging. CloudTrail is used for this example as it supports all required parameters. The following events are supported for auditing in AWS. + +### Procedure + +1. From the top left menu, click **Services**. +2. Underneath **All services**, click on **Management & Governance**. +3. On the right panel, click **Cloud Trail**. +4. Click **Create trail**. +5. Enter a **Trail name**. +6. Under **Storage location**, verify that **Create new S3 bucket** is selected. +7. Under **Log file SSE-KMS encryption**, clear the **Enabled** box. +8. If the logs are to be monitored through CloudWatch, then forward them to CloudWatch by using steps 9 to 13 (If not, skip those steps). +9. Under **CloudWatch Logs**, check the **Enabled** box. +10. Verify **New** is selected for **Log group**. +11. Under **Log group name**, provide a new log group name. +12. Verify **New** is selected for **IAM Role**. +13. For **Role name**, provide a new role name. +14. Click **Next**. +15. For **Event type**, select **Management events** and **Data events**. +16. Verify that **Read** and **Write** are selected for **API Activity**. +17. In the **Data Events** section, click **Switch to basic event selectors**. +18. Click **Continue** > **Add data event type** > **Data event source** and then select **DynamoDB**. +19. Click **NEXT**. +20. Verify that all parameters shown are correct. Then click **Create trail**. + +### Viewing DynamoDB log entries on CloudWatch + +By default, each CloudTrail trail has an associated log group with a name in the format specified during trail creation. You can use this log group, or you can create a new one and associate it with the trail. + +1. On the AWS Console page, open the **Services** menu. +2. Enter `CloudWatch` in the search box. +3. Click **CloudWatch** to redirect to the CloudWatch dashboard. +4. In the left panel, select **Logs**. +5. Click **Log Groups**. + +## Exporting CloudWatch Logs to SQS Using Lambda Function (Optional) + +To achieve load balancing of audit logs between different collectors, the audit logs can be exported from CloudWatch to SQS. + +### Creating the SQS Queue + +1. Go to https://console.aws.amazon.com/. +2. Click **Services**. +3. Search for SQS and click on **Simple Queue Services**. +4. Click **Create Queue**. +5. Select the type as **Standard**. +6. Enter the name for the queue. +7. Keep the rest of the default settings. + +### Create Policy for the Relevant IAM User + +1. For the IAM User using which the SQS logs are to be accessed in Guardium, complete the following steps. +2. Go to https://console.aws.amazon.com/. +3. Go to **IAM service** > **Policies** > **Create Policy**. +4. Select **service as SQS**. +5. Select the following checkboxes: **ListQueues**, **DeleteMessage**, **DeleteMessageBatch**, **GetQueueAttributes**, **GetQueueUrl**, **ReceiveMessage**, **ChangeMessageVisibility**, **ChangeMessageVisibilityBatch**. +6. In the resources, specify the ARN of the queue created in the previous step. +7. Click **Review policy** and specify the policy name. +8. Click **Create policy**. +9. Assign the policy to the user.
+ a. Log in to the IAM console as IAM user (https://console.aws.amazon.com/iam/).
+ b. Go to **Users** on the console and select the relevant IAM user to whom you want to give permissions.
+ c. In the **Permissions** tab, click **Add permissions**.
+ d. Click **Attach existing policies directly**.
+ e. Search for the policy created and check the checkbox next to it.
+ f. Click **Next: Review** > **Add permissions**.
+ +## Creating the Lambda Function + +### Create IAM Role + +Create the IAM role that will be used in the Lambda function setup. The AWS Lambda service requires permission to log events and write to the SQS created. Create the IAM Role **Export-DynamoDB-CloudWatch-to-SQS-Lambda** with **AmazonSQSFullAccess**, **CloudWatchLogsFullAccess**, and **CloudWatchEventsFullAccess** policies. + +1. Go to https://console.aws.amazon.com/. +2. Go to **IAM** > **Roles** > **Create Role**. +4. Under **Use case**, select **Lambda** and click **Next**. +5. Search for ``AmazonSQSFullAccess`` and select it. +6. Search for ``CloudWatchLogsFullAccess`` and select it. +7. Search for ``CloudWatchEventsFullAccess`` and select it. +8. Set the **Role Name**. For example, **Export-DynamoDB-CloudWatch-to-SQS-Lambda**. Then click **Create role**. + +### Create the Lambda Function + +1. Go to https://console.aws.amazon.com/. +2. Go to **Services**. Search for Lambda function. +3. Click **Functions** > **Create Function** +5. Keep **Author from Scratch** selected. +6. Set **Function name**. For example, **Export-DynamoDB-CloudWatch-Logs-To-SQS**. +7. Under **Runtime**, select **Python 3.x**. +8. Under **Permissions**, select **Use an existing role** and select the IAM role created in the previous step (Export-DynamoDB-CloudWatch-to-SQS-Lambda). +9. Click **Create function** and navigate to **Code view**. +10. Add the function code from the DynamoDB Lambda function file (available in the plugin package). +11. Click **Configuration** > **Environment Variables**. +12. Create the following two variables. + - Key = `GROUP_NAME`, value = `` e.g., `/aws/cloudtrail/dynamodb-trail` + - Key = `QUEUE_NAME`, value = `` e.g., `https://sqs.us-east-1.amazonaws.com/1111111111/dynamodb` +13. Save the function. +14. Click **Deploy**. + +### Automating the Lambda Function + +1. Go to the CloudWatch dashboard. +2. Go to **Events** > **Rules** on the left pane. +3. Click **Create Rule**. +4. Enter the name for the rule. For example, **cloudwatchToSqs**. +5. Under **Rule Type**, select **Schedule**. +6. Define the schedule. In **schedule pattern**, select a schedule that runs at a regular rate, such as every 10 minutes. +7. Enter the rate expression, meaning the rate at which the function should execute. This value must match the time specified in the lambda function code that calculates the time delta. For instance, if the function code is set to 2 minutes, set the rate to 2 minutes unless changed in the code. Then click **Next**. +8. Select the **Target1**. Select the **Target Type** as **AWS Service**. +9. Select **Target** as **Lambda Function**. +10. Select the lambda function created in the previous step (Export-DynamoDB-CloudWatch-Logs-To-SQS). +11. Add the tag if needed. Then click **Create Rule**. + +**Note:** Before making any changes to the lambda function code, first disable the rule you created. Deploy the change and then re-enable the rule. + +## Limitations + +- The DynamoDB plug-in does not support IPV6. +- You may need to disable management events in order to avoid heavy traffic and data loss in Guardium. Disabling management events disables logging of the following events: CreateTable, DeleteTable, ListTable, UpdateTable, DescribeTable events. +- The following important fields can not be mapped with DynamoDB audit logs: + - **Client HostName**: Not available with audit logs, so set as N.A. + +## Creating datasource profiles + +You can create a new datasource profile from the **Datasource Profile Management** page. + +### Procedure + +1. Go to **Manage > Universal Connector > Datasource Profile Management** +2. Click the **➕ (Add)** button. +3. You can create a profile by using one of the following methods: + + - To **Create a new profile manually**, go to the **"Add Profile"** tab and provide values for the following fields. + - **Name** and **Description**. + - Select a **Plug-in Type** from the dropdown. For example, `DynamoDB over Cloudwatch Connect 2.0` + + - To **Upload from CSV**, go to the **"Upload from CSV"** tab and upload an exported or manually created CSV file containing one or more profiles. + You can also choose from the following options: + - **Update existing profiles on name match** — Updates profiles with the same name if they already exist. + - **Test connection for imported profiles** — Automatically tests connections after profiles are created. + - **Use ELB** — Enables ELB support for imported profiles. You must provide the number of MUs to be used in the ELB process. + +**Note:** Configuration options vary based on the selected plug-in. + +## Configuring DynamoDB Over CloudWatch Kafka Connect 2.0 + +The following table describes the fields that are specific to DynamoDB over CloudWatch Kafka Connect 2.0 plugin. + +| Field | Description | +|--------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Name** | Unique name of the profile. | +| **Description** | Description of the profile. | +| **Plug-in** | Plug-in type for this profile. Select `DynamoDB Over Cloudwatch Connect 2.0`. A full list of available plug-ins are available on the **Package Management** page. | +| **Credential** | Select AWS Credentials or AWS Role ARN. The credential to authenticate with AWS. Must be created in **Credential Management**, or click **➕** to create one. For more information, see [Creating Credentials](https://www.ibm.com/docs/en/SSMPHH_12.x/com.ibm.guardium.doc.stap/guc/guc_credential_management.html). | +| **Kafka Cluster** | Select the appropriate Kafka cluster from the available Kafka cluster list or create a new Kafka cluster. For more information, see [Managing Kafka clusters](https://www.ibm.com/docs/en/SSMPHH_12.x/com.ibm.guardium.doc.stap/guc/guc_kafka_cluster_management.html). | +| **Label** | Grouping label. For example, customer name or ID. | +| **AWS account region** | Specifies the AWS region where your DynamoDB tables are located (e.g., us-east-1, eu-west-1). | +| **Log groups** | List of CloudWatch log groups to monitor. These are the log groups where DynamoDB audit logs (via CloudTrail) are exported. Format: `/aws/cloudtrail/` | +| **Filter pattern** | CloudWatch Logs filter pattern to apply. Use "None" to retrieve all logs, or specify a pattern to filter specific log events. | +| **Account ID** | Your AWS account ID (12-digit number). This identifies your AWS account. | +| **Cluster name** | The name of your DynamoDB cluster or table identifier. | +| **Ingestion delay (seconds)** | Default value is 900 seconds (15 minutes). This delay accounts for the time it takes for logs to be available in CloudWatch after being generated. | +| **No-traffic threshold (minutes)** | Default value is 60. If there is no incoming traffic for an hour, S-TAP displays a red status. Once incoming traffic resumes, the status returns to green. | +| **Unmask sensitive value** | Optional boolean flag. When enabled, sensitive values in the audit logs will not be masked. | +| **Use Enterprise Load Balancing (ELB)** | Enable this if ELB support is required. | +| **Managed Unit Count** | Number of Managed Units (MUs) to allocate for ELB. | + +**Note:** +- Ensure that the **profile name** is unique. +- Required credentials must be created before or during profile creation. +- The AWS credentials must have appropriate permissions to read CloudWatch logs and CloudTrail events. + +## Testing a Connection + +After creating a profile, you must test the connection to ensure the provided configuration is valid. + +### Procedure + +1. Select the new profile. +2. From the top menu, click **Test Connection**. +3. If the test is successful, you can proceed to installing the profile. + +--- + +## Installing a Profile + +Once the connection test is successful, you can install the profile on **Managed Units (MUs)** or **Edges**. The parsed audit logs are sent to the selected Managed Unit or Edge to be consumed by the **Sniffer**. + +### Procedure + +1. Select the profile. +2. From the **Install** menu, click **Install**. +3. From the list of available MUs and Edges that is displayed, select the ones that you want to deploy the profile to. + +--- + +## Uninstalling or reinstalling profiles + +An installed profile can be uninstalled or reinstalled if needed. + +### Procedure + +1. Select the profile. +2. From the list of available actions, select the desired option: **Uninstall** or **Reinstall**. + +--- diff --git a/docs/KafkaBasedUCs/MariaDBCloudwatchKafkaConnect.md b/docs/KafkaBasedUCs/MariaDBCloudwatchKafkaConnect.md new file mode 100644 index 000000000..0f7ea2bb8 --- /dev/null +++ b/docs/KafkaBasedUCs/MariaDBCloudwatchKafkaConnect.md @@ -0,0 +1,282 @@ +# Configuring MariaDB on AWS datasource profiles for Kafka Connect Plug-ins + +Create and configure datasource profiles through Central Manager for **MariaDB over Cloudwatch Kafka Connect** plug-ins. + +## Meet MariaDB over Cloudwatch Kafka Connect + +* **Environment:** AWS +* **Supported inputs:** Kafka connect Cloudwatch 2.0 (pull) +* **Supported Guardium versions:** + * Guardium Data Protection: Appliance bundle 12.2.2 or later + +Kafka-connect is a framework for streaming data between Apache Kafka and other systems. This connector enables monitoring of MariaDB audit logs through CloudWatch. + +## Creating and configuring a MariaDB database instance + +To create a new MariaDB instance, complete the steps in the [AWS Getting Started Guide](https://aws.amazon.com/getting-started/hands-on/create-mariadb-db/). + +**Note:** When setting the properties under **Additional Configuration**, in the **Log exports** section, select **Audit log** as the log type to publish to Amazon CloudWatch logs. + +## Enabling the MariaDB server audit Logs + +1. Edit **Inbound port** rule. +2. Create a new parameter group. +3. Create a new option group and add **MARIADB_AUDIT_PLUGIN**. +4. Modify **Parameter group** and **Option groups** in DB Instance. + +### Editing an inbound port rule + +1. Select your MariaDB instance. +2. Under **Connectivity & security**, click on the VPC security group. +3. Select **Inbound rules**, then click **Edit inbound rules**. +4. Set custom value to `0.0.0.0/0`. +5. Click **Add rules**, then click **Save rule**. + +### Creating a new parameter group + +To publish logs to CloudWatch, create a new parameter group and set the **log_output** parameter to `FILE`. When you create a database instance, it is associated with the default parameter group and cannot be modified. + +1. Open the Amazon RDS console (https://console.aws.amazon.com/rds). +2. In the navigation pane, choose **Parameter groups**. +3. Choose **Create parameter group** to open the Create parameter group dialog box. +4. In the **Parameter group family** list, choose your engine version. +5. In the **Group name** box, enter the name of the new DB parameter group. +6. In the **Description** box, enter a description for the new DB parameter group. +7. Click **Create**. + +### Configuring log_output. + +1. Select the parameter group, click on **Parameter group action** from the drop-down menu, then click **Edit**. +2. In the parameters filter search box, filter by `log_output`. +3. Using the drop-down menu, set the **log_output** parameter to `FILE`. +4. Click **Save changes**. + +### Creating a new option group and adding MARIADB_AUDIT_PLUGIN. + +You must add **MARIADB_AUDIT_PLUGIN** to enable Server Audit Logs. + +1. In the RDS dashboard, select **Option groups** and then click **Create group**. +2. In the **Create option group** window, complete the following steps.
+ a. For **Name**, type a name for the option group.
+ b. For **Description**, type a brief description of the option group.
+ c. For **Engine**, choose the MariaDB DB engine.
+ d. For **Major engine version**, choose the major version of the DB engine.
+ e. Click **Create**.
+3. To add MARIADB_AUDIT_PLUGIN, complete the following steps.
+ a. Select the created Option group, then click **Add options**.
+ b. Set **Option name** to `MARIADB_AUDIT_PLUGIN`.
+ c. Keep option setting parameters with default values.
+ d. Pass **SERVER_AUDIT_EXCL_USERS** value to `rdsadmin`.
+ e. Set the value for `SERVER_AUDIT_EVENTS` to `QUERY, CONNECT` to see query and connection logs.
+ f. To enable the option immediately, choose **Yes** for **Apply Immediately**.
+ g. Click **Add option**.
+ +For more information about adding the MariaDB plug-in to a MySQL instance, see [AWS documentation](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Appendix.MySQL.Options.AuditPlugin.html). + +**Note:** The **rdsadmin** user queries the database every second to check its health. This activity may cause the log file to grow quickly to a very large size, which could result in unnecessary data processing in the filter. If recording this activity is not required, add the `rdsadmin` user to the `SERVER_AUDIT_EXCL_USERS` list. + +### Modifying parameter and option groups in DB instance + +1. Choose the DB instance hyperlink, then choose **Modify**. +2. In the **Settings** section, confirm the password. +3. Under **Additional configuration**, use the drop-down menu to modify **DB parameter group** and **Option group**. +4. For the last section, keep the default settings and click **Continue**. + +## Connecting to MariaDB Instance + +1. Download and install MySQL Workbench. +2. Copy the endpoint and port of your MariaDB instance. +3. Open MySQL Workbench, choose a database connection, specify endpoint, port, and master credentials, then click **OK**. +4. Open MySQL Workbench query editor with the instance connection, then execute some queries. + +### Viewing MariaDB log entries on CloudWatch + +By default, each database instance has an associated log group with a name in this format: `/aws/rds/instance//audit`. You can use this log group, or you can create a new one and associate it with the database instance. + +1. On the AWS Console page, open the **Services** menu. +2. Enter `CloudWatch` in the search box. +3. Click **CloudWatch** to redirect to the CloudWatch dashboard. +4. In the left panel, select **Logs**. +5. Click **Log Groups**. + +## Exporting CloudWatch logs to SQS Using Lambda Function (Optional) + +To achieve load balancing of audit logs between different collectors, the audit logs can be exported from CloudWatch to SQS. + +### Creating the SQS Queue + +1. Go to https://console.aws.amazon.com/. +2. Click **Services**. +3. Search for SQS and click on **Simple Queue Services**. +4. Click **Create Queue**. +5. Select the type as **Standard**. +6. Enter the name for the queue. +7. Keep the rest of the default settings. + +### Create Policy for the Relevant IAM User + +1. For the IAM User using which the SQS logs are to be accessed in Guardium, complete the following steps. +2. Go to https://console.aws.amazon.com/. +3. Go to **IAM service** > **Policies** > **Create Policy**. +4. Select **service as SQS**. +5. Select the following checkboxes: **ListQueues**, **DeleteMessage**, **DeleteMessageBatch**, **GetQueueAttributes**, **GetQueueUrl**, **ReceiveMessage**, **ChangeMessageVisibility**, **ChangeMessageVisibilityBatch**. +6. In the resources, specify the ARN of the queue created in the previous step. +7. Click **Review policy** and specify the policy name. +8. Click **Create policy**. +9. Assign the policy to the user.
+ a. Log in to the IAM console as IAM user (https://console.aws.amazon.com/iam/).
+ b. Go to **Users** on the console and select the relevant IAM user to whom you want to give permissions.
+ c. In the **Permissions** tab, click **Add permissions**.
+ d. Click **Attach existing policies directly**.
+ e. Search for the policy created and check the checkbox next to it.
+ f. Click **Next: Review** > **Add permissions**.
+ +## Creating the Lambda Function + +### Create IAM Role + +Create the IAM role that will be used in the Lambda function setup. The AWS Lambda service requires permission to log events and write to the SQS created. Create the IAM Role **Export-DynamoDB-CloudWatch-to-SQS-Lambda** with **AmazonSQSFullAccess**, **CloudWatchLogsFullAccess**, and **CloudWatchEventsFullAccess** policies. + +1. Go to https://console.aws.amazon.com/. +2. Go to **IAM** > **Roles** > **Create Role**. +4. Under **Use case**, select **Lambda** and click **Next**. +5. Search for ``AmazonSQSFullAccess`` and select it. +6. Search for ``CloudWatchLogsFullAccess`` and select it. +7. Search for ``CloudWatchEventsFullAccess`` and select it. +8. Set the **Role Name**. For example, **Export-RDS-CloudWatch-to-SQS-Lambda**. Then click **Create role**. + +### Creating the Lambda Function + +1. Go to https://console.aws.amazon.com/. +2. Go to **Services**. Search for Lambda function. +3. Click **Functions** > **Create Function** +5. Keep **Author from Scratch** selected. +6. Set **Function name**. For example, **Export-RDS-CloudWatch-Logs-To-SQS**. +7. Under **Runtime**, select **Python 3.x**. +8. Under **Permissions**, select **Use an existing role** and select the IAM role created in the previous step (Export-RDS-CloudWatch-Logs-To-SQS). +9. Click **Create function** and navigate to **Code view**. +10. Add the function code from the DynamoDB Lambda function file (available in the plugin package). +11. Click **Configuration** > **Environment Variables**. +12. Create the following two variables. + - Key = `GROUP_NAME`, value = `` e.g., `/aws/rds/instance/mariadbsqs/audit` + - Key = `QUEUE_NAME`, value = `` e.g., `https://sqs.us-east-1.amazonaws.com/1111111111/mariadb` +13. Save the function. +14. Click **Deploy**. + + +### Automating the Lambda Function + +1. Go to the CloudWatch dashboard. +2. Go to **Events** > **Rules** on the left pane. +3. Click **Create Rule**. +4. Enter the name for the rule. For example, **cloudwatchToSqs**. +5. Under **Rule Type**, select **Schedule**. +6. Define the schedule. In **schedule pattern**, select a schedule that runs at a regular rate, such as every 10 minutes. +7. Enter the rate expression, meaning the rate at which the function should execute. This value must match the time specified in the lambda function code that calculates the time delta. For instance, if the function code is set to 2 minutes, set the rate to 2 minutes unless changed in the code. Then click **Next**. +8. Select the **Target1**. Select the **Target Type** as **AWS Service**. +9. Select **Target** as **Lambda Function**. +10. Select the lambda function created in the previous step (Export-RDS-CloudWatch-Logs-To-SQS). +11. Add the tag if needed. Then click **Create Rule**. + + +##### Important Note + +Before making any changes to the lambda function code, first disable the rule you created. Deploy the change and then re-enable the rule. + +## Limitations + +- The following important fields could not be mapped with MariaDB audit logs: + - **Source program**: This field is left blank since this information is not embedded in the messages pulled from AWS CloudWatch + - **OS User**: Not available with audit logs + - **Client HostName**: Not available with audit logs when connecting to MariaDB instance through SQL standard and third-party tools + - **serverIP**: This field is populated with 0.0.0.0, as this information is not embedded in the messages pulled from AWS CloudWatch + - **clientPort and serverPort**: Not available with audit logs +- For system-generated LOGIN_FAILED logs, the Dbuser value is not available, so it is set to `N.A`. +- Large SQL statements are truncated by AWS by default, which can cause a GuardUCInvalidRecordException as the event is no longer valid. +- Currently while using ELB, S-TAP registration is restricted to one primary MU, meaning the S-TAP and its logs appear only on the initial primary MU even when multiple primary MUs are present. + +## Creating datasource profiles + +You can create a new datasource profile from the **Datasource Profile Management** page. + +### Procedure + +1. Go to **Manage > Universal Connector > Datasource Profile Management** +2. Click the **➕ (Add)** button. +3. You can create a profile by using one of the following methods: + + - To **Create a new profile manually**, go to the **"Add Profile"** tab and provide values for the following fields. + - **Name** and **Description**. + - Select a **Plug-in Type** from the dropdown. For example, `MariaDB over Cloudwatch Kafka Connect 2.0` + + - To **Upload from CSV**, go to the **"Upload from CSV"** tab and upload an exported or manually created CSV file containing one or more profiles. + You can also choose from the following options: + - **Update existing profiles on name match** — Updates profiles with the same name if they already exist. + - **Test connection for imported profiles** — Automatically tests connections after profiles are created. + - **Use ELB** — Enables ELB support for imported profiles. You must provide the number of MUs to be used in the ELB process. + +**Note:** Configuration options vary based on the selected plug-in. + +## Configuring MariaDB Over CloudWatch Kafka Connect 2.0 + +The following table describes the fields that are specific to MariaDB over CloudWatch Kafka Connect 2.0 plugin. + +| Field | Description | +|--------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Name** | Unique name of the profile. | +| **Description** | Description of the profile. | +| **Plug-in** | Plug-in type for this profile. Select `AWS MariaDB Over Cloudwatch Connect 2.0`. A full list of available plug-ins are available on the **Package Management** page. | +| **Credential** | Select AWS Credentials or AWS Role ARN. The credential to authenticate with AWS. Must be created in **Credential Management**, or click **➕** to create one. For more information, see [Creating Credentials](https://www.ibm.com/docs/en/SSMPHH_12.x/com.ibm.guardium.doc.stap/guc/guc_credential_management.html). | +| **Kafka Cluster** | Select the appropriate Kafka cluster from the available Kafka cluster list or create a new Kafka cluster. For more information, see [Managing Kafka clusters](https://www.ibm.com/docs/en/SSMPHH_12.x/com.ibm.guardium.doc.stap/guc/guc_kafka_cluster_management.html). | +| **Label** | Grouping label. For example, customer name or ID. | +| **AWS account region** | Specifies the AWS region where your RDS MariaDB instance is located (e.g., us-east-1, eu-west-1). | +| **Log groups** | List of CloudWatch log groups to monitor. These are the log groups where MariaDB audit logs are exported. Format: `/aws/rds/instance//audit` | +| **Filter pattern** | CloudWatch Logs filter pattern to apply. Use "None" to retrieve all logs, or specify a pattern to filter specific log events. | +| **Account ID** | Your AWS account ID (12-digit number). This identifies your AWS account. | +| **Cluster name** | The name of your RDS MariaDB cluster or instance identifier. | +| **Ingestion delay (seconds)** | Default value is 900 seconds (15 minutes). This delay accounts for the time it takes for logs to be available in CloudWatch after being generated. | +| **No-traffic threshold (minutes)** | Default value is 60. If there is no incoming traffic for an hour, S-TAP displays a red status. Once incoming traffic resumes, the status returns to green. | +| **Unmask sensitive value** | Optional boolean flag. When enabled, sensitive values in the audit logs will not be masked. | +| **Use Enterprise Load Balancing (ELB)** | Enable this if ELB support is required. | +| **Managed Unit Count** | Number of Managed Units (MUs) to allocate for ELB. | + +**Note:** +- Ensure that the **profile name** is unique. +- Required credentials must be created before or during profile creation. +- The AWS credentials must have appropriate permissions to read CloudWatch logs. + +## Testing a Connection + +After creating a profile, you must test the connection to ensure the provided configuration is valid. + +### Procedure + +1. Select the new profile. +2. From the top menu, click **Test Connection**. +3. If the test is successful, you can proceed to installing the profile. + +--- + +## Installing a Profile + +Once the connection test is successful, you can install the profile on **Managed Units (MUs)** or **Edges**. The parsed audit logs are sent to the selected Managed Unit or Edge to be consumed by the **Sniffer**. + +### Procedure + +1. Select the profile. +2. From the **Install** menu, click **Install**. +3. From the list of available MUs and Edges that is displayed, select the ones that you want to deploy the profile to. + +--- + +## Uninstalling or reinstalling profiles + +An installed profile can be uninstalled or reinstalled if needed. + +### Procedure + +1. Select the profile. +2. From the list of available actions, select the desired option: **Uninstall** or **Reinstall**. + +--- + diff --git a/docs/KafkaBasedUCs/MsSQLOnPremJDBCKafkaConnect.md b/docs/KafkaBasedUCs/MsSQLOnPremJDBCKafkaConnect.md new file mode 100644 index 000000000..1e4d3f836 --- /dev/null +++ b/docs/KafkaBasedUCs/MsSQLOnPremJDBCKafkaConnect.md @@ -0,0 +1,225 @@ +# Configuring MSSQL datasource profiles for JDBC Kafka Connect Plug-ins + +Create and configure datasource profiles through through Central Manager for MSSQL OnPrem JDBC Kafka Connect plug-ins. + +## Meet MSSQL Over JDBC Connect +* Environments: On-prem +* Supported inputs: Kafka connect JDBC 2.0 (pull) +* Supported Guardium versions: + * Guardium Data Protection: Appliance bundle 12.2.2 or later + +Kafka-connect is a framework for streaming data between Apache Kafka and other systems. + +## Before you begin + +Download MSSQL JDBC driver from [mssql-jdbc-7.4.1.jre8](https://github.ibm.com/Activity-Insights/universal-connectors/blob/master/filter-plugin/logstash-filter-mssql-guardium/MssqlOverJdbcPackage/mssql-jdbc-7.4.1.jre8.jar). + +## Configuring the MSSQL database + +Create a database instance. This procedure requires an existing Microsoft SQL Server on-premises installation. + +## Enabling auditing + +1. Connect to the database.
+ a. Launch SQL Server Management Studio and provide the following connection details:
+       i. In the **Server Name** field, enter the **HostName**.
+       ii. Enter the username and master password that you set while creating the database.
+ b. Create database.
+ +2. Audit specifications.
+ a. SQL Server audit allows you to create server audits that can contain the following specifications:
+       i. Server audit specifications for server-level events.
+       ii. Database audit specifications for database-level events.
+ b. When you define an audit, you specify the output location for the results (the audit destination). The audit is created in a disabled state and does not automatically audit any actions. After the audit is enabled, the audit destination receives data from the audit.
+ +3. Create an audit.
+ a. Create an audit in Management Studio by going to **Security > Audits > New Audit**.
+ b. Enter the filepath.
+ c. In the **Maximum file size** field, deselect the **Unlimited** checkbox and enter a specific value.
+ d. Keep the remaining configurations as default.
+ e. Click **OK**.
+ f. Right-click the audit you created and select **Enable**.
+ +4. Create server or database audit specifications. + + - Create a server audit specification.
+ a. In **Management**, navigate to **Security** and expand it.
+ b. Right-click **Server Audit Specifications** and select **New Audit Specification**.
+ c. Select the audit that you created in the previous step.
+ d. Configure the audit log groups based on your requirements. For more information on audit log groups, see the Microsoft documentation website.
+ e. Click **OK**.
+ f. Right-click the database audit specification you created and select **Enable**.
+ + - Create a database audit specification.
+ a. In **Management**, navigate to **Databases** and expand it. Then expand **Security** under the database.
+ b. Right-click **Database Audit Specifications** and select **New Audit Specification**.
+ c. Select the audit that you created in the previous step.
+ d. Configure the audit log groups based on your requirements. For more information on audit log groups, see the Microsoft documentation website.
+ e. Click **OK**.
+ f. Right-click the database audit specification you created and select **Enable**.
+ + +5. Create audit specifications for capturing errors.
+ Excute the following TSQL to capture error events. + + CREATE EVENT SESSION [] + ON SERVER + ADD EVENT sqlserver.error_reported + ( + ACTION + ( + sqlserver.client_hostname, + sqlserver.database_id, + sqlserver.sql_text, + sqlserver.username, + sqlserver.database_name, + sqlserver.session_id, + sqlserver.server_instance_name + ) + WHERE + ( + [severity] >= (11) + ) + ) + ADD TARGET package0.asynchronous_file_target + ( + SET filename=N'' + ) + WITH + ( + MAX_MEMORY=4096 KB, + EVENT_RETENTION_MODE=ALLOW_SINGLE_EVENT_LOSS, + MAX_DISPATCH_LATENCY=30 SECONDS, + MAX_EVENT_SIZE=0 KB, + MEMORY_PARTITION_MODE=NONE, + TRACK_CAUSALITY=OFF, + STARTUP_STATE=ON + ); + GO + + ALTER EVENT SESSION []ON SERVER + STATE = START; + GO + + **Note:** The event name must be the same for **TSQL Create** and **Alter event**. + The xel file path name mentioned in TSQL must match the SQL statement mentioned in input plugin for the 'failure' tag. + + +6. Create a non-admin user to access the audit table without exposing admin credentials.
+ a. Log in to the database using admin credentials and run the following queries:
+ + ```sql + CREATE LOGIN WITH PASSWORD = ''; + CREATE USER FOR LOGIN ; + GRANT SELECT ON sys.fn_get_audit_file TO ; + GRANT CONTROL SERVER TO ; + ``` + + b. In the **Input** section, set the database name as **msdb**.
+ + ```properties + jdbc_connection_string => "jdbc:sqlserver://:;databaseName=msdb;" + ``` + + c. Use the login credentials created in the previous step for the JDBC connection.
+ + ```properties + jdbc_user => "" + jdbc_password => "" + ``` + +## Creating datasource profiles + +You can create a new datasource profile from the **Datasource Profile Management** page. + +### Procedure + +1. Go to **Manage > Universal Connector > Datasource Profile Management** +2. Click the **➕ (Add)** button. +3. You can create a profile by using one of the following methods: + + * To **Create a new profile manually**, go to the **"Add Profile"** tab and provide values for the following fields. + * **Name** and **Description**. + * Select a **Plug-in Type** from the dropdown. For example, `MSSQL OnPrem Over JDBC Kafka Connect 2.0`. + + * To **Upload from CSV**, go to the **"Upload from CSV"** tab and upload an exported or manually created CSV file + containing one or more profiles. You can also choose from the following options: + * **Update existing profiles on name match** — Updates profiles with the same name if they already exist. + * **Test connection for imported profiles** — Automatically tests connections after profiles are created. + * **Use ELB** — Enables ELB support for imported profiles. You must provide the number of MUs to be used in the + ELB process. + +**Note:** Configuration options vary based on the selected plug-in. + +## Configuration: JDBC Kafka Connect 2.0-based Plugins + +The following table describes the fields that are specific to JDBC Kafka Connect 2.0 and similar plugins. + +| Field | Description |Value/Example | +|--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------| +| **Name** | Unique name of the profile. | MSSQL_ONPREM_JDBC_KAFKA_CONNECT | +| **Description** | Description of the profile. | Profile for MSSQL OnPrem over JDBC connect 2.0 plug-in | +| **Plug-in** | Plug-in type for this profile. A full list of available plug-ins is found in the **Package Management** page. | MSSQL OnPrem over JDBC connect 2.0 | +| **Credential** | The credential to authenticate with the datasource. Must be created in **Credential Management**, or click **➕** to create one. | | +| **Kafka Cluster** | Kafka cluster to deploy the Universal Connector. |Select from existing Kafka clusters attached to central management| +| **Label** | Grouping label (e.g., customer name or ID). | | +| **JDBC Driver Library** | JDBC driver for the database. |Download from here [mssql-jdbc-7.4.1.jre8](https://github.ibm.com/Activity-Insights/universal-connectors/blob/master/filter-plugin/logstash-filter-mssql-guardium/MssqlOverJdbcPackage/mssql-jdbc-7.4.1.jre8.jar) | +| **Initial time (milliseconds)** | Initial polling time for audit logs. | 0 | +| **No traffic threshold (minutes)** | Threshold setting for inactivity detection. |60 (default) | +| **Query for MSSQL Audit Log** | SQL query used to extract audit logs. | SELECT * FROM (SELECT CAST(event_time AS DATETIME) AS event_time, CAST(succeeded AS INT) AS succeeded, session_id, CAST(database_name AS NVARCHAR(128)) AS database_name, CAST(client_ip AS NVARCHAR(50)) AS client_ip, CAST(server_principal_name AS NVARCHAR(128)) AS server_principal_name, CAST(application_name AS NVARCHAR(128)) AS application_name, CAST(statement AS NVARCHAR(4000)) AS statement, CAST(server_instance_name AS NVARCHAR(128)) AS server_instance_name, '' AS host_name, DATEDIFF_BIG(ns, '1970-01-01 00:00:00.00000', event_time) AS updatedeventtime, additional_information FROM (SELECT * FROM sys.fn_get_audit_file('/var/opt/mssql/audit/*.sqlaudit', DEFAULT, DEFAULT)) AS audit_data WHERE schema_name NOT IN ('sys') AND object_name NOT IN ('dbo', 'syssubsystems', 'fn_sysdac_is_currentuser_sa', 'backupmediafamily', 'backupset', 'syspolicy_configuration', 'syspolicy_configuration_internal', 'syspolicy_system_health_state', 'syspolicy_system_health_state_internal', 'fn_syspolicy_is_automation_enabled', 'spt_values', 'sysdac_instances_internal', 'sysdac_instances') AND database_principal_name NOT IN ('public') AND ((succeeded = 1) OR (succeeded = 0 AND statement LIKE '%Login failed%')) AND statement != '') AS KAFKA_AUDIT_VIEW +| **Query for SQL failed** | SQL query used to extract audit logs. | SELECT * FROM (SELECT CAST(timestamp_utc AS DATETIME) as timestamp_utc,event_data,DATEDIFF_BIG(ns, '1970-01-01 00:00:00.00000', timestamp_utc) AS updated_timestamp FROM sys.fn_xe_file_target_read_file('/var/opt/mssql/log/*.xel',null,null,null)) AS MSSQL_SQL_FAILED_VIEW | +| **Tracking column type for MSSQL Audit Log** | Tracking column type for MSSQL Audit Log. | timestamp | +| **Tracking column for MSSQL Audit Log** | Tracking column. | event_time | +| **Tracking column type for SQLFailedLog** | Tracking column type for SQLFailedLog. | timestamp | +| **Tracking column for SQLFailedLog** | Tracking column. | event_time | +| **Connection String** | Connection string connect to MSSQL database. | jdbc:sqlserver://;serverName=;databaseName= | + +**Note:** + +- Depending on the plugin type, the configuration may require either: + - A **Connection URL**, or + - Separate fields for **Hostname**, **Port**, and **Service Name / SID** +- Ensure that the **profile name** is unique. +- Required credentials must be created before or during profile creation. + +--- + +## Testing a Connection + +After creating a profile, you must test the connection to ensure the provided configuration is valid. + +### Procedure + +1. Select the new profile. +2. From the top menu, click **Test Connection**. +3. If the test is successful, you can proceed to installing the profile. + +--- + +## Installing a Profile + +Once the connection test is successful, you can install the profile on **Managed Units (MUs)** or **Edges**. The parsed +audit logs are sent to the selected Managed Unit or Edge to be consumed by the **Sniffer**. + +### Procedure + +1. Select the profile. +2. From the **Install** menu, click **Install**. +3. From the list of available MUs and Edges that is displayed, select the ones that you want to deploy the profile to. + +--- + +## Uninstalling or reinstalling profiles + +An installed profile can be uninstalled or reinstalled if needed. + +### Procedure + +1. Select the profile. +2. From the list of available actions, select the desired option: **Uninstall** or **Reinstall**. + +--- + +## Limitations + +- The MSSQL On‑Prem JDBC Kafka Connect plug‑in multiple S‑TAP entries may appear in the S‑TAP report. \ No newline at end of file diff --git a/docs/KafkaBasedUCs/OpensearchCloudwatchKafkaConnect.md b/docs/KafkaBasedUCs/OpensearchCloudwatchKafkaConnect.md new file mode 100644 index 000000000..76f1d2816 --- /dev/null +++ b/docs/KafkaBasedUCs/OpensearchCloudwatchKafkaConnect.md @@ -0,0 +1,157 @@ +# Configuring Amazon OpenSearch datasource profiles for Kafka Connect Plug-ins + +Create and configure datasource profiles through Central Manager for **Amazon OpenSearch over CloudWatch Kafka Connect** plug-ins. + +### Meet Amazon OpenSearch over CloudWatch Connect + +* Tested versions: v1 +* Environments: AWS +* Supported inputs: Kafka connect Cloudwatch 2.0 (pull) +* Supported Guardium versions: + * Guardium Data Protection: Appliance bundle 12.2.2 or later + +Kafka-connect is a framework for streaming data between Apache Kafka and other systems. This connector enables monitoring of Amazon OpenSearch audit logs through CloudWatch. + +## Configuring Amazon OpenSearch Service + +### Before you begin + +* [Prerequisites](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/setting-up.html) + +### Procedure + +1. Go to https://console.aws.amazon.com/. +2. Search and navigate to ```Amazon OpenSearch Service```. +3. To create an OpenSearch domain, see [Getting started with Amazon OpenSearch Service guide](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/gsg.html). + +## Enabling Audit Logs + +1. To Enable audit logs for **CloudWatch Logs** and **OpenSearch Dashboard**, see [Enabling Audit logs](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/audit-logs.html#audit-log-enabling). + +### Viewing Audit Logs on CloudWatch + +By default, each OpenSearch domain has associated log groups with names in this format: +- `/aws/OpenSearchService//audit` +- `/aws/OpenSearchService//profiler` + +1. Open the CloudWatch console https://console.aws.amazon.com/cloudwatch/. +2. In the navigation pane, choose **Log groups**. +3. Choose the **log group** that you specified while enabling audit logs. Within the log group, OpenSearch Service creates a log stream for each node in your domain. +4. In the **Log streams**, select **Search all**. +5. For the read and write events, see the corresponding logs. This process may take several seconds. + +### Supported Audit Log Types + +Cluster communication occurs over two separate layers: **REST layer** and **Transport layer**. The following is the list of **Audit log Categorie**s, with their availability determined by the communication layers: + +* FAILED_LOGIN +* MISSING_PRIVILEGES +* BAD_HEADERS +* SSL_EXCEPTION +* GRANTED_PRIVILEGES +* OPENSEARCH_SECURITY_INDEX_ATTEMPT +* AUTHENTICATED +* INDEX_EVENT +* COMPLIANCE_DOC_READ +* COMPLIANCE_DOC_WRITE +* COMPLIANCE_INTERNAL_CONFIG_READ +* COMPLIANCE_INTERNAL_CONFIG_WRITE + +For more information about the audit logging category and layers, see [Audit log layers and categories](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/audit-logs.html#audit-log-layers). + +For more information about the audit logging fields, see [Audit log field reference](https://docs.opensearch.org/docs/latest/security/audit-logs/field-reference/). + +**Note:** OpenSearch generates a large volume of background audit logs by default. Configure the audit settings appropriately to limit unnecessary entries in the audit logs. + +## Limitations + +- Audit logging in OpenSearch can be accessed in two different ways – via the OpenSearch Dashboards or through CloudWatch Logs. However, this filter plugin only parses and processes audit logs that are streamed to CloudWatch. Audit logs stored directly in OpenSearch indices or viewed in the Dashboards are not supported for parsing. +- FAILED_LOGIN REST messages will appear in **Full SQL** and **Failed Logins** report. +- Certain reserved keywords (template, mappings, get, aliases, user) are automatically prefixed with an underscore (_) during sanitization to prevent OpenSearch URI parsing errors or endpoint conflicts. +- **ClientHostName** is not available in the audit logs for OpenSearch. + +## Creating datasource profiles + +You can create a new datasource profile from the **Datasource Profile Management** page. + +### Procedure + +1. Go to **Manage > Universal Connector > Datasource Profile Management** +2. Click the **➕ (Add)** button. +3. You can create a profile by using one of the following methods: + + - To **Create a new profile manually**, go to the **"Add Profile"** tab and provide values for the following fields. + - **Name** and **Description**. + - Select a **Plug-in Type** from the dropdown. For example, `Amazon OpenSearch Over Cloudwatch Connect 2.0`. + + - To **Upload from CSV**, go to the **"Upload from CSV"** tab and upload an exported or manually created CSV file containing one or more profiles. + You can also choose from the following options: + - **Update existing profiles on name match** — Updates profiles with the same name if they already exist. + - **Test connection for imported profiles** — Automatically tests connections after profiles are created. + - **Use ELB** — Enables ELB support for imported profiles. You must provide the number of MUs to be used in the ELB process. + +**Note:** Configuration options vary based on the selected plug-in. + +## Configuring Amazon OpenSearch Over CloudWatch Kafka Connect 2.0 + +The following table describes the fields that are specific to Amazon OpenSearch over CloudWatch Kafka Connect 2.0 plugin. + +| Field | Description | +|--------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Name** | Unique name of the profile. | +| **Description** | Description of the profile. | +| **Plug-in** | Plug-in type for this profile. Select `Amazon OpenSearch Over Cloudwatch Connect 2.0`. A full list of available plug-ins are available on the **Package Management** page. | +| **Credential** | Select AWS Credentials or AWS Role ARN. The credential to authenticate with AWS. Must be created in **Credential Management**, or click **➕** to create one. For more information, see [Creating Credentials](https://www.ibm.com/docs/en/SSMPHH_12.x/com.ibm.guardium.doc.stap/guc/guc_credential_management.html). | +| **Kafka Cluster** | Select the appropriate Kafka cluster from the available Kafka cluster list or create a new Kafka cluster. For more information, see [Managing Kafka clusters](https://www.ibm.com/docs/en/SSMPHH_12.x/com.ibm.guardium.doc.stap/guc/guc_kafka_cluster_management.html). | +| **Label** | Grouping label. For example, customer name or ID. | +| **AWS account region** | Specifies the AWS region where your OpenSearch domain is located (e.g., us-east-1, eu-west-1). | +| **Log groups** | List of CloudWatch log groups to monitor. These are the log groups where OpenSearch audit logs are exported. Format: `/aws/OpenSearchService//audit` | +| **Filter pattern** | CloudWatch Logs filter pattern to apply. Use "None" to retrieve all logs, or specify a pattern to filter specific log events. | +| **Account ID** | Your AWS account ID (12-digit number). This identifies your AWS account. | +| **Cluster name** | The name of your OpenSearch domain identifier. | +| **Ingestion delay (seconds)** | Default value is 900 seconds (15 minutes). This delay accounts for the time it takes for logs to be available in CloudWatch after being generated. | +| **No-traffic threshold (minutes)** | Default value is 60. If there is no incoming traffic for an hour, S-TAP displays a red status. Once incoming traffic resumes, the status returns to green. | +| **Use Enterprise Load Balancing (ELB)** | Enable this if ELB support is required. | +| **Managed Unit Count** | Number of Managed Units (MUs) to allocate for ELB. | + +**Note:** +- Ensure that the **profile name** is unique. +- Required credentials must be created before or during profile creation. +- The AWS credentials must have appropriate permissions to read CloudWatch logs. + +--- + +## Testing connections + +After creating a profile, you must test the connection to ensure that the provided configuration is valid. + +### Procedure + +1. Select the new profile. +2. From the top menu, click **Test Connection**. +3. If the test is successful, you can proceed to installing the profile. + +--- + +## Installing profiles + +Once the connection test is successful, you can install the profile on **Managed Units (MUs)** or **Edges**. The parsed audit logs are sent to the selected Managed Unit or Edge to be consumed by the **Sniffer**. + +### Procedure + +1. Select the profile. +2. From the **Install** menu, click **Install**. +3. A list of available MUs and Edges are displayed. Choose the specific MUs and Edges to which you want to apply the new profile. + +--- + +## Uninstalling or reinstalling profiles + +An installed profile can be uninstalled or reinstalled if needed. + +### Procedure + +1. Select the profile. +2. From the list of available actions, select the desired option **Uninstall** or **Reinstall**. + +--- diff --git a/docs/KafkaBasedUCs/OracleAutonomousDatabaseKafkaConnect.md b/docs/KafkaBasedUCs/OracleAutonomousDatabaseKafkaConnect.md new file mode 100644 index 000000000..c33a5a4f8 --- /dev/null +++ b/docs/KafkaBasedUCs/OracleAutonomousDatabaseKafkaConnect.md @@ -0,0 +1,220 @@ +# Configuring Oracle Autonomous Database datasource profiles for JDBC Kafka connect plug-ins + +Create and configure datasource profiles through Central Manager for **Oracle Autonomous Database JDBC Kafka Connect** plug-ins. + +## Meet Oracle Autonomous Database Over JDBC Connect + +* Environments: Oracle Cloud Infrastructure (OCI) +* Tested versions: 19c +* Supported inputs: Kafka Input (pull) +* Supported Guardium versions: + * Guardium Data Protection: Appliance bundle 12.2.1 or later + +Kafka-connect is a framework for streaming data between Apache Kafka and other systems. This connector enables monitoring of Oracle Autonomous Database audit logs through JDBC connectivity. + +## Prerequisites + +Before you begin, make sure that you have the following prerequisites: +- An Oracle Cloud Infrastructure (OCI) account +- Access to create and manage Oracle Autonomous Database instances +- Oracle JDBC driver (`ojdbc8.jar` or later) + +## Creating and configuring Oracle Autonomous Database + +### Procedure + +1. Log in to the (Oracle Cloud Infrastructure Console)[https://cloud.oracle.com/]. +2. From the navigation menu, click **Oracle Database** > **Autonomous Database**. +3. Click **Create Autonomous Database**. +4. Configure the following settings: + - **Compartment**: Select the compartment in which you want to create the database. + - **Display name**: Enter a descriptive name for the database. + - **Database name**: Enter a unique database name. Use alphanumeric characters only. + - **Workload type**: Select the workload type that matches your use case: + - **Transaction Processing** (ATP) for online transaction processing (OLTP) workloads. + - **Data Warehouse** (ADW) for analytics workloads. + - **Deployment type**: Select **Shared Infrastructure** or **Dedicated Infrastructure**. +5. Configure the database settings: + - **Database version**: Select the Oracle Database version. Oracle Database 19c or later is recommended. + - **OCPU count**: Select the number of OCPUs. + - **Storage**: Specify the storage capacity in TB. + - **Auto scaling**: Enable this option to automatically scale resources. +6. Set the administrator credentials: + - **Username**: The default administrator username is `ADMIN`. + - **Password**: Enter a strong password that meets Oracle's complexity requirements. + - **Confirm password**: Re-enter the password. +7. Configure network access: + - **Access Type**: Select **Secure access from everywhere** or **Private endpoint access only** based on your security requirements. + - **Mutual TLS (mTLS) authentication**: Keep this option enabled for enhanced security (recommended). +8. Click **Create Autonomous Database**. +9. Wait for the database to be provisioned. The status changes from **PROVISIONING** to **AVAILABLE**. + + +## Downloading the wallet + +Oracle Autonomous Database requires a wallet file for secure connections. The wallet contains credentials and connection information. + +### Procedure + +1. Open the OCI Console and navigate to your Autonomous Database instance. +2. On the Autonomous Database details page, click **Database connection**. +3. In the **Database Connection** dialog, click **Download wallet**. +4. In the **Download wallet** dialog, configure the following settings: + - **Wallet type**: Select **Instance Wallet** (recommended) or **Regional Wallet**. + - **Instance Wallet**: Contains connection information for a single database instance. + - **Regional Wallet**: Contains connection information for all Autonomous Databases in a region. + - **Password**: Enter a password to protect the wallet file. You must provide this password when you configure the JDBC connection. + - **Confirm password**: Re-enter the password. +5. Click **Download**. +6. Save the wallet file (e.g., `Wallet_DatabaseName.zip`) to a secure location. +7. Extract the wallet ZIP file to a directory on the system on which Guardium Universal Connector runs. +8. Save the path of the extracted wallet directory. You need this path when you configure the datasource profile. + +### Wallet contents + +The wallet ZIP file contains the following files: +- **cwallet.sso**: Oracle wallet file (auto-login format) +- **ewallet.p12**: Oracle wallet file (PKCS#12 format) +- **tnsnames.ora**: Network service names and connection strings +- **sqlnet.ora**: SQL*Net configuration +- **ojdbc.properties**: JDBC connection properties +- **keystore.jks**: Java keystore (if applicable) +- **truststore.jks**: Java truststore (if applicable) + +**Note:** +- Keep the wallet file secure. It contains credentials that provide access to your database. +- The wallet password is separate from the database ADMIN password. +- If you regenerate the wallet, you must update all applications that use the old wallet. +- For production environments, consider using Instance Wallet for better security isolation. + +## Enabling auditing + +Oracle Autonomous Database supports both traditional auditing and unified auditing. For this connector, use traditional auditing as it provides audit records in a format that can be queried by using JDBC. + +### Connecting to the database + +1. Use an SQL client that supports Oracle connections, such as SQL*Plus, SQL Developer, or another JDBC-based tool. +2. Configure the connection by using the wallet: + - **Connection Type**: TNS + - **Network Alias**: Select a service name from `tnsnames.ora`, such as `databasename_high`, `databasename_medium`, or `databasename_low`. + - **Wallet Location**: Specify the directory in which you extracted the wallet files. + - **Username**: `ADMIN` (or another user with appropriate privileges) + - **Password**: The ADMIN password set when you created the database + +### Applying audit policies + +Connect to your Oracle Autonomous Database using the `ADMIN` user and run the SQL commands for your auditing requirements. + +**Important:** Use traditional auditing commands (`AUDIT` statement), not unified auditing (`CREATE AUDIT POLICY`). Traditional auditing records can be queried from standard audit views that are accessible through JDBC. + +For more information about traditional auditing, see: +- [Oracle Documentation: AUDIT (Traditional Auditing)](https://docs.oracle.com/en/database/oracle/oracle-database/19/sqlrf/AUDIT-Traditional-Auditing.html) + + +## Creating datasource profiles + +You can create a new datasource profile from the **Datasource Profile Management** page. + +### Procedure + +1. Go to **Manage > Universal Connector > Datasource Profile Management** +2. Click the **➕ (Add)** button. +3. You can create a profile by using one of the following methods: + + * To **Create a new profile manually**, go to the **"Add Profile"** tab and provide values for the following fields. + * **Name** and **Description**. + * Select a **Plug-in Type** from the dropdown. For example, `Oracle Autonomous Database Over JDBC Kafka Connect 2.0`. + + * To **Upload from CSV**, go to the **"Upload from CSV"** tab and upload an exported or manually created CSV file containing one or more profiles. You can also choose from the following options: + * **Update existing profiles on name match** — Updates profiles with the same name if they already exist. + * **Test connection for imported profiles** — Automatically tests connections after profiles are created. + * **Use ELB** — Enables ELB support for imported profiles. You must provide the number of MUs to be used in the ELB process. + +**Note:** Configuration options vary based on the selected plug-in. + +## Configuration: Oracle Autonomous Database JDBC Kafka Connect 2.0 + +The following table describes the fields that are specific to Oracle Autonomous Database JDBC Kafka Connect 2.0 plugin. + +| Field | Description | +|-------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Name** | Unique name of the profile. | +| **Description** | Description of the profile. | +| **Plug-in** | Plug-in type for this profile. Select `Oracle Autonomous Database Over JDBC Kafka Connect 2.0`. A list of available plug-ins is available on the **Package Management** page. | +| **Credential** | The credential to authenticate with the datasource. You can create credentials in **Credential Management**, or click **➕** to create one. Use the `audit_reader` user credentials or ADMIN credentials. | +| **Kafka Cluster** | Select the appropriate Kafka cluster from the available Kafka cluster list or create a new Kafka cluster. For more information, see [Managing Kafka clusters](https://www.ibm.com/docs/en/SSMPHH_12.x/com.ibm.guardium.doc.stap/guc/guc_kafka_cluster_management.html). | +| **Label** | Grouping label. For example, customer name or ID. | +| **JDBC Driver Library** | Upload the Oracle JDBC driver (ojdbc8.jar or later). Download from [Oracle JDBC Downloads](https://www.oracle.com/database/technologies/appdev/jdbc-downloads.html). | +| **Connection URL** | Full JDBC connection string for Oracle Autonomous Database. Format: `jdbc:oracle:thin:@`
The service name can be found in the `tnsnames.ora` file within the wallet. | +| **Query** | SQL query to extract audit logs. Example: `SELECT * FROM DBA_AUDIT_TRAIL WHERE TIMESTAMP > ? ORDER BY TIMESTAMP` | +| **Initial Time** | Initial polling time for audit logs. Format: `YYYY-MM-DD HH:MM:SS` or use relative time like `-1h` for one hour ago. | +| **No Traffic Threshold** | Threshold setting for inactivity detection (in minutes). Default: 60. If there is no incoming traffic for this duration, S-TAP displays a red status. | +| **Use Enterprise Load Balancing (ELB)** | Enable this if ELB support is required. | +| **Managed Unit Count** | Number of Managed Units (MUs) to allocate for ELB. | + +**Note:** +- Make sure that the **profile name** is unique. +- Required credentials must be created before or during profile creation. +- The wallet directory path must be accessible from the system on which the Universal Connector runs. +- The JDBC driver must be compatible with your Oracle Autonomous Database version. +- The connection URL must include the `TNS_ADMIN` parameter that points to the wallet directory. + +### Example connection URL + +``` +jdbc:oracle:thin:@myatp_high?TNS_ADMIN=/opt/guardium/wallets/myatp_wallet +``` + +In this example: +- `myatp_high` is the service name from `tnsnames.ora`. +- `/opt/guardium/wallets/myatp_wallet` is the directory that contains the extracted wallet files. + +--- + +## Testing a connection + +After creating a profile, test the connection to make sure that the provided configuration is valid. + +### Procedure + +1. Select the new profile. +2. From the top menu, click **Test Connection**. +3. If the test is successful, you can proceed to installing the profile. +4. If the test fails, verify the following items: + - The wallet files are in the correct location and are accessible. + - The wallet password is correct (if required). + - The service name in the connection URL matches an entry in `tnsnames.ora`. + - The JDBC driver is compatible with your database version. + - Network connectivity to Oracle Cloud Infrastructure. + +--- + +## Installing a profile + +Once the connection test is successful, you can install the profile on Managed Units (MUs) or Edges. The parsed audit logs are sent to the selected Managed Unit or Edge to be consumed by the Sniffer. + +### Procedure + +1. Select the profile. +2. From the **Install** menu, click **Install**. +3. From the list of available MUs and Edges that is displayed, select the ones to which you want to deploy the profile. + +--- + +## Uninstalling or reinstalling profiles + +You can uninstall or reinstall an installed profile, if needed. + +### Procedure + +1. Select the profile. +2. From the list of available actions, select **Uninstall** or **Reinstall**. + + + +## Limitations + +1. **Wallet Management**: The wallet files must be manually distributed to all systems on which the Universal Connector runs. If the wallet is regenerated, all connectors must be updated. + +2. **Connection Pooling**: JDBC connection pooling behavior can vary depending on the connector configuration and database workload. + diff --git a/docs/KafkaBasedUCs/OracleRDSOverCloudwatchKafkaConnect.md b/docs/KafkaBasedUCs/OracleRDSOverCloudwatchKafkaConnect.md new file mode 100644 index 000000000..066dafad0 --- /dev/null +++ b/docs/KafkaBasedUCs/OracleRDSOverCloudwatchKafkaConnect.md @@ -0,0 +1,279 @@ +# Configuring Oracle RDS datasource profiles for Kafka Connect Plug-ins + +Create and configure datasource profiles through Central Manager for **Oracle RDS over CloudWatch Kafka Connect** plug-ins. + +### Meet Oracle RDS over CloudWatch Connect + +* Environments: AWS +* Supported inputs: Kafka connect Cloudwatch 2.0 (pull) +* Supported Guardium versions: + * Guardium Data Protection: Appliance bundle 12.2.2 or later + +Kafka-connect is a framework for streaming data between Apache Kafka and other systems. This connector enables monitoring of Oracle RDS audit logs through CloudWatch. + +## Configuring AWS RDS Oracle + +For detailed instructions on creating and configuring an AWS RDS Oracle database instance, see [Creating an Oracle DB instance and connecting to a database on an Oracle DB instance](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/CHAP_GettingStarted.CreatingConnecting.Oracle.html#CHAP_GettingStarted.Creating.Oracle). + +### Additional configuration requirements + +After creating your Oracle RDS instance following the Amazon documentation, make sure you complete the following steps. + +1. **Enable CloudWatch Log Exports:** + - In the Additional Configuration section, under Log exports, select the log type '**Audit**' from Amazon CloudWatch log options. + - Click on **Add Rule** and **Save** changes. + - **Note:** You might need to restart the database for changes to take effect. + +2. **Configure Security Group:** + - Edit the VPC security group associated with your database instance to allow traffic on port **1521**. + - In the **Inbound Rules** section, add a rule with: + - Type: Oracle-RDS + - Protocol: TCP + - Port Range: 1521 + - Source: Configure based on your security requirements (specific IP address or IP range) + +## Enabling Auditing + +### Configuring parameter group + +1. Enable auditing by setting up parameters on the parameter group and associating them with the database instance. + a. Select **Parameter Groups** from the left pane on Amazon RDS. + b. Select the newly created parameter group. + c. Click **Edit parameters** on the right corner. + d. Add the following setting: + + ``` + audit_trail = XML, EXTENDED + ``` + +### Associating DB parameter group to database instance + +1. Click **RDS** > **Databases** from the left panel. +2. Select the **Oracle database** instance to be updated. Then click **Modify**. +4. In the **Additional Configuration** section, under database options, select the newly created group from the **DB Parameter Group** drop-down. +5. Click **Continue**. +6. Select the database instance that, in its configuration section, shows the status for the DB Parameter Group as **pending-reboot**. +7. Reboot the Database instance for the changes to take effect. + +### Applying Audit Policies + +To allow the connector to parse and analyze your database queries, you must enable **Traditional Auditing** using the `AUDIT` command. Traditional Auditing records are exported to CloudWatch, making them accessible to this connector. + +**Important:** Do **not** use `CREATE AUDIT POLICY` (Unified Auditing). Unified Auditing records are stored inside the database and cannot be exported to CloudWatch. You must use the Traditional Auditing commands shown below. + +Connect to your Oracle RDS database using your master user (e.g., `ADMIN`) via a SQL client such as SQL*Plus, SQL Developer, or any JDBC-based tool. Depending on your security and compliance requirements, run the appropriate commands below. + +For more information about Traditional Auditing, see: +- [AWS Blog: Security Auditing in Amazon RDS for Oracle](https://aws.amazon.com/blogs/database/part-1-security-auditing-in-amazon-rds-for-oracle/) +- [Oracle Documentation: AUDIT (Traditional Auditing)](https://docs.oracle.com/en/database/oracle/oracle-database/19/sqlrf/AUDIT-Traditional-Auditing.html) + +#### Option 1: Audit all data modifications globally (All Users, All Tables) + +**Note:** This generates high log volume. It captures reads/writes on any table by any user. + +```sql +-- Capture all reads +AUDIT SELECT ANY TABLE BY ACCESS; + +-- Capture all writes +AUDIT INSERT ANY TABLE BY ACCESS; +AUDIT UPDATE ANY TABLE BY ACCESS; +AUDIT DELETE ANY TABLE BY ACCESS; + +-- Capture stored procedure executions +AUDIT EXECUTE ANY PROCEDURE BY ACCESS; +``` + +#### Option 2: Audit specific schema operations + +To audit operations on a specific schema (e.g., `MYSCHEMA`): + +```sql +-- Audit all operations on tables in MYSCHEMA +AUDIT SELECT TABLE, INSERT TABLE, UPDATE TABLE, DELETE TABLE BY MYSCHEMA BY ACCESS; + +-- Audit procedure executions in MYSCHEMA +AUDIT EXECUTE PROCEDURE BY MYSCHEMA BY ACCESS; +``` + +#### Option 3: Audit specific tables + +To audit operations on specific tables: + +```sql +-- Audit a specific table +AUDIT SELECT, INSERT, UPDATE, DELETE ON MYSCHEMA.MYTABLE BY ACCESS; + +-- Audit multiple specific tables +AUDIT SELECT, INSERT, UPDATE, DELETE ON MYSCHEMA.CUSTOMERS BY ACCESS; +AUDIT SELECT, INSERT, UPDATE, DELETE ON MYSCHEMA.ORDERS BY ACCESS; +``` + +#### Option 4: Audit session events (Login/Logout) + +To capture login and logout events: + +```sql +-- Audit all session connections +AUDIT SESSION BY ACCESS; + +-- Audit only failed login attempts +AUDIT SESSION WHENEVER NOT SUCCESSFUL; +``` + +#### Option 5: Audit DDL statements + +To capture Data Definition Language (DDL) operations: + +```sql +-- Audit all DDL statements +AUDIT TABLE BY ACCESS; +AUDIT VIEW BY ACCESS; +AUDIT PROCEDURE BY ACCESS; +``` + +#### To disable auditing + +To stop auditing specific operations: + +```sql +-- Disable global auditing +NOAUDIT SELECT ANY TABLE; +NOAUDIT INSERT ANY TABLE; +NOAUDIT UPDATE ANY TABLE; +NOAUDIT DELETE ANY TABLE; + +-- Stop auditing procedure executions +NOAUDIT EXECUTE ANY PROCEDURE; + +-- Disable auditing on specific tables +NOAUDIT SELECT, INSERT, UPDATE, DELETE ON MYSCHEMA.MYTABLE; + +-- Disable session auditing +NOAUDIT SESSION; +``` + +#### To check current audit settings + +To view which audit options are currently enabled: + +```sql +-- View global statement audit options +SELECT * FROM DBA_STMT_AUDIT_OPTS; + +-- View object-specific audit options +SELECT * FROM DBA_OBJ_AUDIT_OPTS; +``` + +#### To view audit trail records + +To query the audit trail (note: these records are also exported to CloudWatch): + +```sql +-- View recent audit records +SELECT EXTENDED_TIMESTAMP, DB_USER, ACTION, OBJECT_NAME, SQL_TEXT +FROM V$XML_AUDIT_TRAIL +WHERE DB_USER NOT IN ('SYS', 'SYSTEM', 'RDSADMIN') + AND DB_USER != '/' -- Exclude internal user + AND DB_USER IS NOT NULL +ORDER BY EXTENDED_TIMESTAMP DESC +``` + +## Limitations + +1. **Data Ingestion Delay**: There will be a delay in data being observed for reports due to limitations of the Oracle RDS DB instance and CloudWatch log availability. + +2. **Filtered System Users and Operations**: To avoid unnecessary logging and reduce noise, the connector automatically filters out audit records from the following system users and operations: + - **System Users**: Records where `Object_Schema`, `Current_User`, or `DB_User` is `SYS`, `AUDSYS`, or `RDSADMIN` + - **DBMS_OUTPUT Operations**: Any SQL statements containing `DBMS_OUTPUT` calls + - **Empty or Invalid SQL**: Records with empty SQL text or containing only `/` + - **Records without Content**: Records missing both `Comment_Text` and `Sql_Text` + + These filters help focus on application-level database activities and reduce the volume of system-generated audit logs. + +## Creating datasource profiles + +You can create a new datasource profile from the **Datasource Profile Management** page. + +### Procedure + +1. Go to **Manage > Universal Connector > Datasource Profile Management** +2. Click the **➕ (Add)** button. +3. You can create a profile by using one of the following methods: + + - To **Create a new profile manually**, go to the **"Add Profile"** tab and provide values for the following fields. + - **Name** and **Description**. + - Select a **Plug-in Type** from the dropdown. For example, `Amazon RDS Oracle Over Cloudwatch Connect 2.0`. + + - To **Upload from CSV**, go to the **"Upload from CSV"** tab and upload an exported or manually created CSV file containing one or more profiles. + You can also choose from the following options: + - **Update existing profiles on name match** — Updates profiles with the same name if they already exist. + - **Test connection for imported profiles** — Automatically tests connections after profiles are created. + - **Use ELB** — Enables ELB support for imported profiles. You must provide the number of MUs to be used in the ELB process. + +**Note:** Configuration options vary based on the selected plug-in. + +## Configuring Oracle RDS Over CloudWatch Kafka Connect 2.0 + +The following table describes the fields that are specific to Oracle RDS over CloudWatch Kafka Connect 2.0 plugin. + +| Field | Description | +|--------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Name** | Unique name of the profile. | +| **Description** | Description of the profile. | +| **Plug-in** | Plug-in type for this profile. Select `Amazon RDS Oracle Over Cloudwatch Connect 2.0`. A full list of available plug-ins are available on the **Package Management** page. | +| **Credential** | Select AWS Credentials or AWS Role ARN. The credential to authenticate with AWS. Must be created in **Credential Management**, or click **➕** to create one. For more information, see [Creating Credentials](https://www.ibm.com/docs/en/SSMPHH_12.x/com.ibm.guardium.doc.stap/guc/guc_credential_management.html). | +| **Kafka Cluster** | Select the appropriate Kafka cluster from the available Kafka cluster list or create a new Kafka cluster. For more information, see [Managing Kafka clusters](https://www.ibm.com/docs/en/SSMPHH_12.x/com.ibm.guardium.doc.stap/guc/guc_kafka_cluster_management.html). | +| **Label** | Grouping label. For example, customer name or ID. | +| **AWS account region** | Specifies the AWS region where your RDS Oracle instance is located (e.g., us-east-1, eu-west-1). | +| **Log groups** | List of CloudWatch log groups to monitor. These are the log groups where Oracle audit logs are exported. | +| **Filter pattern** | CloudWatch Logs filter pattern to apply. Use "None" to retrieve all logs, or specify a pattern to filter specific log events. | +| **Account ID** | Your AWS account ID (12-digit number). This identifies your AWS account. | +| **Cluster name** | The name of your RDS Oracle cluster or instance identifier. | +| **Ingestion delay (seconds)** | Default value is 900 seconds (15 minutes). This delay accounts for the time it takes for logs to be available in CloudWatch after being generated. | +| **No-traffic threshold (minutes)** | Default value is 60. If there is no incoming traffic for an hour, S-TAP displays a red status. Once incoming traffic resumes, the status returns to green. | +| **Unmask sensitive value** | Optional boolean flag. When enabled, sensitive values in the audit logs will not be masked. | +| **Use Enterprise Load Balancing (ELB)** | Enable this if ELB support is required. | +| **Managed Unit Count** | Number of Managed Units (MUs) to allocate for ELB. | + +**Note:** +- Ensure that the **profile name** is unique. +- Required credentials must be created before or during profile creation. +- The AWS credentials must have appropriate permissions to read CloudWatch logs. + +--- + +## Testing a Connection + +After creating a profile, you must test the connection to ensure the provided configuration is valid. + +### Procedure + +1. Select the new profile. +2. From the top menu, click **Test Connection**. +3. If the test is successful, you can proceed to installing the profile. + +--- + +## Installing a Profile + +Once the connection test is successful, you can install the profile on **Managed Units (MUs)** or **Edges**. The parsed audit logs are sent to the selected Managed Unit or Edge to be consumed by the **Sniffer**. + +### Procedure + +1. Select the profile. +2. From the **Install** menu, click **Install**. +3. From the list of available MUs and Edges that is displayed, select the ones that you want to deploy the profile to. + +--- + +## Uninstalling or reinstalling profiles + +An installed profile can be uninstalled or reinstalled if needed. + +### Procedure + +1. Select the profile. +2. From the list of available actions, select the desired option: **Uninstall** or **Reinstall**. + +--- diff --git a/docs/KafkaBasedUCs/OuaOverConnectJdbcReadme.md b/docs/KafkaBasedUCs/OuaOverConnectJdbcReadme.md new file mode 100644 index 000000000..74d17c6cd --- /dev/null +++ b/docs/KafkaBasedUCs/OuaOverConnectJdbcReadme.md @@ -0,0 +1,130 @@ +# Oracle Unified Audit Universal Connector Over JDBC Connect + +## Meet Oracle Unified Audit Over JDBC Connect +* Tested versions: 19, 21 +* Environments: On-prem, RDS in AWS, Oracle Base Database Service in OCI +* Supported inputs: Kafka Input (pull) +* Supported Oracle versions: 19, 21 +* Supported Guardium versions: + * Guardium Data Protection: appliance bundle 12.1p105 or later. + +**Note**: This readme is also applicable for **OUA over JDBC connect 2.0** and **OUA Multitenant over JDBC connect 2.0** plug-ins. + +Kafka-connect is framework for streaming data between Apache Kafka and other systems. + +Detailed breakdown: +1. Kafka-connect JDBC Connector: pulls data from `UNIFIED_AUDIT_TRAIL`. +2. Kafka-connect JDBC Connector for OUA Multitenant over JDBC connect 2.0: pulls data from `CDB_UNIFIED_AUDIT_TRAIL`. +3. The data in the Kafka topic is consumed by kafka-input plugin and process by the 'guardium-oua-uc' filter plug-in, + a specific Unified Connector designed for your use case. + +**Tip**: IBM recommends creating a Kafka cluster only after your environment is patched with appliance bundle 12.0p120+ 12.0p5002 for Guardium Data Protection version 12.1, as using a Kafka cluster before appliance bundle 12.0p120 + UC 12.0p5002 may provide undesirable results and does not support disaster recovery scenarios. This ensures that profiles using the Kafka cluster are applied correctly. + +### GDP versions available with OUA over JDBC credential support +| Credential types | Patch details for availability | +|--------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **JDBC** | GDP version 12.1 + Appliance bundle patch 105 or 115 + Universal connector patch 1006 and later | +| **JDBC & Kerberos** | GDP version 12.1 + Appliance bundle patch 115 + Universal connector patch 5002 and later | +| **JDBC & Kerberos for OUA 2.0** | GDP version 12.1 + Appliance bundle patch 120 + Universal connector patch p5002 | + + + +### Requirements +1. This feature is currently supported only in environments that is using central manager workflow and a Kafka cluster, in Guardium Data Protection version 12.1. +2. Unified auditing must be enabled in an Oracle database that will be monitored by this method +3. Download Oracle JDBC driver according to the details mentioned in the **[JDBC driver library](https://github.com/IBM/universal-connectors/blob/main/filter-plugin/logstash-filter-oua-guardium/OuaOverConnectJdbcReadme.md#configuring-universal-connector-profile)** field. + +## Setup + +1. **Create a designated Database User for OUA UC to retrieve audit data with minimal privileges (using DBA help) as follows:** + - Assuming the name for the designated Oracle Unified Audit user with minimal permissions will be "guardium" with password "password" + - Connect to Oracle using sysdba account and execute the following commands: + + ``` + CREATE USER IDENTIFIED BY ; + GRANT CONNECT, RESOURCE to ; + GRANT SELECT ANY DICTIONARY TO ; + ``` + + - To verify your new user's privileges, connect to the Oracle instance that you are planning to monitor using the name and credentials for your designated user and run the following command: + + ``` + select count(*) from AUDSYS.AUD$UNIFIED; + ``` + + If there are no errors, then you can use to configure the universal connector. + + - NOTE: If the Guardium user requires only a limited set of privileges, install the UC Q3 2025 patch. Alternatively, you can download the updated packages from [OUA2-P5002](https://github.com/IBM/universal-connectors/releases/tag/OUA2-p5002) and upload to Guardium CM via package management page. Grant the very minimum subset of privileges: + + ``` + CREATE USER IDENTIFIED BY ; + GRANT CONNECT,AUDIT_VIEWER TO CONTAINER=ALL; + ``` + + - To verify your new user's privileges, connect to the Oracle instance that you are planning to monitor using the name and credentials for your designated user and run the following command: + ``` + select count(*) from UNIFIED_AUDIT_TRAIL; + ``` + + - Apply the following policy to capture changes to system parameters: + ``` + CREATE AUDIT POLICY system_param_changes ACTIONS ALTER SYSTEM; + AUDIT POLICY system_param_changes; + ``` + + 2. **To exclude auditing for a database user who executes JDBC queries and avoid self-monitoring, run the following queries** + ``` + # Connect as SYSDBA + sqlplus / AS SYSDBA + + # make sure audit_trail is set to none + SHOW PARAMETER audit_trail; + + # log to PDB + ALTER SESSION SET CONTAINER = ; + + # Create a new user with connect and dictionary privilege (for example: AUDITUSER) + + CREATE USER AUDITUSER IDENTIFIED BY ; + GRANT CONNECT, RESOURCE to AUDITUSER; + GRANT SELECT ANY DICTIONARY TO AUDITUSER; + + # Check which policy enabled + select distinct * from AUDIT_UNIFIED_ENABLED_POLICIES; + + # Remove audit policies like ALL_ACTIONS that include + NOAUDIT POLICY ALL_ACTIONS; + + # Create ALL_ACTIONS policy if not exists + + CREATE AUDIT POLICY ALL_ACTIONS ACTIONS INSERT, SELECT, UPDATE, DELETE; + SELECT * FROM AUDIT_UNIFIED_POLICIES WHERE POLICY_NAME = ' ALL_ACTIONS'; + + # Add ALL_ACTIONS policy to all user except AUDITUSER + AUDIT POLICY ALL_ACTIONS EXCEPT "AUDITUSER"; + ``` +**For further details about configuring audit policies, see [official Oracle documentation](https://docs.oracle.com/en/database/oracle/oracle-database/19/dbseg/configuring-audit-policies.html).** +## Configuring Universal Connector on Guardium Data Protection + +### Before you begin +* Configure the policies you require. See [policies](/docs/#policies) for more information. + +### Configuring Universal Connector Profile +1. To create a datasource profile, see [Creating data source profiles](https://www.ibm.com/docs/en/SSMPHH_12.x/com.ibm.guardium.doc.stap/guc/guc_datasource_profile_management.html). +2. Select '**OUA over JDBC connect**' in the plug-ins list +3. Update the parameters as follows: + +| Field | Description | +|--------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Credential** | Create JDBC credentials. For more information, see [Creating Credentials](https://www.ibm.com/docs/en/SSMPHH_12.x/com.ibm.guardium.doc.stap/guc/guc_credential_management.html). | +| **Kafka cluster** | Select the appropriate Kafka cluster from the available Kafka cluster list or create a new Kafka cluster. For more information, see [Managing Kafka clusters](https://www.ibm.com/docs/en/SSMPHH_12.x/com.ibm.guardium.doc.stap/guc/guc_kafka_cluster_management.html). | +| **No traffic threshold (minutes)** | Default value is 60. If there is no incoming traffic for an hour, S-TAP displays a red status. Once incoming traffic resumes, the status returns to green. | +| **Initial Time (ms)** | The timestamp from which the connector starts polling for changes in the database. Setting this to 0 means the connector starts from the earliest available data. For incremental data fetching, this ensures only new data (after the initial time) is retrieved. | +| **Hostname** | Specifies the hostname or IP address of the Oracle database server. It is the address where the Oracle instance can be accessed for establishing a JDBC connection. | +| **JDBC driver library** | The Oracle JDBC driver JAR file (e.g., `ojdbc8.jar`) is required for the connector to communicate with the Oracle database. Download the [Oracle JDBC driver JAR file](https://download.oracle.com/otn-pub/otn_software/jdbc/234/ojdbc8.jar) and upload it to the Kafka Connect environment.
Note: Uploading multiple OJDBC versions simultaneously is not supported. This action causes Guardium Universal Connector to fail due to classloader conflicts. | +| **Port** | Specifies the port number used to connect to the Oracle database. The default port number is 1521, but it can vary depending on the Oracle configuration. Port 1521 must be open and accessible for the connection. | +| **Service Name / SID** | Specifies the Oracle service name (or SID if it's an older configuration) for the Kafka connector to connect. The service name uniquely identifies a database service within an Oracle environment and is provided by the database administrator. For OUA over JDBC data is retrived from the service itself: unified_audit_trail . | +| **CDB Service Name / SID** | OUA over JDBC Connect 2.0 and OUA multitenant over JDBC Connect, data is retrived from CDB service audit log: cdb_unified_audit_trail. | + + +4. Continue from step 3 of [Creating data source profile topic](https://www.ibm.com/docs/en/SSMPHH_12.x/com.ibm.guardium.doc.stap/guc/guc_datasource_profile_management.html) to complete creating a datasource profile. diff --git a/docs/KafkaBasedUCs/PostgresqlCloudwatchKafkaConnect.md b/docs/KafkaBasedUCs/PostgresqlCloudwatchKafkaConnect.md new file mode 100644 index 000000000..f84b66b78 --- /dev/null +++ b/docs/KafkaBasedUCs/PostgresqlCloudwatchKafkaConnect.md @@ -0,0 +1,298 @@ +# Configuring Postgres datasource profiles for Kafka Connect Plug-ins + +Create and configure datasource profiles datasource profiles through Central Manager for **AWS +Postgres over CloudWatch Kafka Connect** plug-ins. + +### Meet Postgres over CloudWatch Connect + +* Environments: AWS +* Supported inputs: Kafka connect Cloudwatch 2.0 (pull) +* Supported Guardium versions: + * Guardium Data Protection: Appliance bundle 12.2.1 or later + +Kafka-connect is a framework for streaming data between Apache Kafka and other systems. This connector enables +monitoring of Postgres audit logs through CloudWatch. + +## Optional: Configuring native logging + +Enable encryption on the database instances. In **Additional configuration** > **Log exports**, select the +Postgresql log type to publish to Amazon CloudWatch. + +## Enabling the PGAudit extension + +There are different ways of auditing and logging in Postgres. This procedure used PGAudit, the open +source audit logging extension for PostgreSQL 9.5+. This extension supports logging for Sessions or Objects. + +**Note:** Configure either **Session Auditing** or **Object Auditing**. You cannot enable both at the same time. + +### 1. Creating a database parameter group. + +When you create a database instance, it is associated with the default parameter group. To create a new database parameter group, complete the following steps. + +1. Go to **Services** > **Database** > **Parameter groups** +2. From the left panel, click **Create Parameter Group**. +3. Enter the parameter group details.
+ a. Select the parameter group family. For example, **postgres12**. This version should match the version of the database you created and with which this parameter group will be associated.
+ b. Enter the **DB parameter group name**.
+ c. Enter the **DB parameter group description**.
+4. Click **Save**. The new group appears in the **Parameter Groups** section. + +### 2a. Enabling PGAudit Session auditing + +Session Auditing allows you to log activities that are selected in the **pgaudit.log** parameter for logging. Be cautious when selecting which activities to log, as logged activities can affect database instance performance. + +1. From the Amazon RDS left panel, select **Parameter Groups**. +2. Select the parameter group you created. +3. Click **Edit parameters** and add the following settings.
+ a. **pgaudit.log** = ``all, -misc`` (Select options from the **Allowed values** list. You can specify multiple values separated by commas. Values that are marked with "**-**" are excluded from logging.)
+ b. **pgaudit.log_catalog** = ``0``
+ c. **pgaudit.log_parameter** = ``0``
+ d. **shared_preload_libraries** = ``pgaudit``
+ e. **log_error_verbosity** = ``default``
+ +### 2b. Enabling PGAudit Object Auditing + +Object auditing affects performance less than session auditing due to the fine-grained criteria of tables and columns that you can select for auditing. + +1. Set the following parameters.
+ a. **pgaudit.log** = ``none`` (since this is not needed for extensive SESSION logging)
+ b. **pgaudit.role** = ``rds_pgaudit``
+ c. **pgaudit.log_catalog** = ``0``
+ d. **pgaudit.log_parameter** = ``0``
+ e. **shared_preload_libraries** = ``pgaudit``
+ f. **log_error_verbosity** = ``default``
+ +2. Provide the required permissions to the **rds_pgaudit** role when associating it with the table to be audited. For example, ```GRANT ALL ON TO rds_pgaudit```. + This grant enables full **SELECT**, **INSERT**, **UPDATE**, and **DELETE** logging on the relation. + +### 3. Associating the DB parameter group with the database instance + +1. Go to **Services** > **Database** > **RDS** > **Databases**. +2. Click the Aurora Postgres database instance that you want to update. +3. Click **Modify**. +4. Go to **Additional Configurations** > **Database Options** > **DB Parameter Group menu**, and select the newly-created group. +5. Click **Continue**. +6. Select the database instance in its configuration section. The state of the DB Parameter Group is pending-reboot. +7. Reboot the database instance for the changes to take effect. + + +## Viewing the PGAudit logs + +The PGAudit logs (both Session and Object logs) can be seen in log files in RDS, and also on CloudWatch. + +### Viewing the auditing details in RDS log files + +The RDS log files can be viewed, watched, and downloaded. The name of the RDS log file is modifiable and is controlled +by the **log_filename** parameter. + +1. Go to **Services** > **Database** > **RDS** > **Databases**. +2. Select the database instance. +3. Select the **Logs & Events** section. +4. The end of the Logs section lists the files that contain the auditing details. The newest file is on the last page. + +### Viewing the logs entries on CloudWatch + +By default, each database instance has an associated log group with a name in this format: `/aws/rds/instance/< +instance_name>/postgresql`. You can use this log group, or you can create a new one and associate it with the database +instance. + +1. On the AWS Console page, open the **Services** menu. +2. Enter the CloudWatch string in the search box. +3. Click **CloudWatch** to redirect to the CloudWatch dashboard. +4. In the left panel, select **Logs**. +5. Click **Log Groups**. + +## Exporting CloudWatch logs to SQS Using Lambda Function (Optional) + +To achieve load balancing of audit logs between different collectors, the audit logs can be exported from CloudWatch to SQS. + +### Creating the SQS Queue + +1. Go to https://console.aws.amazon.com/. +2. Click **Services**. +3. Search for SQS and click on **Simple Queue Services**. +4. Click **Create Queue**. +5. Select the type as **Standard**. +6. Enter the name for the queue. +7. Keep the rest of the default settings. + +### Creating policies for the relevant IAM User + +1. For the IAM User using which the SQS logs are to be accessed in Guardium, complete the following steps. +2. Go to https://console.aws.amazon.com/. +3. Go to **IAM service** > **Policies** > **Create Policy**. +4. Select **service as SQS**. +5. Select the following checkboxes: **ListQueues**, **DeleteMessage**, **DeleteMessageBatch**, **GetQueueAttributes**, **GetQueueUrl**, **ReceiveMessage**, **ChangeMessageVisibility**, **ChangeMessageVisibilityBatch**. +6. In the resources, specify the ARN of the queue created in the previous step. +7. Click **Review policy** and specify the policy name. +8. Click **Create policy**. +9. Assign the policy to the user.
+ a. Log in to the IAM console as IAM user (https://console.aws.amazon.com/iam/).
+ b. Go to **Users** on the console and select the relevant IAM user to whom you want to give permissions.
+ c. In the **Permissions** tab, click **Add permissions**.
+ d. Click **Attach existing policies directly**.
+ e. Search for the policy created and check the checkbox next to it.
+ f. Click **Next: Review** > **Add permissions**.
+ + +## Creating the Lambda Function + +### Create IAM Role + +Create the IAM role that will be used in the Lambda function setup. The AWS Lambda service requires permission to log events and write to the SQS created. Create the IAM Role **Export-DynamoDB-CloudWatch-to-SQS-Lambda** with **AmazonSQSFullAccess**, **CloudWatchLogsFullAccess**, and **CloudWatchEventsFullAccess** policies. + +1. Go to https://console.aws.amazon.com/. +2. Go to **IAM** > **Roles** > **Create Role**. +4. Under **Use case**, select **Lambda** and click **Next**. +5. Search for ``AmazonSQSFullAccess`` and select it. +6. Search for ``CloudWatchLogsFullAccess`` and select it. +7. Search for ``CloudWatchEventsFullAccess`` and select it. +8. Set the **Role Name**. For example, **Export-RDS-CloudWatch-to-SQS-Lambda**. Then click **Create role**. + +### Creating the Lambda Function + +1. Go to https://console.aws.amazon.com/. +2. Go to **Services**. Search for ``Lambda function``. +3. Click **Functions** > **Create Function** +5. Keep **Author from Scratch** selected. +6. Set **Function name**. For example, **Export-RDS-CloudWatch-Logs-To-SQS**. +7. Under **Runtime**, select **Python 3.x**. +8. Under **Permissions**, select **Use an existing role** and select the IAM role created in the previous step (Export-RDS-CloudWatch-Logs-To-SQS). +9. Click **Create function** and navigate to **Code view**. +10. Add the function code from [lambdaFunction](./PostgresOverSQSPackage/postgresLambda.py). +11. Click **Configuration** > **Environment Variables**. +12. Create the following two variables. + - Key = `GROUP_NAME`, value = `` e.g., `/aws/rds/instance/mariadbsqs/audit` + - Key = `QUEUE_NAME`, value = `` e.g., `https://sqs.us-east-1.amazonaws.com/1111111111/mariadb` +13. Save the function. +14. Click **Deploy**. + +### Automating the lambda function + +**Note**: AWS has migrated CloudWatch Events to Amazon EventBridge. Use the EventBridge service to create scheduling +rules for Lambda functions. + +1. Go to the AWS Console and search for ``EventBridge``. +2. To open the EventBridge dashboard, click **Amazon EventBridge**. +3. In the left navigation pane, click **Rules** under **Events**. +4. Click **Create rule**, and enter the rule details.
+ a. **Name**: Enter a name for the rule. For example, `cloudwatchToSqs`.
+ b. **Description**: (Optional) Add a description.
+ c. **Event bus**: Select **default**.
+5. In the **Rule type** field, select **Schedule** and click **Next**. +6. Define the schedule pattern.
+ a. Select **A schedule that runs at a regular rate, such as every 10 minutes**.
+ b. Enter the rate expression (e.g., ``2`` minutes). This value must match the time specified in the lambda function + code that calculates the time delta. If the function code is set to 2 minutes, set the rate to 2 minutes unless it is + changed in the code.
+ c. Click **Next**. +7. Select the target.
+ a. In the **Target types** field, select **AWS service**.
+ b. In the **Select a target** field, select **Lambda function**.
+ c. In the **Function** field, select the Lambda function that you created in the previous step. For example, * + *Export-RDS-CloudWatch-Logs-To-SQS**.
+ d. Click **Next**.
+8. (Optional) Add tags if needed, then click **Next**. +9. Review the rule configuration and click **Create rule**. + +**Note:** Before making any changes to the Lambda function code, you must disable this rule. Once you deploy the change, +you can re-enable the rule. + +## Limitations + +1. The postgres plug-in does not support IPV6 +2. When Postgres UC is configured to be used with SQS, the multiline characters in the query are not preserved in FullSql Reports +3. When Postgres UC is configured to be used with SQS, the queries containing single line comments will not be supported +4. PGAudit logs the batch queries multiple times, so the report will show multiple entries for the same item +5. Client HostName and OS User fields couldn't be mapped with the logs, so set as empty. + +## Creating datasource profiles + +You can create a new datasource profile from the **Datasource Profile Management** page. + +### Procedure + +1. Go to **Manage > Universal Connector > Datasource Profile Management** +2. Click the **➕ (Add)** button. +3. You can create a profile by using one of the following methods: + + - To **Create a new profile manually**, go to the **"Add Profile"** tab and provide values for the following fields. + - **Name** and **Description**. + - Select a **Plug-in Type** from the dropdown. For example, `AWS Postgres Over Cloudwatch Connect 2.0`. + + - To **Upload from CSV**, go to the **"Upload from CSV"** tab and upload an exported or manually created CSV file + containing one or more profiles. + You can also choose from the following options: + - **Update existing profiles on name match** — Updates profiles with the same name if they already exist. + - **Test connection for imported profiles** — Automatically tests connections after profiles are created. + - **Use ELB** — Enables ELB support for imported profiles. You must provide the number of MUs to be used in the + ELB process. + +**Note:** Configuration options vary based on the selected plug-in. + +## Configuring Postgres Over CloudWatch Kafka Connect 2.0 + +The following table describes the fields that are specific to Postgres over CloudWatch Kafka Connect 2.0 plugin. + +| Field | Description | +|-----------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Name** | Unique name of the profile. | +| **Description** | Description of the profile. | +| **Plug-in** | Plug-in type for this profile. Select `AWS Postgres Over Cloudwatch Connect 2.0`. A full list of available plug-ins are available on the **Package Management** page. | +| **Credential** | Select AWS Credentials or AWS Role ARN. The credential to authenticate with AWS. Must be created in **Credential Management**, or click **➕** to create one. For more information, see [Creating Credentials](https://www.ibm.com/docs/en/SSMPHH_12.x/com.ibm.guardium.doc.stap/guc/guc_credential_management.html). | +| **Kafka Cluster** | Select the appropriate Kafka cluster from the available Kafka cluster list or create a new Kafka cluster. For more information, see [Managing Kafka clusters](https://www.ibm.com/docs/en/SSMPHH_12.x/com.ibm.guardium.doc.stap/guc/guc_kafka_cluster_management.html). | +| **Label** | Grouping label. For example, customer name or ID. | +| **AWS account region** | Specifies the AWS region where your Postgres instance is located (e.g., us-east-1, eu-west-1). | +| **Log groups** | List of CloudWatch log groups to monitor. These are the log groups where Postgres audit logs are exported. | +| **Filter pattern** | CloudWatch Logs filter pattern to apply. Use "None" to retrieve all logs, or specify a pattern to filter specific log events. | +| **Account ID** | Your AWS account ID (12-digit number). This identifies your AWS account. | +| **Cluster name** | The name of your Postgres cluster or instance identifier. | +| **Ingestion delay (seconds)** | Default value is 900 seconds (15 minutes). This delay accounts for the time it takes for logs to be available in CloudWatch after being generated. | +| **No-traffic threshold (minutes)** | Default value is 60. If there is no incoming traffic for an hour, S-TAP displays a red status. Once incoming traffic resumes, the status returns to green. | +| **Unmask sensitive value** | Optional boolean flag. When enabled, sensitive values in the audit logs will not be masked. | +| **Use Enterprise Load Balancing (ELB)** | Enable this if ELB support is required. | +| **Managed Unit Count** | Number of Managed Units (MUs) to allocate for ELB. | + +**Note:** + +- Ensure that the **profile name** is unique. +- Required credentials must be created before or during profile creation. +- The AWS credentials must have appropriate permissions to read CloudWatch logs. + +--- + +## Testing a Connection + +After creating a profile, you must test the connection to ensure the provided configuration is valid. + +### Procedure + +1. Select the new profile. +2. From the top menu, click **Test Connection**. +3. If the test is successful, you can proceed to installing the profile. + +--- + +## Installing a Profile + +Once the connection test is successful, you can install the profile on **Managed Units (MUs)** or **Edges**. The parsed +audit logs are sent to the selected Managed Unit or Edge to be consumed by the **Sniffer**. + +### Procedure + +1. Select the profile. +2. From the **Install** menu, click **Install**. +3. From the list of available MUs and Edges that is displayed, select the ones that you want to deploy the profile to. + +--- + +## Uninstalling or reinstalling profiles + +An installed profile can be uninstalled or reinstalled if needed. + +### Procedure + +1. Select the profile. +2. From the list of available actions, select the desired option: **Uninstall** or **Reinstall**. + +--- diff --git a/docs/KafkaBasedUCs/PostgresqlPubsubKafkaConnect.md b/docs/KafkaBasedUCs/PostgresqlPubsubKafkaConnect.md new file mode 100644 index 000000000..187cecd38 --- /dev/null +++ b/docs/KafkaBasedUCs/PostgresqlPubsubKafkaConnect.md @@ -0,0 +1,210 @@ +# PostgreSQL over Pub/Sub Source Connector + +This connector ingests PostgreSQL database audit logs from Google Cloud Pub/Sub into IBM Guardium GDP using Kafka Connect. + +## Meet PostgreSQL over Pub/Sub Connect + +* Environments: On-prem +* Supported inputs: Kafka connect Pubsub 2.0 (pull) +* Supported Guardium versions: + * Guardium Data Protection: Appliance bundle 12.2.2 or later + +Kafka-connect is a framework for streaming data between Apache Kafka and other systems. + +## Configuring the PostgreSQL on GCP + +1. Review the [prerequisites](https://cloud.google.com/sql/docs/postgres/create-instance#before_you_begin). +2. [Create a PostgreSQL instance](https://cloud.google.com/sql/docs/postgres/create-instance#create-2nd-gen). +3. Set and save the password for the postgres admin user. +4. Enable the message ordering option. + +## Configuring logging + +1. Log in to the instance using `gcloud` in **SQL Instance > Connect using Cloud Shell**. Use the password for the postgres admin user that is defined during instance creation. +2. In SQL Instances, select and edit the instance to add the following flags: + - **cloudsql.enable_pgaudit**: on + - **pgaudit.log**: all +3. In the Cloud Shell terminal, run the following command.
+ `CREATE EXTENSION pgaudit;` + +### Configuring GCP for the input plug-in +1. [Create a topic in Pub/Sub](https://cloud.google.com/pubsub/docs/create-topic#create_a_topic_2). +2. [Create a subscription in Pub/Sub](https://cloud.google.com/pubsub/docs/create-subscription#create_a_pull_subscription) +3. [Create a service account](https://developers.google.com/workspace/guides/create-credentials#create_a_service_account). + - To provide subscription access to the service account, select the **Pub/Sub Subscriber** role from the role selection list when you are creating the service account. + - You do not need to grant users access to this service account. +4. [Create credentials for a service account](https://developers.google.com/workspace/guides/create-credentials#create_credentials_for_a_service_account). The key is used by the Logstash input plugin configuration file. +5. [Create a log sink in Pub/Sub](https://cloud.google.com/logging/docs/export/configure_export_v2#creating_sink). + - To specify which logs to route, use the following inclusion filter in the **Choose logs to include in sink** field during log sink creation. This filter captures relevant data access and activity logs. + + (resource.type="cloudsql_database" resource.labels.database_id=":" + logName=("projects//logs/cloudaudit.googleapis.com%2Fdata_access") + protoPayload.request.@type="type.googleapis.com/google.cloud.sql.audit.v1.PgAuditEntry") + OR + (resource.type="cloudsql_database" resource.labels.database_id=":" + logName="projects//logs/cloudsql.googleapis.com%2Fpostgres.log" + severity=(EMERGENCY OR ALERT OR CRITICAL OR ERROR OR WARNING OR NOTICE OR DEBUG OR DEFAULT)) + + +## Enabling audit logs + +The inclusion filter that is used during log sink creation makes sures that only relevant logs are routed. + +### Viewing or downloading logs + +To view or download the generated logs, make sure that the appropriate Identity and Access Management (IAM) roles are assigned. +These roles control access to logs in GCP. + +* **View logs**: + - roles/logging.viewer (Logs Viewer) + - roles/logging.privateLogViewer (Private Logs Viewer) +* **Download logs**: + - roles/logging.admin (Logging Admin) + - roles/logging.viewAccessor (Logs View Accessor) + +For more information on IAM roles and access control, see [Access Control with IAM](https://cloud.google.com/logging/docs/access-control). + +### Setting destination permissions +To route audit logs to a specific destination, such as Pub/Sub topic and subscription, complete the following steps: + +1. [Get sink writer's identity](https://cloud.google.com/logging/docs/export/configure_export_v2#dest-auth). +2. If you have owner access to the destination, [set access controls](https://cloud.google.com/pubsub/docs/access-control#console). Copy the sink writer's identity and enter it in the **New Principals** field when you configure access policies for topics and subscriptions.
+ * For **topics**, assign the **Pub/Sub Publisher** and **Pub/Sub Subscriber** role.
+ * For **subscriptions**, assign the **Pub/Sub Publisher** role. + +### Supported audit logs +1. PgAudit - `INFO` logs +2. postgres.log - `EMERGENCY`, `ALERT`, `CRITICAL`, `ERROR`, `WARNING`, `NOTICE`, `DEBUG`, `DEFAULT` logs + +**Note:** +- The **serverHostName** field is populated with the name of the PostgreSQL instance connection. +- The **serviceName** field is populated with Cloud SQL Service. For more information, see [Cloud SQL for PostgreSQL](https://cloud.google.com/sql/docs/postgres). + +## Limitations + +The following fields are not displayed because the information is not included in the error messages from Google Cloud: +- **SQL string that caused the exception** - This column in the report is blank. +- **sourceProgram** - This field is blank. Queries can be run either from Cloud Shell using `gcloud` or from a locally installed PostgreSQL server. +- **clientIP** and **serverIP** - These fields are populated with `0.0.0.0`. + +## Creating datasource profiles + +You can create a new datasource profile from the **Datasource Profile Management** page. + +### Procedure + +1. Go to **Manage > Universal Connector > Datasource Profile Management** +2. Click the **➕ (Add)** button. +3. You can create a profile by using one of the following methods: + + * To create a new profile manually, go to the **"Add Profile"** tab and provide values for the following fields. + * **Name** and **Description**. + * Select a **Plug-in Type** from the dropdown. For example, **PostgreSQL Over PubSub Kafka Connect 2.0**. + + * To upload from CSV, go to the **Upload from CSV** tab and upload an exported or manually created CSV file + containing one or more profiles. You can also choose from the following options: + * **Update existing profiles on name match** — Updates profiles with the same name if they already exist. + * **Test connection for imported profiles** — Automatically tests connections after profiles are created. + * **Use ELB** — Enables ELB support for imported profiles. You must provide the number of MUs to be used in the + ELB process. + +**Note:** Configuration options vary based on the selected plug-in. + +## Configuration: PubSub Kafka Connect 2.0-based Plugins + +The following table describes the fields that are specific to Pub/Sub Kafka Connect 2.0 and similar plugins. + +| Field | Description | +|--------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------| +| **Name** | Unique name of the profile. | +| **Description** | Description of the profile. | +| **Plug-in** | Plug-in type for this profile. A full list of available plug-ins is available on the **Package Management** page. | +| **Credential** | The credential to authenticate with the datasource. Create the credential in **Credential Management**, or click **➕** to create a new one. | +| **Kafka Cluster** | Kafka cluster to deploy the universal connector. | +| **Label** | Grouping label (e.g, **customer name** or **ID**). | +| **GCP project ID** | Google Cloud project ID that contains the Pub/Sub subscription. | +| **Pub/Sub Subscription ID** | Pub/Sub subscription ID from which messages are consumed. | +| **GCP Topic** | Pub/Sub topic name. | +| **Maximum poll records** | The maximum number of records returned in a single poll. | +| **Expected events per second** | Expected events per second. This value is used to automatically calculate the **parallel.pull.count** parameter when it is not set. Calculation formula: ceil(expected.eps / 1000). | +| **Number of parallel pull streams** | Number of parallel pull streams to use. If not specified, this value is automatically calculated based on **expected.eps** (1 subscriber per 1000 EPS). | +| **No traffic threshold (minutes)** | The time period after which the system detects inactivity. | + +## Testing a Connection + +After creating a profile, you must test the connection to ensure the provided configuration is valid. + +### Procedure + +1. Select the new profile. +2. From the top menu, click **Test Connection**. +3. If the test is successful, you can proceed to installing the profile. + +--- + +## Installing a Profile + +Once the connection test is successful, you can install the profile on **Managed Units (MUs)** or **Edges**. The parsed +audit logs are sent to the selected Managed Unit or Edge to be consumed by the Sniffer. + +### Procedure + +1. Select the profile. +2. From the **Install** menu, click **Install**. +3. From the list of available MUs and Edges that is displayed, select the ones that you want to deploy the profile to. + +--- + +## Uninstalling or reinstalling profiles + +An installed profile can be uninstalled or reinstalled if needed. + +### Procedure + +1. Select the profile. +2. From the list of available actions, select the desired option: **Uninstall** or **Reinstall**. + +--- + +## Configuring GCP for production + +You must configure **exactly-once** delivery on your Pub/Sub subscription to prevent duplicate audit log entries at the Pub/Sub level before the messages reach the connector. + +### Procedure + +1. To enable **exactly-once** delivery on your Pub/Sub subscription, use only one of the following commands based on your scenario. + * For an existing subscription: + ```bash + gcloud pubsub subscriptions update \ + --enable-exactly-once-delivery + ``` + * For a new subscription: + ```bash + gcloud pubsub subscriptions create \ + --topic= \ + --enable-exactly-once-delivery \ + --ack-deadline=600 + ``` + +### Troubleshooting + +#### Messages are not being processed + +1. Verify that all Kafka worker settings are added. +2. Make sure the worker is restarted after configuration changes. +3. Restart the connector after the worker restart. + +#### "Quota exceeded" errors in GCP + +1. Check the current quota usage in the GCP console. +2. Request a quota increase. +3. Temporarily reduce the number of tasks until the quota is increased. + +#### High number of unacknowledged messages in Pub/Sub + +1. Verify that the connector is running by using the following command.
+ ``` + curl http://localhost:8083/connectors//status + ``` +2. Check for errors in the logs. +3. Verify that exactly-once delivery is enabled on the subscription. diff --git a/docs/KafkaBasedUCs/RDSMySqlCloudwatchKafkaConnect.md b/docs/KafkaBasedUCs/RDSMySqlCloudwatchKafkaConnect.md new file mode 100644 index 000000000..f3a3f6635 --- /dev/null +++ b/docs/KafkaBasedUCs/RDSMySqlCloudwatchKafkaConnect.md @@ -0,0 +1,181 @@ +# Configuring RDS MySql datasource profiles for Kafka Connect Plug-ins + +Create and configure datasource profiles through Central Manager for **RDS MySql +over CloudWatch Kafka Connect** plug-ins. + +### Meet RDS MySql over CloudWatch Connect + +* Environments: AWS +* Supported inputs: Kafka connect Cloudwatch 2.0 (pull) +* Supported Guardium versions: + * Guardium Data Protection: Appliance bundle 12.2.2 or later + +Kafka-connect is a framework for streaming data between Apache Kafka and other systems. This connector enables +monitoring of RDS MySQL audit logs through CloudWatch. + +***Note:*** +* The client source program is not available in messages sent by MySQL. This data is sent only in the first audit log message upon database connection and the filter plug-in doesn't aggregate data from different messages. +* On the reports page, the `use` statement does not display the account ID in the **Database Name** column. + +## Creating and configuring a MySQL database instance + +### Creating a MySQL database instance + +To create a new MySQL instance, see [Create and Connect to a MySQL Database with Amazon RDS](https://aws.amazon.com/getting-started/hands-on/create-mysql-db/). When setting the properties under **Additional Configuration**, in the Log exports section select **Audit log** and **Error log** as the log types to publish to Amazon. + + +### Creating a new parameter group + +1. Open the Amazon RDS console (https://console.aws.amazon.com/rds). +2. In the navigation pane, choose **Parameter groups**. +3. Choose **Create parameter group** to open the Create parameter group dialog box. +4. In the **Parameter group family** list, choose your engine version. +5. In the **Group name** box, enter the name of the new DB parameter group. +6. In the **Description** box, enter a description for the new DB parameter group. +7. Click **Create**. +8. Go back to **Parameter groups** from the navigation pane. +9. In the **Parameter groups list**, choose the parameter group that you just created. +10. Click **Parameter group actions** > **Edit**. +11. Use the **Filter parameters** field to search for the **log_output** parameter. +12. Set the value of the **log_output** parameter to `FILE`. +13. Click **Save changes**. + +## Enabling Auditing + +1. Click **Parameter Groups** > **Create Parameter Groups**. +2. Provide the following details: + • Parameter group family: Provide aurora-mysql version + • Type: DB cluster parameter group + • Group name: Name of Group + • Description: Privide description +3. Click **Create**. +4. Select **DB Parameter** > **Parameter group actions** > **Edit**. +5. Update the value of the parameters and add the following settings: + • **server_audit_events** = `CONNECT,QUERY_DCL,QUERY_DDL,QUERY_DML` + • **server_audit_excl_users** = `rdsadmin` + • **server_audit_logging** = `1` + • **server_audit_logs_upload** = `1` + • **log_output** = `FILE` +6. Click **Save changes**. +7. Go to **Database Clustor** > **Modify** > **Additional Configuration** > **Database options**. +8. Change the DB clustor parameter group. +8. Click **Continue** > **Apply immediately**. +10. Click **Modify Cluster**. +11. Reboot the DB Cluster for the changes to take effect. + +## Viewing the Audit logs + +The audit logs can be seen in log files in RDS, and also on CloudWatch. + + ### Viewing the auditing details in RDS log files + +The RDS log files can be viewed, watched, and downloaded. The name of the RDS log file is modifiable and is controlled +by the **log_filename** parameter. + +1. Go to **Services** > **Database** > **RDS** > **Databases**. +2. Select the database instance. +3. Select the **Logs & Events** section. +4. The end of the Logs section lists the files that contain the auditing details. The newest file is on the last page. + +### Viewing the logs entries on CloudWatch + +By default, each database instance has an associated log group. You can use this log group, or you can create a new one and associate it with the database +instance. + +1. On the AWS Console page, open the **Services** menu. +2. Enter the CloudWatch string in the search box. +3. Click **CloudWatch** to redirect to the CloudWatch dashboard. +4. In the left panel, select **Logs**. +5. Click **Log Groups**. + +## Creating datasource profiles + +You can create a new datasource profile from the **Datasource Profile Management** page. + +### Procedure + +1. Go to **Manage > Universal Connector > Datasource Profile Management** +2. Click the **➕ (Add)** button. +3. You can create a profile by using one of the following methods: + + - To **Create a new profile manually**, go to the **"Add Profile"** tab and provide values for the following fields. + - **Name** and **Description**. + - Select a **Plug-in Type** from the dropdown. For example, `AWS RDS MySql Over Cloudwatch Connect 2.0`. + + - To **Upload from CSV**, go to the **"Upload from CSV"** tab and upload an exported or manually created CSV file + containing one or more profiles. + You can also choose from the following options: + - **Update existing profiles on name match** — Updates profiles with the same name if they already exist. + - **Test connection for imported profiles** — Automatically tests connections after profiles are created. + - **Use ELB** — Enables ELB support for imported profiles. You must provide the number of MUs to be used in the + ELB process. + +**Note:** Configuration options vary based on the selected plug-in. + +## Configuring AWS RDS MySql Over CloudWatch Kafka Connect 2.0 + +The following table describes the fields that are specific to RDS MySQL over CloudWatch Kafka Connect 2.0 plugin. + +| Field | Description | +|-----------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Name** | Unique name of the profile. | +| **Description** | Description of the profile. | +| **Plug-in** | Plug-in type for this profile. Select `AWS RDS MySql Over Cloudwatch Connect 2.0`. A full list of available plug-ins are available on the **Package Management** page. | +| **Credential** | Select AWS Credentials or AWS Role ARN. The credential to authenticate with AWS. Must be created in **Credential Management**, or click **➕** to create one. For more information, see [Creating Credentials](https://www.ibm.com/docs/en/SSMPHH_12.x/com.ibm.guardium.doc.stap/guc/guc_credential_management.html). | +| **Kafka Cluster** | Select the appropriate Kafka cluster from the available Kafka cluster list or create a new Kafka cluster. For more information, see [Managing Kafka clusters](https://www.ibm.com/docs/en/SSMPHH_12.x/com.ibm.guardium.doc.stap/guc/guc_kafka_cluster_management.html). | +| **Label** | Grouping label. For example, customer name or ID. | +| **AWS account region** | Specifies the AWS region where your RDS MySQL instance is located (e.g., us-east-1, eu-west-1). | +| **Log groups** | List of CloudWatch log groups to monitor. These are the log groups where RDS MySQL audit logs are exported. | +| **Filter pattern** | CloudWatch Logs filter pattern to apply. Use "None" to retrieve all logs, or specify a pattern to filter specific log events. | +| **Account ID** | Your AWS account ID (12-digit number). This identifies your AWS account. | +| **Cluster name** | The name of your RDS MySQL cluster or instance identifier. | +| **Ingestion delay (seconds)** | Default value is 900 seconds (15 minutes). This delay accounts for the time it takes for logs to be available in CloudWatch after being generated. | +| **No-traffic threshold (minutes)** | Default value is 60. If there is no incoming traffic for an hour, S-TAP displays a red status. Once incoming traffic resumes, the status returns to green. | +| **Unmask sensitive value** | Optional boolean flag. When enabled, sensitive values in the audit logs will not be masked. | +| **Use Enterprise Load Balancing (ELB)** | Enable this if ELB support is required. | +| **Managed Unit Count** | Number of Managed Units (MUs) to allocate for ELB. | + +**Note:** + +- Ensure that the **profile name** is unique. +- Required credentials must be created before or during profile creation. +- The AWS credentials must have appropriate permissions to read CloudWatch logs. + +--- + +## Testing a Connection + +After creating a profile, you must test the connection to ensure the provided configuration is valid. + +### Procedure + +1. Select the new profile. +2. From the top menu, click **Test Connection**. +3. If the test is successful, you can proceed to installing the profile. + +--- + +## Installing a Profile + +Once the connection test is successful, you can install the profile on **Managed Units (MUs)** or **Edges**. The parsed +audit logs are sent to the selected Managed Unit or Edge to be consumed by the **Sniffer**. + +### Procedure + +1. Select the profile. +2. From the **Install** menu, click **Install**. +3. From the list of available MUs and Edges that is displayed, select the ones that you want to deploy the profile to. + +--- + +## Uninstalling or reinstalling profiles + +An installed profile can be uninstalled or reinstalled if needed. + +### Procedure + +1. Select the profile. +2. From the list of available actions, select the desired option: **Uninstall** or **Reinstall**. + +--- + diff --git a/docs/KafkaBasedUCs/RedshiftCloudwatchKafkaConnect.md b/docs/KafkaBasedUCs/RedshiftCloudwatchKafkaConnect.md new file mode 100644 index 000000000..22202d503 --- /dev/null +++ b/docs/KafkaBasedUCs/RedshiftCloudwatchKafkaConnect.md @@ -0,0 +1,217 @@ +# Configuring Redshift datasource profiles for Kafka Connect Plug-ins + +Create and configure datasource profiles through Central Manager for **Redshift +over CloudWatch Kafka Connect** plug-ins. + +### Meet Redshift over CloudWatch Connect + +* Environments: AWS +* Supported inputs: Kafka connect Cloudwatch 2.0 (pull) +* Supported Guardium versions: + * Guardium Data Protection: Appliance bundle 12.2.1 or later. + +Kafka-connect is a framework for streaming data between Apache Kafka and other systems. This connector enables +monitoring of Redshift audit logs through CloudWatch. + +**Note:** While connecting a third-party tool (sqlworkbench/j), you need to add an inbound rule to the security group inside the +Redshift cluster: + +- Navigate to the Redshift Console and click on the cluster which is created and go to the **properties** tab. + +- In **Network and security settings**, select the security group inside **VPC security group**. + +- Once the Security Groups page opens, go to the **Inbound rules** section and click **Edit inbound rules**. + +- Click **Add rule** and provide the following details: + + 1. Type : Redshift + + 2. Protocol : TCP (default) + + 3. Port range: 5439 (default) + + 4. Source : Custom (0.0.0.0/0) + +- Click **Save rules**. + +## Enabling auditing for Redshift + +1. Navigate to the Redshift Console and click on the cluster that you just created and go to **Properties** tab. +2. From the **Properties** tab, click **Edit** > **Edit Audit Logging**. +3. In the **Configure enable audit logging** option, choose **Enable**. +4. Select **Create new bucket**. +5. Enter a bucket name with an S3 key prefix and then click **Save changes**. + +## Creating a parameter group + +1. Go to the **CONFIG** tab. +2. Select **Workload management** and choose the default parameter group. Editing is disabled for the default parameter + group in the **Parameters** tab. No provision is available to edit the default parameter group, so you need to create + a new parameter group. +3. Click **Create**. +4. Choose a name and description for the Parameter group and click **Create**. + +## Configuring and modifying a parameter group + +After creating a new parameter group, you can modify the parameters. + +1. Open the newly created parameter group and select the **Parameters** tab. +2. Click **Edit Parameters**. +2. In **enable_user_activity_logging**, change the value from **false** to **true**. +3. Click **Save**. + +## Add a new Parameter Group to a cluster + +### Procedure + +1. Navigate to the Redshift Console, click on the cluster that you just created, and go to the **Properties** tab. +2. Click **Edit** and select **Edit parameter group**. +3. Select the parameter group you have created and modified. +4. Click **Save changes**. + + +## Connecting to the database + +Once the cluster has been created, go to the **Properties** tab + +In the **Database configurations** section, you can see that the default database name is `dev`, the default port AWS +Redshift listens to is `5439`, and the default username is `awsuser`. + +1. Go to the **Editor** tab and click **Connect to database**. +2. Choose the created cluster name. +3. Add a database name and database user. +4. Click **Connect**. + +## Running the query + +1. Create a table with details and append the row with the created table. +2. Run a query with the run button. + +## Viewing the logs entries on S3 + +Go to the S3 buckets from the search box and find the details of the generated logs (UserActivity/Connection) as shown in the following example. + +`s3`/<`bucket`>/<`prefix/`>/`AWSLogs/`/<`Account ID`>`redshift/`/<`region/`>/``/``/``/ See the +generated UserActivity/Connection logs here. + +## Viewing the logs entries on Cloudwatch + +Go to Cloudwatch from the search box and find the details of the generated logs (UserActivity/Connection) in the following log groups: +`/aws/redshift/cluster/ds-redshift-cluster/connectionlog` +`/aws/redshift/cluster/ds-redshift-cluster/useractivitylog` + +**Note** : Logs are not captured from `/aws/redshift/cluster/ds-redshift-cluster/userlog` cloudwatch group. + +## Limitations + +1. The log files appear in the s3 bucket in hourly batches, and sometimes even later. A typical delay is 30-120 minutes. + +2. The following important fields can not be mapped with Redshift audit logs: + - Source program : Not available with User Activity/Connection logs + - Server IP : Not available with User Activity/Connection logs + - Client HostName : Not available with User Activity/Connection logs + - Client IP : Not available with User Activity logs (mentioned in AWS documentation) and in case of Connection logs, + it only appears in the Guardium quickSearch (QS) page. +3. Error Logs : UserActivity logs do not capture any error logs related with Syntax errors or Authentication errors. That's why we capture connection logs only for + Authentication Failure Logs (It appears on Guardium the Guardium quicksearch screen as "LOGIN_FAILED" only if failed + log-in is attempted). +4. Due to parser limitations, the following are the details of the changes and limitations: + - Queries having `MINUS` operator appeara with `EXCEPT` operator in Guardium. + - The keyword `TOP` is removed from the Select Queries. + - Select queries with `PIVOT/UNPIVOT` is not parsed by the Connector. +5. CREATE MATERIALIZED VIEW commands appear multiple times on the S3 bucket, the Guardium full SQL report, and sniffer + logs. +6. Any query having key constraints (primary key, foreign key, unique key) may create duplication in logs because they + are not enforced by Amazon Redshift. For more information, [AWS: Table constraints](https://docs.aws.amazon.com/redshift/latest/dg/t_Defining_constraints.html). + +## Creating datasource profiles + +You can create a new datasource profile from the **Datasource Profile Management** page. + +### Procedure + +1. Go to **Manage > Universal Connector > Datasource Profile Management** +2. Click the **➕ (Add)** button. +3. You can create a profile by using one of the following methods: + + - To **Create a new profile manually**, go to the **"Add Profile"** tab and provide values for the following fields. + - **Name** and **Description**. + - Select a **Plug-in Type** from the dropdown. For example, `Redshift Over Cloudwatch Connect 2.0`. + + - To **Upload from CSV**, go to the **"Upload from CSV"** tab and upload an exported or manually created CSV file + containing one or more profiles. + You can also choose from the following options: + - **Update existing profiles on name match** — Updates profiles with the same name if they already exist. + - **Test connection for imported profiles** — Automatically tests connections after profiles are created. + - **Use ELB** — Enables ELB support for imported profiles. You must provide the number of MUs to be used in the + ELB process. + +**Note:** Configuration options vary based on the selected plug-in. + +## Configuring Redshift Over CloudWatch Kafka Connect 2.0 + +The following table describes the fields that are specific to Redshift over CloudWatch Kafka Connect 2.0 plugin. + +| Field | Description | +|-----------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Name** | Unique name of the profile. | +| **Description** | Description of the profile. | +| **Plug-in** | Plug-in type for this profile. Select `Redshift Over Cloudwatch Connect 2.0`. A full list of available plug-ins are available on the **Package Management** page. | +| **Credential** | Select AWS Credentials or AWS Role ARN. The credential to authenticate with AWS. Must be created in **Credential Management**, or click **➕** to create one. For more information, see [Creating Credentials](https://www.ibm.com/docs/en/SSMPHH_12.x/com.ibm.guardium.doc.stap/guc/guc_credential_management.html). | +| **Kafka Cluster** | Select the appropriate Kafka cluster from the available Kafka cluster list or create a new Kafka cluster. For more information, see [Managing Kafka clusters](https://www.ibm.com/docs/en/SSMPHH_12.x/com.ibm.guardium.doc.stap/guc/guc_kafka_cluster_management.html). | +| **Label** | Grouping label. For example, customer name or ID. | +| **AWS account region** | Specifies the AWS region where your Redshift instance is located (e.g., us-east-1, eu-west-1). | +| **Log groups** | List of CloudWatch log groups to monitor. These are the log groups where Redshift audit logs are exported. | +| **Filter pattern** | CloudWatch Logs filter pattern to apply. Use "None" to retrieve all logs, or specify a pattern to filter specific log events. | +| **Account ID** | Your AWS account ID (12-digit number). This identifies your AWS account. | +| **Cluster name** | The name of your Redshift cluster or instance identifier. | +| **Ingestion delay (seconds)** | Default value is 900 seconds (15 minutes). This delay accounts for the time it takes for logs to be available in CloudWatch after being generated. | +| **No-traffic threshold (minutes)** | Default value is 60. If there is no incoming traffic for an hour, S-TAP displays a red status. Once incoming traffic resumes, the status returns to green. | +| **Unmask sensitive value** | Optional boolean flag. When enabled, sensitive values in the audit logs will not be masked. | +| **Use Enterprise Load Balancing (ELB)** | Enable this if ELB support is required. | +| **Managed Unit Count** | Number of Managed Units (MUs) to allocate for ELB. | + +**Note:** + +- Ensure that the **profile name** is unique. +- Required credentials must be created before or during profile creation. +- The AWS credentials must have appropriate permissions to read CloudWatch logs. + +--- + +## Testing a Connection + +After creating a profile, you must test the connection to ensure the provided configuration is valid. + +### Procedure + +1. Select the new profile. +2. From the top menu, click **Test Connection**. +3. If the test is successful, you can proceed to installing the profile. + +--- + +## Installing a Profile + +Once the connection test is successful, you can install the profile on **Managed Units (MUs)** or **Edges**. The parsed +audit logs are sent to the selected Managed Unit or Edge to be consumed by the **Sniffer**. + +### Procedure + +1. Select the profile. +2. From the **Install** menu, click **Install**. +3. From the list of available MUs and Edges that is displayed, select the ones that you want to deploy the profile to. + +--- + +## Uninstalling or reinstalling profiles + +An installed profile can be uninstalled or reinstalled if needed. + +### Procedure + +1. Select the profile. +2. From the list of available actions, select the desired option: **Uninstall** or **Reinstall**. + +--- + diff --git a/docs/KafkaBasedUCs/SapHanaJDBCKafkaConnect.md b/docs/KafkaBasedUCs/SapHanaJDBCKafkaConnect.md new file mode 100644 index 000000000..ca087ee0d --- /dev/null +++ b/docs/KafkaBasedUCs/SapHanaJDBCKafkaConnect.md @@ -0,0 +1,262 @@ +# Configuring SapHana data source profiles for JDBC Kafka Connect plug-ins + +Create and configure data source profiles through central manager for SAP HANA JDBC Kafka Connect plug-ins. + +## Meet SAP HANA Over JDBC Connect +* Environments: On-prem +* Supported inputs: Kafka connect JDBC 2.0 (pull) +* Supported Guardium versions: + * Guardium Data Protection: 12.1 UC patch 5007 and above + * Guardium Data Protection: Appliance bundle 12.2.1 and above + +Kafka-connect is a framework for streaming data between Apache Kafka and other systems. + +### Before you begin + +Download the [SAP HANA JDBC driver](https://tools.hana.ondemand.com/#hanatools). + +## Creating a user for audit log configuration + +Create a dedicated user with audit privileges to manage and monitor audit log configurations in SAP HANA. + +**Note**: You can also use system user to audit log configurations. + +### Procedure + +1. Create a new user by running the following command. + ```sql + CREATE USER your_username PASSWORD 'your_password'; + ``` +2. Assign the required audit privileges to the user. + ```sql + GRANT AUDIT ADMIN TO your_username; + ``` +3. Commit the changes to the database. + ```sql + COMMIT; + ``` + +Verify that all the changes are committed to the database. + +### Procedure + +1. In SAP HANA Studio, expand the system on which you want to enable auditing. +2. Expand the **Security** folder. +3. Double-click the **Security option**. +4. From the **Auditing status** drop-down menu, select **Enabled**. By default, the auditing status is disabled. +5. To save the changes, click **Deploy** or press **F8**. +6. Restart the database instance to apply the changes. + +### Enabling CSTABLE-based auditing + +Configure CSTABLE-based auditing in SAP HANA to store audit trails in database tables by using the JDBC input plug-in. + +**Note:** CSTABLE-based auditing stores audit trails in a database table and requires the JDBC input plug-in. + +### Procedure + +1. In SAP HANA Studio, select the **SYSTEMDB** user. Then, open **SQL Console**. +2. Enable auditing for a multiple container tenant database. + ``` + ALTER SYSTEM ALTER CONFIGURATION ('global.ini', 'system') + set ('auditing configuration', 'global_auditing_state') = 'true'; + ``` +3. Set the audit trail target to a table (``CSTABLE``). + ``` + ALTER SYSTEM ALTER CONFIGURATION ('global.ini', 'system') + set ('auditing configuration', 'default_audit_trail_type') = 'CSTABLE'; + ``` +4. To prevent storing unwanted system logs in the audit table, store them in CSV files using the following command + ``` + ALTER SYSTEM ALTER CONFIGURATION ('global.ini', 'system') + set ('auditing configuration', 'critical_audit_trail_type') = 'CSVTEXTFILE'; + ``` +5. Restart the container and refresh the added systems. + +## Creating an audit policy + +Here we will check about CSTABLE base auditing. +* CSTABLE base auditing - Audit-trail target is a table, requires JDBC input plug-in. + +### Enabling CSTABLE base auditing logs: + +An audit policy defines the actions to be audited. To create an audit policy, the user must have AUDIT ADMIN system +privileges. +You need to create an audit policy for both types of auditing. + +To perform the below steps, open SAP HANA studio (Eclipse) Select SYSTEMDB user, right-click, select SQL Console, +and then run these Commands: + +1. For multiple container tenant database you can enable auditing for CSV File target by using the following command + 1. SAP HANA Command For Enable Auditing: + ``` + ALTER SYSTEM ALTER CONFIGURATION ('global.ini', 'system') + set ('auditing configuration', 'global_auditing_state') = 'true'; + ``` + 2. To select the target as a table, use the following command + ``` + ALTER SYSTEM ALTER CONFIGURATION ('global.ini', 'system') + set ('auditing configuration', 'default_audit_trail_type') = 'CSTABLE'; + ``` + 3. To avoid unwanted system logs use below command to store all system logs in table. + ``` + ALTER SYSTEM ALTER CONFIGURATION ('global.ini', 'system') + set ('auditing configuration', 'critical_audit_trail_type') = 'CSVTEXTFILE'; + ``` + +2. After running the previous command, restart the container and refresh the added systems. + +## Common steps for either auditing type + +### Creating an audit policy + +An audit policy defines the actions to be audited. In order to create an audit policy, the user must have +AUDIT ADMIN system rights. Creating an audit policy is a common step for both types of auditing. + +You need to create an audit policy for both types of auditing. +#### Procedure + +1. In the SAP HANA Studio, expand the database. +2. Expand the **Security** folder. +3. Double-click the **Security option**. +4. Under the **Audit Policies** panel, click the **green plus** icon. +5. Enter your policy name. +6. In the **Audited actions** field, click **…** (ellipsis). Then select the actions to audit. +7. In the **Audited actions status** column, select when to create an audit record. + - SUCCESSFUL - Logs successfully executed actions + - UNSUCCESSFUL - Logs unsuccessfully executed actions + - ALL - Logs both successful and unsuccessful actions +8. Select the audit level. + - EMERGENCY + - CRITICAL + - ALERT + - WARNING + - INFO (default) +9. (Optional) Filter the users you would like to audit. From the **Users** column, click ***…*** (ellipsis) and add + users. +10. (Optional) Specify the target object(s) to be audited. This option is available only when auditing `SELECT`, + `INSERT`, `UPDATE`, or `DELETE` actions. +11. To save the changes, click **Deploy** or press **F8**. +12. Restart the database instance to apply the changes. + +## Configuring audit policies in SAP HANA + +Configure audit policies in SAP HANA to monitor session activities, DML operations, and DDL changes. + +1. For audit session-related logs (Connect, Disconnect, User Validation), complete the following steps.
+ a. Go to the tab **Audited action** tab. From the **Session Management** and **System Configuration** menus, select + the **Connect** checkbox.
+ b. Set **Audited Action Status** to ``Unsuccessful``.
+ +2. For audit DML logs, complete the following steps.
+ a. From the tab **Audited action** tab, select the **Data Query** and **Manipulation** checkbox.
+ b. Set **Audited Action Status** to ``Successful``.
+ c. Specify a **Target Object** for this policy. + +3. For audit DDL Logs, complete the following steps.
+ a. Go to the tab **Audited action** tab. From the **Data Definition** menu, select the **Create Table**, **Create + Function**, **Create Procedure**, **Drop Function**, **Drop Procedure**, and **Drop Table** checkboxes.
+ b. Set **Audited Action Status** to ``Successful``.
+ +## Viewing SAP HANA audit logs for CSTABLE-based auditing + +### Procedure + +1. Connect to the database. +2. Right-click on it and click **SQL console**. Then run the following command. + ```sql + select * from AUDIT_LOG; + ``` + +## Creating datasource profiles + +You can create a new datasource profile from the **Datasource Profile Management** page. + +### Procedure + +1. Go to **Manage > Universal Connector > Datasource Profile Management** +2. Click the **➕ (Add)** button. +3. You can create a profile by using one of the following methods: + + * To create a new profile manually, go to the **"Add Profile"** tab and provide values for the following fields. + * **Name** and **Description**. + * Select a **Plug-in Type** from the dropdown. For example, **SAP HANA Over JDBC Kafka Connect 2.0**. + + * To upload from CSV, go to the **Upload from CSV** tab and upload an exported or manually created CSV file + containing one or more profiles. You can also choose from the following options: + * **Update existing profiles on name match** — Updates profiles with the same name if they already exist. + * **Test connection for imported profiles** — Automatically tests connections after profiles are created. + * **Use ELB** — Enables ELB support for imported profiles. You must provide the number of MUs to be used in the + ELB process. + +**Note:** Configuration options vary based on the selected plug-in. + +--- + +## Configuring JDBC Kafka Connect 2.0-based plugins + +The following table describes the fields that are specific to JDBC Kafka Connect 2.0 and similar plugins. + +| Field | Description | +|-----------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------| +| **Name** | Unique name of the profile. | +| **Description** | Description of the profile. | +| **Plug-in** | Plug-in type for this profile. A full list of available plug-ins are available on the **Package Management** page. | +| **Credential** | The credential to authenticate with the datasource. Create the credential in **Credential Management**, or click **➕** to create a new one. | +| **Kafka Cluster** | Kafka cluster to deploy the universal connector. | +| **Label** | Grouping label (e.g, customer name or ID). | +| **JDBC Driver Library** | JDBC driver for the database. | +| **Port** | Port that is used to connect to the database. | +| **Hostname** | Hostname of the database. | +| **Query** | SQL query that is used to extract audit logs. | +| **Service Name / SID** | The database **service name** or **SID**. | +| **Initial Time** | Initial polling time for audit logs. | +| **No Traffic Threshold** | Threshold setting for inactivity detection. | +| **Use Enterprise Load Balancing (ELB)** | Enable this if ELB support is required. | +| **Managed Unit Count** | Number of Managed Units (MUs) to allocate for ELB. | + +**Note:** +- Depending on the plugin type, the configuration may require either: + - A **Connection URL**, or + - Separate fields for **Hostname**, **Port**, and **Service Name / SID** +- Make sure that the **Profile name** is unique. +- Required credentials must be created before or during profile creation. + +--- + +## Testing a Connection + +After creating a profile, you must test the connection to make sure that the provided configuration is valid. + +### Procedure + +1. Select the new profile. +2. From the top menu, click **Test Connection**. +3. If the test is successful, you can proceed to installing the profile. + +--- + +## Installing a Profile + +After the connection test succeeds, you can install the profile on **Managed Units (MUs)** or **Edges**. The parsed +audit logs are sent to the selected Managed Unit or Edge for to be consumed by the **Sniffer**. + +### Procedure + +1. Select the profile. +2. From the list of available actions, select the desired option **Uninstall** or **Reinstall**. +3. From the list of available MUs and Edges that is displayed, select the ones that you want to deploy the profile to. + +--- + +## Uninstalling or reinstalling profiles + +An installed profile can be **uninstalled** or **reinstalled** if needed. + +### Procedure + +1. Select the profile. +2. From the list of available actions, select the desired option: **Uninstall** or **Reinstall**. + +--- + diff --git a/docs/KafkaBasedUCs/SnowflakeJDBCKafkaConnect.md b/docs/KafkaBasedUCs/SnowflakeJDBCKafkaConnect.md new file mode 100644 index 000000000..4866a7e97 --- /dev/null +++ b/docs/KafkaBasedUCs/SnowflakeJDBCKafkaConnect.md @@ -0,0 +1,176 @@ +# Configuring Snowflake data source profiles for JDBC Kafka Connect plug-ins + +Create and configure data source profiles through central manager for Snowflake JDBC Kafka Connect plug-ins. + +## Meet Snowflake Over JDBC Connect +* Environments: On-prem +* Supported inputs: Kafka connect JDBC 2.0 (pull) +* Supported Guardium versions: + * Guardium Data Protection: Appliance bundle 12.2.2 or later + +Kafka-connect is a framework for streaming data between Apache Kafka and other systems. + +## Before you begin + +Download +the [Snowflake JDBC driver](https://repo1.maven.org/maven2/net/snowflake/snowflake-jdbc/3.13.30/snowflake-jdbc-3.13.30.jar). Download the jdbc driver `jar` file from the maven repository 3.13.30 from [here](https://repo1.maven.org/maven2/net/snowflake/snowflake-jdbc/3.13.30/snowflake-jdbc-3.13.30.jar), 3.16.0 from [here](https://repo1.maven.org/maven2/net/snowflake/snowflake-jdbc/3.16.0/snowflake-jdbc-3.16.0.jar) . + +## Configuring the Snowflake database + +Create the required type of [Snowflake account](https://signup.snowflake.com/). + +Upon account creation, a Snowflake database is created and the details are provided over e-mail. Snowflake also provides +the option to choose +a cloud provider to host the database. For more information, see [Snowflake documentation](https://docs.snowflake.com/). + +## Enabling audit logs + +### Providing permissions to JDBC users + +The user that is defined in the **jdbc_user** parameter must have permissions to execute SQL statements. + +To test this, replace `:sql_last_value` with an epoch time value and run the query against Snowflake by using the +specified user. Make sure that the user has access to +the [Snowflake database](https://docs.snowflake.com/en/sql-reference/account-usage.html#enabling-snowflake-database-usage-for-other-roles). + +Run the following queries to provide access to a specific role. + +```sql + use role accountadmin; + CREATE ROLE ; + CREATE USER PASSWORD = '' ; + + GRANT ROLE TO USER ; + ALTER USER SET DEFAULT_ROLE = ; + + grant imported privileges on database snowflake to role sysadmin; + grant imported privileges on database snowflake to role ; + GRANT USAGE ON WAREHOUSE COMPUTE_WH TO ROLE ; + GRANT SELECT ON ALL TABLES IN SCHEMA snowflake.account_usage TO ROLE ; +``` + +## Viewing the audit logs + +To view the audit logs, query the following tables: + +- `SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY`
+- `SNOWFLAKE.ACCOUNT_USAGE.SESSIONS`
+- `SNOWFLAKE.ACCOUNT_USAGE.LOGIN_HISTORY`
+ +## Creating datasource profiles + +You can create a new datasource profile from the **Datasource Profile Management** page. + +### Procedure + +1. Go to **Manage > Universal Connector > Datasource Profile Management** +2. Click the **➕ (Add)** button. +3. You can create a profile by using one of the following methods: + + * To create a new profile manually, go to the **"Add Profile"** tab and provide values for the following fields. + * **Name** and **Description**. + * Select a **Plug-in Type** from the dropdown. For example, **Snowflake Over JDBC Kafka Connect 2.0**. + + * To upload from CSV, go to the **Upload from CSV** tab and upload an exported or manually created CSV file + containing one or more profiles. You can also choose from the following options: + * **Update existing profiles on name match** — Updates profiles with the same name if they already exist. + * **Test connection for imported profiles** — Automatically tests connections after profiles are created. + * **Use ELB** — Enables ELB support for imported profiles. You must provide the number of MUs to be used in the + ELB process. + +**Note:** Configuration options vary based on the selected plug-in. + +## Limitations + +1. The `SNOWFLAKE.ACCOUNT_USAGE` data view contains tables that the plug-in uses for audit logs. Because the data view + refreshes periodically, there can be a delay in retrieving audit data. +2. The server portal must remain at 443, as configuration support is deprecated. For more information, + see [Connection parameters reference + ](https://docs.snowflake.com/en/user-guide/snowsql-start#p-port-deprecated). + +## Configuring the key pair authentication. + +You can authenticate to the database by using key pair authentication instead of password authentication. For more +information about setting up key pair authentication, see [KeyPairAuth_README](../../filter-plugin/logstash-filter-snowflake-guardium/KeyPairAuth_README.md). + +## (Optional) Configuring the proxy + +To connect the database through an intermediate proxy, configure the **jdbc_connection_string** parameter as shown in +the following example. + +```text +jdbc_connection_string => "jdbc:snowflake://...snowflakecomputing.com/?warehouse= +&db=&useProxy=true&proxyHost=&proxyPort=" +``` + +## Preventing authentication token expiration + +To prevent authentication token expiration, configure the [ +``CLIENT_SESSION_KEEP_ALIVE``](https://docs.snowflake.com/en/sql-reference/parameters#client-session-keep-alive) and [ +``CLIENT_SESSION_KEEP_ALIVE_HEARTBEAT_FREQUENCY``](https://docs.snowflake.com/en/sql-reference/parameters#client-session-keep-alive-heartbeat-frequency) +fields in the **jdbc_connection_string** parameter as shown in the following example. + ```text + jdbc_connection_string => jdbc:snowflake://...snowflakecomputing.com/?warehouse=&db= + &CLIENT_SESSION_KEEP_ALIVE=true&CLIENT_SESSION_KEEP_ALIVE_HEARTBEAT_FREQUENCY=60 + ``` + +## Configuring JDBC Kafka Connect 2.0-based plugins + +Below is a description of the fields specific to JDBC Kafka Connect 2.0 and similar plugins: + +| Field | Description | Value/Example | +|-----------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Name** | Unique name of the profile | SNOWFLAKE_JDBC_KAFKA_CONNECT | +| **Description** | Description of the profile | Profile for Snowflake over JDBC connect 2.0 plug-in | +| **Plug-in** | Plug-in type for this profile. A full list of available plug-ins is available on the **Package Management** page | Snowflake over JDBC connect 2.0 | +| **Credential** | The credential to authenticate with the datasource. Create the credential in **Credential Management**, or click **➕** to create a new one |The credential to authenticate with the datasource. Create the credential in **Credential Management**, or click **➕** to create a new one | Note: When using the KeyPair Authentication. use the below Connection string. # jdbc_connection_string => "jdbc:snowflake://...snowflakecomputing.com/?user=&warehouse=&db=&schema=&private_key_file=${THIRD_PARTY_PATH}/.p8&authenticator=snowflake_jwt" | +| **Kafka Cluster** | Kafka cluster to deploy the universal connector | Select from existing Kafka clusters attached to central management | +| **Label** | Grouping label (e.g., **customer name** or **ID**) | | +| **JDBC Driver Library** | JDBC driver for the database | Download from here (https://repo1.maven.org/maven2/net/snowflake/snowflake-jdbc/3.13.30/snowflake-jdbc-3.13.30.jar) | +| **Port** | Port used to connect to the database | Snowflake port (e.g., 30041) | +| **Hostname** | Hostname of the database | Snowflake hostname | +| **Query for execution status** | SQL query used to extract execution status audit logs | SELECT * FROM (SELECT QH.DATABASE_NAME as DATABASE_NAME,QH.SESSION_ID as SESSION_ID, TO_TIMESTAMP_LTZ(QH.END_TIME) as QUERY_TIMESTAMP, LH.CLIENT_IP as CLIENT_IP, CURRENT_IP_ADDRESS() as SERVER_IP, QH.USER_NAME as USER_NAME, S.CLIENT_ENVIRONMENT, QH.QUERY_ID, QH.QUERY_TEXT, QH.QUERY_TYPE, QH.QUERY_TAG , QH.ROLE_NAME, S.CLIENT_APPLICATION_ID, QH.WAREHOUSE_NAME, QH.ERROR_CODE AS QUERY_ERROR_CODE, QH.ERROR_MESSAGE AS QUERY_ERROR_MESSAGE, QH.EXECUTION_STATUS AS QUERY_EXECUTION_STATUS, DATE_PART(epoch_millisecond, QH.END_TIME) AS QUERY_TIMESTAMP_EPOCH FROM SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY QH LEFT JOIN SNOWFLAKE.ACCOUNT_USAGE.SESSIONS S ON S.SESSION_ID = QH.SESSION_ID LEFT JOIN SNOWFLAKE.ACCOUNT_USAGE.LOGIN_HISTORY LH ON S.LOGIN_EVENT_ID = LH.EVENT_ID WHERE QH.EXECUTION_STATUS <> 'RUNNING') AS QUERY_HISTORY_VIEW | +| **Query for Login success** | SQL query used to extract login success audit logs | SELECT * FROM (SELECT LH.USER_NAME AS USER_NAME, LH.CLIENT_IP AS CLIENT_IP, CURRENT_IP_ADDRESS() as SERVER_IP, (LH.REPORTED_CLIENT_TYPE \|\| ' ' \|\| LH.REPORTED_CLIENT_VERSION) AS CLIENT_APPLICATION_ID, LH.IS_SUCCESS AS LOGIN_SUCCESS, LH.ERROR_CODE AS LOGIN_ERROR_CODE, LH.ERROR_MESSAGE AS LOGIN_ERROR_MESSAGE, TO_TIMESTAMP_LTZ(LH.EVENT_TIMESTAMP) AS LOGIN_TIMESTAMP, DATE_PART(epoch_millisecond, LH.EVENT_TIMESTAMP) AS LOGIN_TIMESTAMP_EPOCH FROM SNOWFLAKE.ACCOUNT_USAGE.LOGIN_HISTORY LH WHERE LH.IS_SUCCESS LIKE '%NO%') AS QUERY_LOGIN_VIEW | +| **Tracking column type for execution status** | Data type of the column used to track execution status | Numeric | +| **Tracking column for execution status** | Column used to track execution status | query_timestamp_epoch | +| **Tracking column type for Login success** | Data type of the column used to track login success | Numeric | +| **Tracking column for Login success** | Column used to track login success | query_timestamp_epoch | +| **Connection String** | Connection string to connect to Snowflake database | jdbc:snowflake:///?user=>&warehouse=&db= | + +## Testing a Connection + +After creating a profile, you must test the connection to make sure that the provided configuration is valid. + +### Procedure + +1. Select the new profile. +2. From the top menu, click **Test Connection**. +3. If the test is successful, you can proceed to installing the profile. + +--- + +## Installing a Profile + +After the connection test succeeds, you can install the profile on **Managed Units (MUs)** or **Edges**. The parsed +audit logs are sent to the selected Managed Unit or Edge for to be consumed by the **Sniffer**. + +### Procedure + +1. Select the profile. +2. From the **Install** menu, click **Install**. +3. From the displayed list of available MUs and Edges, select the ones to which you want to deploy the profile. + +--- + +## Uninstalling or reinstalling profiles + +An installed profile can be uninstalled or reinstalled if needed. + +### Procedure +1. Select the profile. +2. From the list of available actions, select the desired option **Uninstall** or **Reinstall**. + +--- + + + diff --git a/docs/KafkaBasedUCs/SpannerPubsubKafkaConnect.md b/docs/KafkaBasedUCs/SpannerPubsubKafkaConnect.md new file mode 100644 index 000000000..d0482e1d0 --- /dev/null +++ b/docs/KafkaBasedUCs/SpannerPubsubKafkaConnect.md @@ -0,0 +1,234 @@ +# Spanner over over Pub/Sub Source Connector + +This connector enables IBM Guardium Data Protection (GDP) to monitor and collect audit logs from Spanner databases +through Google Cloud Pub/Sub using Kafka Connect. + +## Meet Spanner over PubSub Connect + +* Environments: On-prem +* Supported inputs: Kafka connect Pubsub 2.0 (pull) +* Supported Guardium versions: + * Guardium Data Protection: Appliance bundle 12.2.2 or later + +Kafka-connect is a framework for streaming data between Apache Kafka and other systems. + +## Configuring Spanner on GCP + +1. [Create a Spanner instance](https://cloud.google.com/spanner/docs/create-manage-instances). +2. [Create a database](https://cloud.google.com/spanner/docs/create-manage-databases). +3. [Create a log sink in Pub/Sub](https://cloud.google.com/logging/docs/export/configure_export_v2#creating_sink).
+ To specify which logs to route, use the following inclusion filter in the Choose logs to include in sink field during + log sink creation. This filter captures relevant data access and activity logs. + + ``` + resource.type="spanner_instance" resource.labels.instance_id="" + (logName="projects//logs/cloudaudit.googleapis.com%2Fdata_access" + AND (protoPayload.request.queryMode="PROFILE" + OR protoPayload.request.@type="type.googleapis.com/google.spanner.v1.ExecuteSqlRequest") + AND -protoPayload.request.queryMode="PLAN" + AND -protoPayload.request.requestOptions.requestTag:* + ) + OR + (logName="projects//logs/cloudaudit.googleapis.com%2Factivity" + AND (type.googleapis.com/google.spanner.admin.database.v1.UpdateDatabaseDdlRequest + OR type.googleapis.com/google.spanner.admin.database.v1.CreateDatabaseRequest) + AND operation.producer="spanner.googleapis.com" + ) + AND -protoPayload.request.sql="SELECT 1" + ``` + +## Configuring GCP for the input plug-in + +1. [Create a topic in Pub/Sub](https://cloud.google.com/pubsub/docs/create-topic#create_a_topic_2). +2. [Create a subscription in Pub/Sub](https://cloud.google.com/pubsub/docs/create-subscription#create_a_pull_subscription) +3. [Create service account credentials](https://developers.google.com/workspace/guides/create-credentials#create_a_service_account): + - To grant subscription access to the service account, select the **Pub/Sub Subscriber** role from the role + selection list during the service account creation process. + - You do not need to grant users access to this service account. +4. [Create credentials for a service account](https://developers.google.com/workspace/guides/create-credentials#create_credentials_for_a_service_account). + This key is used in the Kafka Connect connector configuration. + +## Enabling audit logs + +The inclusion filter that is used during log sink creation makes sure that only relevant logs are routed. + +### Viewing or downloading logs + +To view or download the generated logs, make sure that the appropriate Identity and Access Management (IAM) roles are +assigned. +These roles control access to logs in GCP. + +* **View logs**: + - roles/logging.viewer (Logs Viewer) + - roles/logging.privateLogViewer (Private Logs Viewer) +* **Download logs**: + - roles/logging.admin (Logging Admin) + - roles/logging.viewAccessor (Logs View Accessor) + +For more information on IAM roles and access control, +see [Access Control with IAM](https://cloud.google.com/logging/docs/access-control). + +### Setting destination permissions + +To route audit logs to a specific destination, such as Pub/Sub topic and subscription, follow these steps: + +1. [Get sink writer's identity](https://cloud.google.com/logging/docs/export/configure_export_v2#dest-auth). +2. If you have owner access to the + destination, [set access controls](https://cloud.google.com/pubsub/docs/access-control#console). Copy the sink + writer's identity and enter it in the **New Principals** field when you configure access policies for topics and + subscriptions.
+ * For **topics**, assign the **Pub/Sub Publisher** and **Pub/Sub Subscriber** role.
+ * For **subscriptions**, assign the **Pub/Sub Publisher** role. + +## Limitations + +1. Error logs are not generated in GCP for Spanner and this connector does not support error traffic in Guardium. +2. The audit/data access log doesn't contain a server IP. The default value for the server IP is set to `0.0.0.0`. +3. The following important fields cannot be mapped: + - Source program + - OS User + - Client HostName +4. Spanner does not require a DDL query for Drop Database operations because databases can only be deleted through the + Spanner UI. When a database is deleted through the UI, query parameters are not captured in audit logs. Therefore, + these operations do not appear in the full SQL report. +5. When you use GCP, duplicate entries can appear in both the reports and audit logs. + +## Configuring Guardium + +The Guardium universal connector is the Guardium entry point for native audit and data access logs. The Guardium +universal connector identifies and parses the received events, and converts them to a standard Guardium format. The +output of the Guardium universal connector is forwarded to the Guardium sniffer on the collector for policy and auditing +enforcements. You can configure Guardium to read the native audit and data access logs by customizing the Spanner +template. + +### Before you begin + +* Configure the policies that you need. For more information, see [Policies](/docs/#policies). +* You must have permissions for the S-Tap Management role. By default, the admin user is assigned the S-Tap Management + role. + +## Creating datasource profiles + +You can create a new datasource profile from the **Datasource Profile Management** page. + +### Procedure + +1. Go to **Manage > Universal Connector > Datasource Profile Management** +2. Click the **➕ (Add)** button. +3. You can create a profile by using one of the following methods: + + * To create a new profile manually, go to the **"Add Profile"** tab and provide values for the following fields. + * **Name** and **Description**. + * Select a **Plug-in Type** from the dropdown. For example, **Spanner Over PubSub Kafka Connect 2.0**. + + * To upload from CSV, go to the **Upload from CSV** tab and upload an exported or manually created CSV file + containing one or more profiles. You can also choose from the following options: + * **Update existing profiles on name match** — Updates profiles with the same name if they already exist. + * **Test connection for imported profiles** — Automatically tests connections after profiles are created. + * **Use ELB** — Enables ELB support for imported profiles. You must provide the number of MUs to be used in the + ELB process. + +**Note:** Configuration options vary based on the selected plug-in. + +## Configuration: Pub/Sub Kafka Connect 2.0-based Plugins + +The following table describes the fields that are specific to Pub/Sub Kafka Connect 2.0 and similar plugins. + +| Field | Description | +|-------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Name** | Unique name of the profile. | +| **Description** | Description of the profile. | +| **Plug-in** | Plug-in type for this profile. A full list of available plug-ins are available on the **Package Management** page. | +| **Credential** | The credential to authenticate with the datasource. Must be created in **Credential Management**, or click **➕** to create one. | +| **Kafka Cluster** | Kafka cluster to deploy the universal connector. | +| **Label** | Grouping label. For example, **customer name** or **ID**. | +| **GCP project id** | Google Cloud project ID that contains the Pub/Sub subscription. | +| **Pub/Sub Subscription ID** | Pub/Sub subscription ID from which messages are consumed. | +| **GCP Topic** | Pub/Sub topic name. | +| **Maximum poll records** | The maximum number of records returned in a single poll | +| **Expected events per second** | Expected events per second. This value is used to automatically calculate the **parallel.pull.count** parameter when it not set. Calculation formula: ceil(expected.eps / 1000). | | +| **Number of parallel pull streams** | Number of parallel pull streams to use. If not specified, this value is automatically calculated based on **expected.eps** (1 subscriber per 1000 EPS). | +| **No traffic threshold (minutes)** | The time period after which the system detects inactivity. | + +## Testing a Connection + +After creating a profile, you must test the connection to ensure that the provided configuration is valid. + +### Procedure + +1. Select the new profile. +2. From the top menu, click **Test Connection**. +3. If the test is successful, you can proceed to installing the profile. + +--- + +## Installing a Profile + +Once the connection test is successful, you can install the profile on **Managed Units (MUs)** or **Edges**. The parsed +audit logs are sent to the selected Managed Unit or Edge to be consumed by the Sniffer. + +### Procedure + +1. Select the profile. +2. From the **Install** menu, click **Install**. +3. From the list of available MUs and Edges that is displayed, select the ones that you want to deploy the profile to. + +--- + +## Uninstalling or reinstalling profiles + +An installed profile can be **uninstalled** or **reinstalled** if needed. + +### Procedure + +1. Select the profile. +2. From the list of available actions, select the desired option: **Uninstall** or **Reinstall**. + +--- + +## Required Configuration Changes for Production + +## Configuring GCP for production + +You must configure **exactly-once** delivery on your Pub/Sub subscription to prevent duplicate audit log entries at the +Pub/Sub level before the messages reach the connector. + +### Procedure + +1. To enable **exactly-once** delivery on your Pub/Sub subscription, use only one of the following commands based on + your scenario. + * For an existing subscription: + ```bash + gcloud pubsub subscriptions update \ + --enable-exactly-once-delivery + ``` + * For a new subscription: + ```bash + gcloud pubsub subscriptions create \ + --topic= \ + --enable-exactly-once-delivery \ + --ack-deadline=600 + ``` + +## Troubleshooting + +#### Messages are not being processed + +1. Verify that all Kafka worker settings are added. +2. Make sure the worker is restarted after configuration changes. +3. Restart the connector after the worker restart. + +#### "Quota exceeded" errors in GCP + +1. Check the current quota usage in the GCP console. +2. Request a quota increase. +3. Temporarily reduce the number of tasks until the quota is increased. + +#### High number of unacknowledged messages in Pub/Sub + +1. Verify that the connector is running by using the following command.
+ ``` + curl http://localhost:8083/connectors//status + ``` +2. Check for errors in logs. +3. Verify that exactly-once delivery is enabled on the subscription. \ No newline at end of file diff --git a/docs/KafkaBasedUCs/SybaseJDBCKafkaConnect.md b/docs/KafkaBasedUCs/SybaseJDBCKafkaConnect.md new file mode 100644 index 000000000..aecef366f --- /dev/null +++ b/docs/KafkaBasedUCs/SybaseJDBCKafkaConnect.md @@ -0,0 +1,344 @@ +# Sybase Datasource Profile Configuration Guide for JDBC Kafka Connect Plug-ins + +This guide provides instructions for creating and configuring datasource profiles through Central Manager for **Sybase +JDBC Kafka Connect** plug-ins. + +## Meet Sybase Over JDBC Connect + +* Tested versions: 16.1 +* Environments: On-prem +* Supported inputs: Kafka connect JDBC 2.0 (pull) +* Supported Guardium versions: + * Guardium Data Protection: 12.1 UC patch 5007 and above + * Guardium Data Protection: Appliance bundle 12.2.1 and above + +Kafka-connect is a framework for streaming data between Apache Kafka and other systems. + +## Configuring the Sybase Adaptive Server Enterprise (ASE) server + +### Procedure +1. Install Sybase ASE version 16.0 on your system based on your operating system. +2. Set up your system administrator (SA) credentials. +3. Install the `isql` utility. This command-line SQL interface is required to connect to and interact with Sybase ASE. +4. Download the `JTDS` JDBC driver by completing the following steps.
+ a. Go to the official jTDS website [https://jtds.sourceforge.net/](https://jtds.sourceforge.net/).
+ b. From the navigation menu, click on the **Download** link.
+ c. Click on the `jtds` folder, and select the desired version from the available releases (e.g., `1.3.1`).
+ d. Download the distribution package (e.g., `jtds-1.3.1-dist.zip`).
+ e. Extract the archive and locate the driver file `jtds-1.3.1.jar` in the extracted folder.
+ +## Connecting to the Sybase server + +### Procedure +1. Connect to your Sybase ASE instance by using the `isql` command-line utility with your system administrator credentials. + + ```isql -U sa -P -S ``` + + Parameters: +* `-U sa`: Connect as the system administrator (SA).
+* `-P `: Your system administrator account password,
+* `-S `: The name of your Sybase server instance. + +## Setting server timezone to UTC + +### Procedure + +1. To check the current time and date settings on your Sybase database server, run the `timedatectl` command. + + ``` + [sybase16@sybase16-sysqa ~]$ timedatectl + Local time: Thu 2025-10-23 14:21:13 EDT + Universal time: Thu 2025-10-23 18:21:13 UTC + RTC time: Thu 2025-10-23 18:21:13 + Time zone: America/New_York (EDT, -0400) + System clock synchronized: yes + NTP service: active + RTC in local TZ: no + ``` + +2. If needed, change timezone by running the `timedatectl set-timezone UTC` command. + +3. After connecting to your Sybase ASE instance by using the `isql` utility, verify that the local timezone is set to UTC. + + ``` + 1> SELECT getdate() AS LocalTime, getutcdate() AS UTCTime + 2> go + LocalTime UTCTime + ------------------------------- ------------------------------- + Oct 23 2025 6:22PM Oct 23 2025 6:22PM + + (1 row affected) + ``` + +## Creating database devices + +### Before you begin + +A directory is required to store the database devices that you create. If this directory doesn't exist already, create it by using the following command. + +``` +sudo mkdir -p $SYBASE/$SYBASE_ASE/data +``` + +### Procedure + +1. Create a device to store table data. + + ``` + 1> disk init + name = "auditdev", + physname = "$SYBASE/$SYBASE_ASE/data/audit_device.dat", + size = "500M" + 2> go + ``` + +2. Create a device to store audit logs. + + ``` + 1> disk init + name = "auditlogdev", + physname = "$SYBASE/$SYBASE_ASE/data/audit_log_device.dat", + size = "500M" + 2> go + ``` + +Parameters: + +* `name`: Logical name for the device.
+* `physname`: Physical file path on disk where the device is created.
+* `size`: Megabytes allocated for this device (change the size based on your needs). + + +## Creating `sybsecurity` databases + +1. To store and manage audit information, create a `sybsecurity` database and allocate storage space based on your requirements. + + ``` + 1> CREATE DATABASE sybsecurity + ON auditdev = '500M' + LOG ON auditlogdev = '250M' + 2> go + ``` + +2. Configure the database by running the following security installation script. + + ``` + isql -U sa -P password -S server_name -i $SYBASE/$SYBASE_ASE/scripts/installsecurity + ``` + +3. Verify that the database is successfully created. + + ``` + 1> sp_helpdb sybsecurity + 2> go + ``` + +4. Restart the Sybase server. + +## Enabling auditing + +1. Access the security database and configure the audit parameters. + + ``` + 1> use sybsecurity + 2> go + ``` + +2. Enable the auditing feature. + + ``` + 1> sp_configure "auditing", 1 + 2> go + ``` + +3. Allow the configuration updates. + + ``` + 1> sp_configure "allow updates", 1 + 2> go + ``` + +4. Set the audit policies. + To ensure that the server remains stable when the audit log device runs out of space, configure Sybase ASE to suspend auditing gracefully instead of crashing. This prevents system failure by halting audit operations when the audit device is full. + + ``` + 1> sp_configure "suspend audit when device full", 1 + 2> go + ``` + +5. Enable comprehensive auditing for the system administrative role (change this based on your needs). + + ``` + 1> sp_audit "all", "sa_role", "all", "on" + 2> go + ``` + +6. Customize the audit settings. + + The `sp_audit` command accepts the following four parameters: + * the option type ('insert'), + * the user scope ('all' for all users), + * the object to monitor ('sybasetable'), + * the setting ('on' to enable). + + For example, the following commands enable auditing for `sybasetable` insert operations. + + ``` + 1> create database sybasedb + 2> go + 1> use sybasedb + 2> go + 1> create table sybasetable (id INT) + 2> go + 1> sp_audit 'insert', 'all', 'sybasetable', 'on' + 2> go + Audit option has been changed and has taken effect immediately. + (return status = 0) + ``` + + For more information on customizing audit settings, see [sp_audit](https://help.sap.com/docs/SAP_ASE/29a04b8081884fb5b715fe4aa1ab4ad2/ab54050ebc2b1014b5d9ca93507f4a1d.html). + +7. Verify the audit configurations. + + ``` + -- Check audit status + 1> sp_displayaudit + 2> go + + -- View current audit records + 1> select count(*) from sybsecurity..sysaudits_01 + 2> go + ``` + +## Limitations + +#### 1. Failed Login Attempt Logging + +Sybase ASE generates multiple audit log entries for failed login attempts, depending on the failure type. + + * **Invalid Username**: When a login attempt fails due to an invalid username, Sybase ASE generates four audit log entries. + + For example, the `extrainfo` values in the following Sybase audit table: + + | ExtraInfo Column | + |------------------------------------------------------------------------------------------------| + | `; ; ; ; sybase16-hostname, 9.00.100.100, network password encryption not set, 16106.14.1; ; ;` | + | `; ; ; ; 4002.14.1; ; ;` | + | `; ; ; ; sybase16-hostname, 9.00.100.100, network password encryption not set, 16106.14.1; ; ;` | + | `; ; ; ; 4002.14.1; ; ;` | + + * **Invalid Password**: When a valid username is provided with an incorrect password, Sybase ASE generates two audit log entries. + + For example, the `extrainfo` values (for username `sa`) in the following Sybase audit table: + + | ExtraInfo Column | + |---------------------------------------------------------------------------------------------------------------------------------------------------------------| + | `sa_role sso_role oper_role sybase_ts_role mon_role; ; ; ; sybase16-hostname, 9.00.100.100, network password rsa encryption with nonce, 4067.14.1; ; sa/ase;` | + | `sa_role sso_role oper_role sybase_ts_role mon_role; ; ; ; 4002.14.1; ; sa/ase;` | + + **Note:** Failed login attempts are tracked and reported in the **SQL Errors** and **Failed Login** reports. + +#### 2. Missing Audit Record Fields + +The following fields are not available in the Sybase ASE audit records and are populated with default values: + + * **Server IP**: Default value `"0.0.0.0"` + * **Server Hostname**: Default value `"N.A."` + * **Server Port**: Derived from the JDBC connection string + +#### 3. Field Truncation + +`Client Hostname` and `Source Program` fields may be truncated due to character length limitations in the Sybase ASE system tables. + +#### 4. Query Length Limitations + +Sybase ASE splits long queries across multiple audit records in audit table. When processing these records, queries may appear incomplete and may be dropped if they exceed length limits. +This affects long queries or complex statements. + +## Creating datasource profiles + +You can create a new datasource profile from the **Datasource Profile Management** page. + +### Procedure + +1. Go to **Manage > Universal Connector > Datasource Profile Management** +2. Click the **➕ (Add)** button. +3. You can create a profile by using one of the following methods: + + * To **Create a new profile manually**, go to the **"Add Profile"** tab and provide values for the following fields. + * **Name** and **Description**. + * Select a **Plug-in Type** from the dropdown. For example, `Sybase Over JDBC Kafka Connect 2.0`. + + * To **Upload from CSV**, go to the **"Upload from CSV"** tab and upload an exported or manually created CSV file containing one or more profiles. You can also choose from the following options: + * **Update existing profiles on name match** — Updates profiles with the same name if they already exist. + * **Test connection for imported profiles** — Automatically tests connections after profiles are created. + * **Use ELB** — Enables ELB support for imported profiles. You must provide the number of MUs to be used in the ELB process. + +**Note:** Configuration options vary based on the selected plug-in. + +## Configuration: JDBC Kafka Connect 2.0-based Plugins + +The following table describes the fields that are specific to JDBC Kafka Connect 2.0 and similar plugins. + +| Field | Description | +|-------------------------------|-----------------------------------------------------------------------------------------------------------------------------| +| **Name** | Unique name of the profile. | +| **Description** | Description of the profile. | +| **Plug-in** | Plug-in type for this profile. A full list of available plug-ins are available on the **Package Management** page. | +| **Credential** | The credential to authenticate with the datasource. Must be created in **Credential Management**, or click **➕** to create one. | +| **Kafka Cluster** | Kafka cluster to deploy the universal connector. | +| **Label** | Grouping label. For example, customer name or ID. | +| **JDBC Driver Library** | JDBC driver for the database. | +| **Port** | Port that is used to connect to the database. | +| **Hostname** | Hostname of the database. | +| **Query** | SQL query that is used to extract audit logs. | +| **Service Name / SID** | The database **service name** or **SID**. | +| **Initial Time** | Initial polling time for audit logs. | +| **No Traffic Threshold** | Threshold setting for inactivity detection. | +| **Use ELB** | Enable this if Enterprise Load Balancing (ELB) support is required. | +| **Managed Unit Count** | Number of Managed Units (MUs) to allocate for ELB. | + + +**Note:** +- Depending on the plugin type, the configuration may require either: + - A **Connection URL**, or + - Separate fields for **Hostname**, **Port**, and **Service Name / SID** +- Ensure that the **profile name** is unique. +- Required credentials must be created before or during profile creation. + +--- + +## Testing a Connection + +After creating a profile, you must **test the connection** to ensure the provided configuration is valid. + +### Procedure + +1. Select the new profile. +2. From the top menu, click **Test Connection**. +3. If the test is successful, you can proceed to installing the profile. + +--- + +## Installing a Profile + +Once the connection test is successful, you can install the profile on **Managed Units (MUs)** or **Edges**. The parsed audit logs are sent to the selected Managed Unit or Edge to be consumed by the **Sniffer**. + +### Procedure + +1. Select the profile. +2. From the **Install** menu, click **Install**. +3. From the list of available MUs and Edges that is displayed, select the ones that you want to deploy the profile to. + +--- + +## Uninstalling or reinstalling profiles + +An installed profile can be **uninstalled** or **reinstalled** if needed. + +### Procedure + +1. Select the profile. +2. From the list of available actions, select the desired option: **Uninstall** or **Reinstall**. + +--- diff --git a/docs/KafkaBasedUCs/TeradataJDBCKafkaConnect.md b/docs/KafkaBasedUCs/TeradataJDBCKafkaConnect.md new file mode 100644 index 000000000..783c3109e --- /dev/null +++ b/docs/KafkaBasedUCs/TeradataJDBCKafkaConnect.md @@ -0,0 +1,223 @@ +# Teradata Datasource Profile Configuration Guide for JDBC Kafka Connect Plug-ins + +This guide provides instructions for creating and configuring datasource profiles through Central Manager for **Teradata JDBC Kafka Connect** plug-ins. + +### Meet Teradata over JDBC Connect + +* Tested versions: 20.0 +* Environments: On-prem +* Supported inputs: Kafka connect JDBC 2.0 (pull) +* Supported Guardium versions: + * Guardium Data Protection: 12.1 UC patch 5007 and above + * Guardium Data Protection: Appliance bundle 12.2.1 and above + +Kafka-connect is a framework for streaming data between Apache Kafka and other systems. + +## Configuring the Teradata server + +### Before you begin +There are multiple ways to install a Teradata server. For this example, we assume that we already have a working Teradata server setup. + +Download the [Teradata JDBC Driver](https://downloads.teradata.com/download/connectivity/jdbc-driver). + +## Enabling auditing + +### Procedure +1. Connect to the Teradata server by using SSH. + +2. Log in to the Teradata system by using BTEQ with the credentials of a user (such as DBC), that has the permissions to execute ``DBQLAccessMacro``. In the following command, enter the password for the DBC user.
+ + ``` + bteq .logon /dbc, + ``` + +3. Create a user to read logs from audit tables by using the following command. + + ``` + CREATE USER AS PERMANENT = 100000000 BYTES PASSWORD = "" + ``` + +4. Grant read access of objects within the DBC user to newly created user by using the following command. + + ``` + GRANT SELECT ON "dbc" TO ""; + ``` + +5. Enable Database Query Logging. + + **Note:** Query logging involves a range of table/view combinations within the DBC database. Enable query logging for users, accounts, or applications only when necessary. + + You can enable query logging for all users or for specific users. In the following example command, query logging is enabled for all users. + + ``` + BEGIN QUERY LOGGING WITH SQL LIMIT SQLTEXT=0 ON ALL; + ``` + + For more information about Database Query Logging, see [DBQL](https://docs.teradata.com/r/qOek~PvFMDdCF0yyBN6zkA/f7yJJ4siIiBUpoQVvvAwpQ). + +6. To exit the BTEQ terminal, type `exit;`. +7. Set the database time zone by configuring the following ``dbscontrol`` fields. + + a. Set the `"18. System TimeZone String"` parameter to the timezone that you want to configure for your database.
+ b. Set the `"57. TimeDateWZControl "` parameter to 2. + +8. Close the terminal. + +9. Verify that a logging rule is created in a table. Log in with DBC user. Run the following query by using any client utility. + In the following example, the Teradata Studio Express is used. + + ``` + select * from DBC.DBQLRulesV; + ``` + +### Notes: +For **Teradata VCE on Azure**, enable audit logging by using the following command. + + BEGIN QUERY LOGGING WITH SQL LIMIT SQLTEXT=0 ON ALL; + +Verify that the audit log is enabled by running the following command. + + select * from DBC.DBQLRulesV; + + +## Disabling auditing +You can disable auditing can be disabled by using the credentials of a user (such as DBC) that has the permissions to execute ``DBQLAccessMacro``. + +To disable query logging, run the following command. + + END QUERY LOGGING WITH SQL LIMIT SQLTEXT=0 ON ALL; + + +## Archiving and deleting DBQL logs + +There are many ways to archive and delete DBQL logs. To delete old log data from system tables manually, complete the following steps. + +### Before you begin +Before you perform cleanup activities on DBQL logs, it is recommended to disable DBQL logging to prevent potential performance issues. Disabling logging makes sure that the delete process does not lock the DBQL tables, which could slowdown the system if DBQL needs to flush cache to the same table for query logging. + +**Note:** You cannot delete data that is less than 30 days old. + +### Procedure + +1. Back up log data.
+ a. Create a duplicate log table in another database by using the ``Copy Table`` syntax for the `CREATE TABLE` statement.
+ ``` + CT DBC.tablename AS databasename.tablename + ``` + b. Back up the table to tape storage in accordance with your site backup policy.
+ c. Drop the duplicate table using a ``DROP TABLE`` statement. +3. Log on to Teradata Studio as DBADMIN or another administrative user with DELETE privileges on the DBC database. +4. In the **Query** window, enter an SQL statement to purge old log entries. For example, + + `DELETE FROM DBC.object_name WHERE (Date - LogDate) > number_of_days ;` + + Examples for using the above query. + ``` + DELETE FROM DBC.DBQLOGTBL WHERE (DATE '2021-12-16' - cast(starttime as DATE)) > 30 ; + DELETE FROM DBC.DBQLSqlTbl WHERE (DATE '2021-12-16' - cast(collecttimestamp as DATE)) > 30 ; + ``` + +## Limitations + +1. The Teradata sniffer parser may not parse certain operations accurately. This plug-in does not support the following operations:
+ a. User Management
+ b. DBQL Queries
+ c. Timestamp configuration
+ d. Cast operations
+ e. Stored Procedure and User Defined Functions
+2. The following fields are not found in TeradataDB audit logs:
+ a. ``Client HostName`` : Not Available with audit logs.
+3. The Teradata auditing does not audit authentication failure (Login Failed) operations. +4. In case of the EC2 guardium instance, Teradata traffic takes longer (25-30 min) to populate data in the full SQL report. +5. This plug-in supports queries that are approximately 32,000 characters long. When the count of characters in a query exceeds the given count, the remaining part of the query is stored in other rows and the **SQLTextInfo** column of the `DBC.DBQLSqlTbl` table has more than one row per ``QueryID``. +6. Client IP and Server IP are retrieved from DBC.QryLogClientAttrV view using ClientIPAddrByClient and ServerIPAddrByServer fields respectively, as recommended by Teradata support. The deprecated logonsource field is no longer used for IP address retrieval. + + For more information on DBC.QryLogClientAttrV, please refer to this [documentation](https://docs.teradata.com/r/Enterprise_IntelliFlex_VMware/Data-Dictionary/Views-Reference/QryLogClientAttrV). + +## Creating datasource profiles +You can create a new datasource profile from the **Datasource Profile Management** page. + +### Procedure + +1. Go to **Manage > Universal Connector > Datasource Profile Management** +2. Click the **➕ (Add)** button. +3. You can create a profile by using one of the following methods: + + - To **Create a new profile manually**, go to the **"Add Profile"** tab and provide values for the following fields. + - **Name** and **Description**. + - Select a **Plug-in Type** from the dropdown. For example, `Teradata Over JDBC Connect 2.0`. + + - To **Upload from CSV**, go to the **"Upload from CSV"** tab and upload an exported or manually created CSV file containing one or more profiles. + You can also choose from the following options: + - **Update existing profiles on name match** — Updates profiles with the same name if they already exist. + - **Test connection for imported profiles** — Automatically tests connections after profiles are created. + - **Use ELB** — Enables ELB support for imported profiles. You must provide the number of MUs to be used in the ELB process. + +**Note:** Configuration options vary based on the selected plug-in. + +## Configuring JDBC Kafka Connect 2.0-based plugins + +The following table describes the fields that are specific to JDBC Kafka Connect 2.0 and similar plugins. + +| Field | Description | +|--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Name** | Unique name of the profile. | +| **Description** | Description of the profile. | +| **Plug-in** | Plug-in type for this profile. A full list of available plug-ins are available on the **Package Management** page. | +| **Credential** | The credential to authenticate with the datasource. Must be created in **Credential Management**, or click **➕** to create one. | +| **Kafka Cluster** | Kafka cluster to deploy the Universal Connector. | +| **Label** | Grouping label. For example, customer name or ID. | +| **JDBC Driver Library** | JDBC driver for the database. | +| **Port** | Port used to connect to the database. | +| **Hostname** | Hostname of the database. | +| **Query** | SQL query that is used to extract audit logs. | +| **Service Name / SID** | The database **service name** or **SID**. | +| **Initial Time** | Initial polling time for audit logs. | +| **No Traffic Threshold** | Threshold setting for inactivity detection. | +| **Use ELB** | Enable this if Enterprise Load Balancing (ELB) support is required. | +| **Managed Unit Count** | Number of Managed Units (MUs) to allocate for ELB. | + + +**Note:** +- Depending on the plugin type, the configuration may require either: + - A **Connection URL**, or + - Separate fields for **Hostname**, **Port**, and **Service Name / SID** +- Ensure that the **profile name** is unique. +- Required credentials must be created before or during profile creation. + +--- + +## Testing connections + +After creating a profile, you must **test the connection** to ensure that the provided configuration is valid. + +### Procedure + +1. Select the new profile. +2. From the top menu, click **Test Connection**. +3. If the test is successful, you can proceed to installing the profile. + +--- + +## Installing profiles + +Once the connection test is successful, you can install the profile on **Managed Units (MUs)** or **Edges**. The parsed audit logs are sent to the selected Managed Unit or Edge to be consumed by the **Sniffer**. + +### Procedure + +1. Select the profile. +2. From the **Install** menu, click **Install**. +3. A list of available MUs and Edges are displayed. Choose the specific MUs and Edges to which you want to apply the new profile. + +--- + +## Uninstalling or reinstalling profiles + +An installed profile can be **uninstalled** or **reinstalled** if needed. + +### Procedure + +1. Select the profile. +2. From the list of available actions, select the desired option **Uninstall** or **Reinstall**. + +--- diff --git a/docs/KafkaBasedUCs/WatsonXKafkaConnect.md b/docs/KafkaBasedUCs/WatsonXKafkaConnect.md new file mode 100644 index 000000000..902c035a1 --- /dev/null +++ b/docs/KafkaBasedUCs/WatsonXKafkaConnect.md @@ -0,0 +1,147 @@ +# Configuring WatsonX data source profiles for Kafka Connect plug-ins + +You can create and configure datasource profiles through Central Manager for **WatsonX Kafka Connect** plug-ins. + +## About WatsonX Kafka Connect + +* **Environment:** IBM Cloud +* **Supported inputs:** Kafka connect WatsonX 2.0 (pull) +* **Supported Guardium versions:** + * Guardium Data Protection: Appliance bundle 12.2.2 or later + +Kafka Connect is a framework for streaming data between Apache Kafka and other systems. You can use this connector to monitor WatsonX audit logs through Kafka. + +## Configuring WatsonX on IBM Cloud + +### Setting up WatsonX + +1. Log in to the WatsonX portal. +2. From the menu in the upper right, select **IBM Cloud Pak for Data**. +3. Under **Instances**, click **View all**. +4. Create a WatsonX instance or select an existing instance. +5. From **Access information**, note the access token. +6. From **Configuration**, note the instance ID. +7. In the upper right, click **Admin** > **Profile and settings**. +8. Click **API key**, and then generate a new API key. +9. Return to your instance, then click **Open** in the upper right to open the WatsonX dashboard. +10. From the menu on the left, click **Configurations** > **IBM Guardium** > **Enable** to enable audit logging.
+ **Note:** You must complete this step after you install the profile to avoid installation failure. +11. From the menu on the left, click **Data Manager**. +12. Run queries. +13. Retrieve the SSL certificate by completing the following steps.
+ a. Log in to the CPD cluster from the terminal and run the following command:
+ ``` oc get secret ibm-lh-tls-secret -n cpd-instance -o jsonpath='{.data.ca\.crt}' | base64 --decode > ca.crt```
+ + b. Transfer the file to your local machine by running the following command outside your cluster:
+ ``` scp @:ca.crt ```
+ +## Limitations + +The WatsonX Kafka Connect plug-in has the following limitations: + +* **Duplicate log entries:** Audit logs might contain duplicate entries due to the nature of the data streaming process. +* **Query operation support:** WatsonX does not support `UPDATE` and `DELETE` query operations for audit logging. +* **Authentication model:** The query engine does not require separate login authentication. Therefore `LOGIN FAILED` exceptions are not captured in the audit logs. +* **Access token expiration:** Access tokens expire after 20 minutes. While this limitation does not affect deployed connectors during normal operation, you must manually refresh tokens if the connector is redeployed after an extended period of inactivity. +* **ELB:** Currently, ELB is not supported because token expiration prevents reliable failover. +* **Single Subscriber Per Instance**: Only one connector can be deployed per WatsonX instance ID at a time, even across different Guardium systems. Running multiple connectors with the same instance ID or testing a connection while a connector is running causes subscription conflicts and failures. + +## Configuring credentials + +The following table describes the credential fields required for WatsonX authentication. + +| Field | Description | +|--------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Credential Type** | Select `WatsonX Credentials` as the credential type. | +| **API Key** | The API key generated in step 8 of the WatsonX setup. | +| **Access Token** | The access token obtained from **Access information** in step 5. This token expires every 20 minutes. You can regenerate the token from the portal as needed. | +| **SSL Truststore** | Upload the SSL certificate. + + +### Creating WatsonX credentials + +### Procedure + +1. Click **Manage > Universal Connector > Credential Management**. +2. Click the **➕ (Add)** button to create a new credential. +3. Provide the following information: + +--- + +## Creating Datasource Profiles + +You can create a new datasource profile from the **Datasource Profile Management** page. + +### Procedure + +1. Go to **Manage > Universal Connector > Datasource Profile Management**. +2. Click the **➕ (Add)** button. +3. You can create a profile by using one of the following methods: + + - To **Create a new profile manually**, go to the **"Add Profile"** tab and provide values for the following fields. + - **Name** and **Description**. + - From the dropdown, select a **Plug-in Type** . For example, `WatsonX over Kafka Connect 2.0` + + - To **Upload from CSV**, go to the **"Upload from CSV"** tab and upload an exported or manually created CSV file containing one or more profiles. + You can also choose from the following options: + - **Update existing profiles on name match** — Updates profiles with the same name if they already exist. + - **Test connection for imported profiles** — Automatically tests connections after profiles are created. + - **Use ELB** — Enables ELB support for imported profiles. You must provide the number of MUs to be used in the ELB process. + +**Note:** Configuration options vary based on the selected plug-in. + +## Configuring WatsonX Kafka Connect 2.0 + +The following table describes the fields that are specific to WatsonX Kafka Connect 2.0 plugin. + +| Field | Description | +|--------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Name** | Unique name of the profile. | +| **Description** | Description of the profile. | +| **Plug-in** | Plug-in type for this profile. Select **WatsonX over Kafka Connect 2.0**. A complete list of available plug-ins is available on the **Package Management** page. | +| **Credential** | The credential to authenticate with WatsonX. The credential must be created in **Credential Management**, or click **➕** to create one. For more information, see [Creating Credentials](https://www.ibm.com/docs/en/SSMPHH_12.x/com.ibm.guardium.doc.stap/guc/guc_credential_management.html). | +| **Kafka Cluster** | Select the appropriate Kafka cluster from the available Kafka cluster list or create a new Kafka cluster. For more information, see [Managing Kafka clusters](https://www.ibm.com/docs/en/SSMPHH_12.x/com.ibm.guardium.doc.stap/guc/guc_kafka_cluster_management.html). | +| **Label** | Grouping label. For example, customer name or ID. | +| **Username** | The username that you use to log in to WatsonX. | +| **WatsonX URL** | The URL of your WatsonX cluster. | +| **Instance ID** | The instance ID that you obtained from step 6. | +| **No-traffic threshold (minutes)** | The default value is 60. If no incoming traffic is received for an hour, S-TAP displays a red status. When incoming traffic resumes, the status returns to green. | +| **Use Enterprise Load Balancing (ELB)** | Enable this option if ELB support is required. | +| **Managed Unit Count** | Number of Managed Units (MUs) to allocate for ELB. | + +**Note:** +- Make sure that the **profile name** is unique. + +## Testing a connection + +After you create a profile, test the connection to make sure that the configuration is valid. + +### Procedure + +1. Select the new profile. +2. From the menu, click **Test Connection**. +3. If the test is successful, you can install the profile. + +--- + +## Installing a profile + +After the connection test is successful, you can install the profile on Managed Units (MUs) or Edges. The parsed audit logs are sent to the selected Managed Unit or Edge to be consumed by the Sniffer. + +### Procedure + +1. Select the profile. +2. From the **Install** menu, click **Install**. +3. From the list of available MUs and Edges, select the ones where you want to deploy the profile. +4. After the profile is installed, complete step 10 from the WatsonX setup section to enable audit logging. + +--- + +## Uninstalling or reinstalling profiles + +You can uninstall or reinstall an installed profile if needed. + +### Procedure + +1. Select the profile. +2. From the list of available actions, select **Uninstall** or **Reinstall**. \ No newline at end of file diff --git a/docs/Logstash9UpgradeLimitations.md b/docs/Logstash9UpgradeLimitations.md new file mode 100644 index 000000000..f40a8fcd1 --- /dev/null +++ b/docs/Logstash9UpgradeLimitations.md @@ -0,0 +1,203 @@ +# Logstash 9.3.3 Upgrade Guide + +## Overview + +This document provides guidance for applying patch 5008, which upgrades the Universal Connector from Logstash 8.3.3 to Logstash 9.3.3. + +**Important Considerations:** +- Logstash 9.3.3 introduces changes to SSL/TLS configuration parameters +- Configuration updates are required for Universal Connector deployments +- Both Legacy Flow and Central Manager Flow are affected + +**Impact by Deployment Type:** +- **Legacy Flow:** Requires manual updates to Logstash configuration files with new parameter names +- **Central Manager Flow:** Requires Universal Connector profile reinstallation after applying patch 5008 + +## Configuration Parameter Changes + +Logstash 9.3.3 has updated SSL/TLS configuration parameters in the TCP input plugin. The following parameter names have changed: + +### Parameter Name Changes + +| Old Parameter (Logstash 8.3.3) | New Parameter (Logstash 9.3.3) | Description | +|---------------------------------|--------------------------------|-------------| +| `ssl_enable` | `ssl_enabled` | Enable/disable SSL | +| `ssl_cert` | `ssl_certificate` | Path to SSL certificate | +| `ssl_verify` | **Deprecated/Removed** | SSL verification (replaced by `ssl_client_authentication`) | + +**Note:** The following parameters remain unchanged: +- `ssl_key` +- `ssl_key_passphrase` +- `ssl_client_authentication` + +The `ssl_verify` parameter has been deprecated in Logstash 9.3.3 and replaced with `ssl_client_authentication`. + +**Current Support:** +- Guardium Universal Connectors support `ssl_client_authentication => "none"` (equivalent to `ssl_verify => false`) +- The values `"optional"` and `"required"` are not currently supported + +**Deployment-Specific Impact:** + +**Legacy Flow:** +- Configuration files must be manually updated with the new parameter names +- This document provides the complete migration guide + +**Central Manager Flow:** +- After applying patch 5008, the Universal Connector profiles must be reinstalled through the Central Manager UI +- The SSL verification UI option has been updated in GDP 12.2.3 +- Fresh installations of GDP 12.2.3 use `ssl_client_authentication => "none"` for SSL connections +- When applying patch 5008, the `ssl_verify` checkbox is replaced with a textbox that can be left empty or set to `none` + +## Affected Connectors + +The following Universal Connectors are affected by these changes: + +### 1. MongoDB over Syslog +- **Configuration File:** `filter-plugin/logstash-filter-mongodb-guardium/MongoDBOverSyslogPackage/mongodbSyslog.conf` +- **Impact:** TCP input configuration requires parameter updates + +### 2. PostgreSQL over Syslog (EDB) +- **Configuration File:** `filter-plugin/logstash-filter-onPremPostgres-guardium/PostgresOverSyslogPackage/PostgresEDB/filter.conf` +- **Impact:** TCP input configuration requires parameter updates + +### 3. CockroachDB over Syslog +- **Configuration File:** `filter-plugin/logstash-filter-cockroachdb-guardium/CockroachDBOverSyslogPackage/CockroachDBOverSyslog.conf` +- **Impact:** TCP input configuration requires parameter updates + +### 4. Milvus over Filebeat +- **Configuration File:** `filter-plugin/logstash-filter-milvus-guardium/milvusOverFilebeat.conf` +- **Impact:** Beats input configuration requires parameter updates + +### 5. MySQL Percona over Filebeat +- **Configuration File:** `filter-plugin/logstash-filter-mysql-percona-guardium/perconaFilebeat.conf` +- **Impact:** Beats input configuration requires parameter updates + +### 6. TCP Syslog Input Plugin +- **Documentation:** `input-plugin/logstash-input-tcp-syslog/README.md` +- **Impact:** Documentation and examples updated + +### 7. Beats Input Plugin +- **Configuration File:** `input-plugin/logstash-input-beats/FilebeatInputPackage/Filebeat/input.conf` +- **Documentation:** `input-plugin/logstash-input-beats/README.md` +- **Impact:** Configuration and documentation updated + +## Migration Guide + +### For Legacy Flow Deployments + +When applying **patch 5008** or performing a **fresh install of GDP 12.2.3**, the Universal Connector will use Logstash 9.3.3. Configuration files must be manually updated with the new parameter names after the upgrade. + + +#### Step 1: Identify Affected Configurations + +Review all your Universal Connector configurations that use TCP or Beats input plugins with SSL/TLS enabled. + +#### Step 2: Update Configuration Parameters + +Replace the deprecated parameters with their new equivalents: + +**Before (Logstash 8.3.3):** +```conf +input { + tcp { + port => 6514 + ssl_enable => true + ssl_cert => "${SSL_DIR}/tls.crt" + ssl_key => "${SSL_DIR}/tls.key" + ssl_key_passphrase => "${ssl_key_passphrase}" + ssl_verify => false + } +} +``` + +**After (Logstash 9.3.3):** +```conf +input { + tcp { + port => 6514 + ssl_enabled => true + ssl_certificate => "${SSL_DIR}/tls.crt" + ssl_key => "${SSL_DIR}/tls.key" + ssl_key_passphrase => "${ssl_key_passphrase}" + ssl_client_authentication => "none" + } +} +``` + +#### Step 3: Test Configuration + +Before deploying to production: + +1. Validate your installed UC configuration: + +2. Test with sample data in a non-production environment + +3. Verify SSL/TLS connections are working correctly + + +### For Central Manager (CM Flow) Deployments + +After applying **patch 5008**, all datasource profiles must be reinstalled to ensure they are using the updated Logstash 9.3.3 configuration: + +1. Navigate to **Manage > Universal Connector > Datasource Profile Management** +2. Select all the profiles +3. Follow the profile reinstallation process through the Central Manager UI + + +**Important Note for Syslog Profiles:** +For syslog-based profiles, you **must** edit and save the profile before reinstalling it. This ensures that the profile configuration is properly updated with the new Logstash 9.3.3 parameter names. Simply reinstalling without editing and saving first may not apply the necessary configuration updates. + +**Note:** Profile reinstallation is necessary to ensure Universal Connector configurations are properly updated to work with Logstash 9.3.3 after applying patch 5008. + +## Compatibility Notes + +Configuration changes introduced in Logstash 9.3.3 are not backward compatible: + +- Old parameter names (`ssl_enable`, `ssl_cert`, `ssl_verify`) are not supported in Logstash 9.3.3 +- Configurations must be updated appropriately for GDP version 12.2.3+ or Patch 5008 + +## Post-Upgrade Validation + +After applying patch 5008, verify the following: + +- [ ] TCP input configurations use updated parameter names +- [ ] Beats input configurations use updated parameter names +- [ ] SSL/TLS connections are functioning correctly +- [ ] (Central Manager only) Universal Connector profiles reinstalled + +## Troubleshooting + +### Common Issues + +#### Issue 1: Configuration Validation Fails +**Symptom:** Logstash fails to start with configuration errors related to SSL parameters + +**Solution:** +- Verify all `ssl_enable` instances are changed to `ssl_enabled` +- Verify all `ssl_cert` instances are changed to `ssl_certificate` +- Remove or replace `ssl_verify` with `ssl_client_authentication => "none"` + - Note: Only `"none"` is currently supported by Guardium Universal Connectors + +#### Issue 2: SSL/TLS Connection Failures +**Symptom:** Data sources cannot connect to Logstash + +**Solution:** +- Verify certificate paths are correct +- Ensure `ssl_enabled => true` is set +- Verify `ssl_client_authentication => "none"` is configured (currently the only supported value) +- For Central Manager deployments, ensure all profiles have been reinstalled +- For patch 5008, the `ssl_verify` textbox can be left empty or set to `none` + +#### Issue 3: No Data Flowing After Upgrade +**Symptom:** Logstash starts but no events are received + +**Solution:** +- Verify the Universal Connector is up and running +- For Central Manager deployments, ensure all profiles have been reinstalled through the UI + +**Applies To:** +- Guardium Data Protection 12.2.3 and above (includes Logstash 9.3.3) +- Patch 5008 (Logstash 9.3.3) +- Logstash 9.3.3 and above (standalone installations) +**Previous Versions:** +- GDP < 12.2.3 (Logstash 8.3.3) \ No newline at end of file diff --git a/docs/available_plugins.md b/docs/available_plugins.md index 1e60c437e..ca7295781 100644 --- a/docs/available_plugins.md +++ b/docs/available_plugins.md @@ -4,53 +4,61 @@ The following plug-ins are supported by the latest product versions. Exceptions are noted next to each plug-in name. To see if a particular plug-in is supported by older versions, please refer to the "supported versions" section inside each plug-in page. -* [Amazon Aurora MySQL](../filter-plugin/logstash-filter-aurora-mysql-guardium/README.md) -* [Amazon DynamoDB](../filter-plugin/logstash-filter-dynamodb-guardium/README.md) -* [Amazon RDS for MariaDB](../filter-plugin/logstash-filter-mariadb-aws-guardium/README.md) -* [Amazon RDS for MySQL](../filter-plugin/logstash-filter-mysql-aws-guardium/README.md) -* [Amazon RDS for Postgres and Aurora Postgres](../filter-plugin/logstash-filter-postgres-guardium/README.md) -* [Amazon Redshift](../filter-plugin/logstash-filter-redshift-aws-guardium/README.md) +* [Adabas](../filter-plugin/logstash-filter-adabas-guardium/README.md) +* [Alloy_DB](../filter-plugin/logstash-filter-alloydb-guardium/mainREADME.md) +* [Amazon Aurora MySQL](../filter-plugin/logstash-filter-aurora-mysql-guardium/mainREADME.md) +* [Amazon DynamoDB](../filter-plugin/logstash-filter-dynamodb-guardium/mainREADME.md) +* [Amazon OpenSearch](../filter-plugin/logstash-filter-opensearch-guardium/mainREADME.md) +* [Amazon RDS for MariaDB](../filter-plugin/logstash-filter-mariadb-aws-guardium/mainREADME.md) +* [Amazon RDS for MySQL](../filter-plugin/logstash-filter-mysql-aws-guardium/mainREADME.md) +* [Amazon RDS for Postgres and Aurora Postgres](../filter-plugin/logstash-filter-postgres-guardium/mainREADME.md) +* [Amazon Redshift](../filter-plugin/logstash-filter-redshift-aws-guardium/mainREADME.md) * [Amazon S3](../filter-plugin/logstash-filter-s3-guardium/README.md) * [Apache Cassandra](../filter-plugin/logstash-filter-cassandra-guardium/README.md) * [Azure Apache Solr](../filter-plugin/logstash-filter-azure-apachesolr-guardium/README.md) * [Azure Cosmos](../filter-plugin/logstash-filter-cosmos-azure-guardium/README.md) -* [Azure Database for PostgreSQL](../filter-plugin/logstash-filter-azure-postgresql-guardium/README.md) -* [Azure MYSQL](../filter-plugin/logstash-filter-mysql-azure-guardium/README.md) -* [Azure SQL](../filter-plugin/logstash-filter-azure-sql-guardium/README.md) +* [Azure Database for PostgreSQL](../filter-plugin/logstash-filter-azure-postgresql-guardium/mainREADME.md) +* [Azure MYSQL](../filter-plugin/logstash-filter-mysql-azure-guardium/mainREADME.md) +* [Azure SQL](../filter-plugin/logstash-filter-azure-sql-guardium/mainREADME.md) * [Capella](../filter-plugin/logstash-filter-capella-guardium/README.md) * [Couchbase](../filter-plugin/logstash-filter-couchbasedb-guardium/README.md) * [CouchDB](../filter-plugin/logstash-filter-couchdb-guardium/README.md) -* [Databricks](../filter-plugin/logstash-filter-databricks-guardium/README.md) +* [Databricks](../filter-plugin/logstash-filter-databricks-guardium/mainREADME.md) * [DocumentDB](../filter-plugin/logstash-filter-documentdb-aws-guardium/README.md) * [Elasticsearch](../filter-plugin/logstash-filter-elasticsearch-guardium/README.md) -* [Google Cloud Apache Solr](../filter-plugin/logstash-filter-pubsub-apachesolr-guardium/README.md) -* [Google Cloud BigQuery](../filter-plugin/logstash-filter-pubsub-bigquery-guardium/README.md) -* [Google Cloud BigTable](../filter-plugin/logstash-filter-pubsub-bigtable-guardium/README.md) +* [Google Cloud Apache Solr](../filter-plugin/logstash-filter-pubsub-apachesolr-guardium/mainREADME.md) +* [Google Cloud BigQuery](../filter-plugin/logstash-filter-pubsub-bigquery-guardium/mainREADME.md) +* [Google Cloud BigTable](../filter-plugin/logstash-filter-pubsub-bigtable-guardium/mainREADME.md) * [Google Cloud Firebase](../filter-plugin/logstash-filter-pubsub-firebase-realtime-guardium/README.md) * [Google Cloud Firestore](../filter-plugin/logstash-filter-pubsub-firestore-guardium/README.md) * [Google Cloud MySQL](../filter-plugin/logstash-filter-pubsub-mysql-guardium/README.md) -* [Google Cloud PostgreSQL](../filter-plugin/logstash-filter-pubsub-postgresql-guardium/README.md) -* [Google Cloud Spanner](../filter-plugin/logstash-filter-pubsub-spanner-guardium/README.md) +* [Google Cloud PostgreSQL](../filter-plugin/logstash-filter-pubsub-postgresql-guardium/mainREADME.md) +* [Google Cloud Spanner](../filter-plugin/logstash-filter-pubsub-spanner-guardium/mainREADME.md) * [Greenplum](../filter-plugin/logstash-filter-onPremGreenplumdb-guardium/README.md) * [HDFS](../filter-plugin/logstash-filter-hdfs-guardium/README.md) * [IBM Cloud PostgreSQL](../filter-plugin/logstash-filter-postgres-ibmcloud-guardium/README.md) * [Intersystems IRIS](../filter-plugin/logstash-filter-intersystems-iris-guardium/README.md) * [MariaDB](../filter-plugin/logstash-filter-mariadb-guardium/README.md) * [Microsoft SQL Server](../filter-plugin/logstash-filter-mssql-guardium/MssqlOverJdbcPackage/README.md) +* [Milvus](../filter-plugin/logstash-filter-milvus-guardium/README.md) * [MongoDB](../filter-plugin/logstash-filter-mongodb-guardium/README.md) * [MySQL](../filter-plugin/logstash-filter-mysql-guardium/README.md) * [MySQL Percona](../filter-plugin/logstash-filter-mysql-percona-guardium/README.md) * [Neo4j](../filter-plugin/logstash-filter-neo4j-guardium/README.md) * [Neptune](../filter-plugin/logstash-filter-neptune-aws-guardium/README.md) +* [Oracle RDS Over CloudWatch Kafka Connect](KafkaBasedUCs/OracleRDSOverCloudwatchKafkaConnect.md) * [Oracle](../filter-plugin/logstash-filter-oua-guardium/README.md) * [PostgreSQL](../filter-plugin/logstash-filter-onPremPostgres-guardium/README.md) * [ProgressDB](../filter-plugin/logstash-filter-progressdb-guardium/README.md) * [SAP HANA](../filter-plugin/logstash-filter-saphana-guardium/README.md) * [ScyllaDB](../filter-plugin/logstash-filter-scylldb-guardium/README.md) +* [Singlestore](../filter-plugin/logstash-filter-singlestore-guardium/README.md) * [Snowflake](../filter-plugin/logstash-filter-snowflake-guardium/README.md) * [Teradata](../filter-plugin/logstash-filter-teradatadb-guardium/README.md) * [Trino](../filter-plugin/logstash-filter-trino-guardium/README.md) * [Yugabyte](../filter-plugin/logstash-filter-yugabyte-guardium/README.md) +* [Sybase](../docs/KafkaBasedUCs/SybaseJDBCKafkaConnect.md) +* [WatsonX](../docs/KafkaBasedUCs/WatsonXKafkaConnect.md) ## Developing Plug-ins Users can develop their own universal connector plug-ins, if needed, and contribute them back to the open source project, if desired. diff --git a/docs/general topics/sample_data_sources_configurations.md b/docs/general topics/sample_data_sources_configurations.md index 4732757bb..1cff7abe6 100644 --- a/docs/general topics/sample_data_sources_configurations.md +++ b/docs/general topics/sample_data_sources_configurations.md @@ -175,7 +175,8 @@ First, configure the MongoDB native audit logs so that they can be parsed by Gua ``` filebeat.inputs - - type: log + - type: filestream + - id: enabled: **true** paths: - **/var/log/mongodb/auditLog.json** @@ -194,7 +195,8 @@ First, configure the MongoDB native audit logs so that they can be parsed by Gua # Each -is an input. Most options can be set at the input level, so # you can use different inputs for various configurations. # Below are the input specific configurations. - -type: log + -type: filestream + -id: # Change to true to enable this input configuration. enabled: true # Paths that should be crawled and fetched. Glob based paths. diff --git a/docs/template-logstash-filter-guardium/CHANGELOG.md b/docs/template-logstash-filter-guardium/CHANGELOG.md new file mode 100644 index 000000000..214c9d7b7 --- /dev/null +++ b/docs/template-logstash-filter-guardium/CHANGELOG.md @@ -0,0 +1,49 @@ +# Changelog +Notable changes will be documented in this file. + +## [0.0.1] - YYYY-MM-DD +### Added +- Initial template creation +- Basic filter plugin structure +- Parser implementation template +- Unit test templates +- Documentation and getting started guide + +### Changed +- N/A + +### Fixed +- N/A + +### Removed +- N/A + +--- + +## Template Usage Instructions + +When using this template, update this changelog with your actual changes: + +1. Replace YYYY-MM-DD with actual dates +2. Document all changes in appropriate sections (Added, Changed, Fixed, Removed) +3. Follow semantic versioning (MAJOR.MINOR.PATCH) +4. Keep the most recent version at the top + +Example entry: +``` +## [1.0.0] - 2024-01-15 +### Added +- Support for SELECT queries +- Support for INSERT operations +- Error handling for connection failures + +### Changed +- Improved performance for large result sets +- Updated dependencies to latest versions + +### Fixed +- Fixed issue with timestamp parsing +- Corrected IP address extraction logic + +### Removed +- Deprecated legacy configuration options \ No newline at end of file diff --git a/docs/template-logstash-filter-guardium/GETTING_STARTED.md b/docs/template-logstash-filter-guardium/GETTING_STARTED.md new file mode 100644 index 000000000..20578bbbd --- /dev/null +++ b/docs/template-logstash-filter-guardium/GETTING_STARTED.md @@ -0,0 +1,379 @@ +# Getting Started - Creating Your Custom Logstash Filter Plugin for Guardium + +> **Important**: This template is stored in `docs/template-logstash-filter-guardium` for reference. Before you begin, copy it to the `filter-plugin/` directory where it can access the build system and shared resources: +> ```bash +> # From the universal-connectors root directory +> cp -r docs/template-logstash-filter-guardium filter-plugin/logstash-filter-YOURDATASOURCE-guardium +> cd filter-plugin/logstash-filter-YOURDATASOURCE-guardium +> ``` + +# Getting Started - Creating Your Custom Logstash Filter Plugin for Guardium + +This template provides a starting point for creating your own custom Logstash filter plugin for IBM Security Guardium Universal Connector. + +## Overview + +This template is based on successful implementations like: +- Azure Cosmos DB filter plugin +- Google Cloud BigTable filter plugin + +It includes all the necessary structure, build configuration, and example code to help you create a filter plugin for your specific data source. + +## Prerequisites + +Before you begin, ensure you have: +- Java Development Kit (JDK) 8 or 11 +- Gradle (will be downloaded automatically via wrapper) +- Understanding of your data source's audit log format +- Access to the universal-connectors repository structure + +## Step-by-Step Guide + +### 1. Copy and Rename the Template + +```bash +# Navigate to the filter-plugin directory +cd /path/to/universal-connectors/filter-plugin + +# Copy the template +cp -r template-logstash-filter-guardium logstash-filter-YOURDATASOURCE-guardium + +# Replace YOURDATASOURCE with your actual data source name (e.g., mongodb, postgresql, etc.) +``` + +### 2. Update Project Names and Identifiers + +Replace all placeholders throughout the project: + +**Placeholders to replace:** +- `DATASOURCE_PLACEHOLDER` → Your data source name in lowercase (e.g., `mongodb`, `postgresql`) +- `DATASOURCE_NAME` → Your data source display name (e.g., `MongoDB`, `PostgreSQL`) +- `YOUR_PACKAGE_NAME` → Your Java package name (e.g., `mongodb`, `postgresql`) +- `YourFilterClass` → Your main filter class name (e.g., `MongoDbGuardiumFilter`) +- `your_filter_name` → Logstash plugin name (e.g., `mongodb_guardium_filter`) + +**Files to update:** +1. `settings.gradle` - Update `rootProject.name` +2. `build.gradle` - Update group, description, pluginInfo fields +3. All Java source files - Update package names and class names +4. `README.md` - Update all documentation +5. Directory names under `src/main/java/com/ibm/guardium/` + +### 3. Understand the Core Components + +#### A. Main Filter Class (`YourDataSourceGuardiumFilter.java`) + +This is the entry point for your plugin. It: +- Receives log events from Logstash +- Validates and filters relevant events +- Calls the Parser to convert events to Guardium records +- Returns processed events + +**Key responsibilities:** +```java +@LogstashPlugin(name = "your_datasource_guardium_filter") +public class YourDataSourceGuardiumFilter implements Filter { + // 1. Validate incoming events + // 2. Parse JSON/log format + // 3. Call Parser.parseRecord() + // 4. Handle exceptions + // 5. Tag and return events +} +``` + +#### B. Parser Class (`Parser.java`) + +Converts your data source's audit logs into Guardium Record format. It creates: +- **Record**: Top-level Guardium object +- **Accessor**: Who accessed the data (user, client info) +- **SessionLocator**: Network information (IPs, ports) +- **Data/Construct**: What was accessed (SQL/NoSQL queries, objects) +- **ExceptionRecord**: Error information (if applicable) +- **Time**: Timestamp information + +#### C. ApplicationConstants Class + +Define all constants used in your plugin: +```java +public class ApplicationConstants { + // Field names from your audit logs + public static final String TIMESTAMP = "timestamp"; + public static final String USER_NAME = "userName"; + + // Default values + public static final String DEFAULT_IP = "0.0.0.0"; + public static final String UNKNOWN_STRING = ""; + + // Error tags + public static final String LOGSTASH_TAG_JSON_PARSE_ERROR = "your_datasource_json_parse_error"; +} +``` + +### 4. Implement Your Parser Logic + +#### Step 4.1: Analyze Your Audit Logs + +Collect sample audit logs from your data source and identify: +- Timestamp format +- User/principal information +- Client IP address +- Database/schema/table names +- Query or operation performed +- Success/failure indicators +- Error messages + +#### Step 4.2: Map to Guardium Record Structure + +Create a mapping document: + +``` +Your Audit Log Field → Guardium Field +---------------------------------------- +timestamp → Time (millis, offset) +user_email → Accessor.dbUser +client_ip → SessionLocator.clientIp +database_name → Record.dbName +query_text → Data.Construct.fullSql +operation_type → Sentence.verb +table_name → SentenceObject.name +error_code → ExceptionRecord.exceptionTypeId +error_message → ExceptionRecord.description +``` + +#### Step 4.3: Implement parseRecord() Method + +```java +public static Record parseRecord(JsonObject input) { + Record record = new Record(); + + // 1. Extract and set time + record.setTime(parseTime(input)); + + // 2. Extract database information + record.setDbName(extractDatabaseName(input)); + + // 3. Parse accessor (user info) + record.setAccessor(parseAccessor(input)); + + // 4. Parse session locator (network info) + record.setSessionLocator(parseSessionLocator(input)); + + // 5. Check for errors + if (hasError(input)) { + record.setException(parseException(input)); + } else { + record.setData(parseData(input)); + } + + // 6. Set session ID and app user + record.setSessionId(extractSessionId(input)); + record.setAppUserName(extractAppUserName(input)); + + return record; +} +``` + +### 5. Write Unit Tests + +Create test cases for: +- Successful operations +- Failed operations +- Different operation types (SELECT, INSERT, UPDATE, DELETE, etc.) +- Edge cases (missing fields, malformed JSON) +- Different user types + +Example test structure: +```java +@Test +public void testSuccessfulQuery() { + String auditLog = "{ /* your sample log */ }"; + Event event = new org.logstash.Event(); + event.setField("message", auditLog); + + Collection results = filter.filter( + Collections.singletonList(event), + matchListener + ); + + assertEquals(1, results.size()); + assertNotNull(event.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); +} +``` + +### 6. Update Configuration Files + +#### build.gradle +- Update `group` to match your package +- Update `description` +- Update `pluginInfo.pluginName` +- Update `pluginInfo.pluginClass` +- Add any specific dependencies your plugin needs + +#### settings.gradle +- Update `rootProject.name` + +#### VERSION +- Start with version `0.0.1` + +### 7. Create Sample Configuration File + +Create a `.conf` file showing how to use your plugin: + +``` +input { + # Your input plugin configuration + # e.g., file, kafka, http, etc. +} + +filter { + if [type] == "your_datasource" { + your_datasource_guardium_filter {} + } +} + +output { + # Guardium output configuration +} +``` + +### 8. Write Documentation (README.md) + +Your README should include: +1. **Overview**: What data source and what it does +2. **Prerequisites**: Versions tested, environment requirements +3. **Data Source Configuration**: How to enable audit logging +4. **Viewing Audit Logs**: Where to find logs +5. **Limitations**: Known limitations and unsupported features +6. **Guardium Configuration**: How to install and configure in Guardium +7. **Supported Operations**: List of supported audit log types + +### 9. Build Your Plugin + +```bash +# Build the plugin +./gradlew gem + +# The gem file will be created in the project root +# logstash-filter-your_datasource_guardium_filter-X.X.X.gem +``` + +### 10. Test Your Plugin + +#### Local Testing +```bash +# Install in local Logstash +/path/to/logstash/bin/logstash-plugin install /path/to/your-plugin.gem + +# Run with your config +/path/to/logstash/bin/logstash -f your-config.conf +``` + +#### Unit Testing +```bash +./gradlew test +./gradlew jacocoTestReport # Generate coverage report +``` + +### 11. Package for Guardium + +Create the offline package structure: +``` +your-datasource-package/ +├── logstash-filter-your_datasource_guardium_filter.zip +├── your_datasource.conf +└── gi_templates.json (if using Guardium Insights) +``` + +## Common Patterns and Best Practices + +### 1. Error Handling +```java +try { + JsonObject inputJSON = new Gson().fromJson(messageString, JsonObject.class); + Record record = Parser.parseRecord(inputJSON); + // ... process record +} catch (Exception exception) { + log.error("Error parsing event", exception); + event.tag(LOGSTASH_TAG_JSON_PARSE_ERROR); +} +``` + +### 2. IP Address Handling +```java +SessionLocator sessionLocator = new SessionLocator(); +if (Util.isIPv6(clientIp)) { + sessionLocator.setIpv6(true); + sessionLocator.setClientIpv6(clientIp); + sessionLocator.setServerIpv6(DEFAULT_IPV6); +} else { + sessionLocator.setIpv6(false); + sessionLocator.setClientIp(clientIp); + sessionLocator.setServerIp(DEFAULT_IP); +} +``` + +### 3. Time Parsing +```java +public static Time parseTime(String dateString) { + ZonedDateTime date = ZonedDateTime.parse(dateString); + long millis = date.toInstant().toEpochMilli(); + int minOffset = date.getOffset().getTotalSeconds() / 60; + return new Time(millis, minOffset, 0); +} +``` + +### 4. Null Safety +```java +String value = jsonObject.has("field") && jsonObject.get("field") != null + ? jsonObject.get("field").getAsString() + : DEFAULT_VALUE; +``` + +### 5. Logging +```java +private static Logger log = LogManager.getLogger(YourClass.class); + +if (log.isDebugEnabled()) { + log.debug("Processing event: {}", event); +} +log.error("Error occurred", exception); +``` + +## Troubleshooting + +### Plugin Not Loading +- Check `@LogstashPlugin` annotation name matches `pluginInfo.pluginName` in build.gradle +- Verify package structure matches `group` in build.gradle +- Ensure all required dependencies are included + +### Events Not Being Processed +- Add debug logging to see what events are received +- Check event tags for error indicators +- Verify your filter condition in Logstash config + +### Build Failures +- Ensure `LOGSTASH_CORE_PATH` and `GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH` are set +- Check Java version compatibility +- Review dependency versions in `versions.yml` + +### Test Failures +- Verify test data matches your actual audit log format +- Check for timezone issues in time parsing +- Ensure all required fields are present in test data + +## Additional Resources + +- [Guardium Universal Connector Documentation](https://github.com/IBM/universal-connectors) +- [Logstash Plugin Development](https://www.elastic.co/guide/en/logstash/current/contributing-to-logstash.html) +- [Guardium Record Structure](https://github.com/IBM/universal-connectors/blob/main/common/src/main/java/com/ibm/guardium/universalconnector/commons/structures/Record.java) + +## Support + +For questions or issues: +1. Check existing filter plugins for similar implementations +2. Review the universal-connectors repository documentation +3. Contact the Guardium Universal Connector team + +## License + +This template is provided under the Apache 2.0 License. \ No newline at end of file diff --git a/docs/template-logstash-filter-guardium/LICENSE b/docs/template-logstash-filter-guardium/LICENSE new file mode 100644 index 000000000..d64569567 --- /dev/null +++ b/docs/template-logstash-filter-guardium/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/docs/template-logstash-filter-guardium/README.md b/docs/template-logstash-filter-guardium/README.md new file mode 100644 index 000000000..159fd5ede --- /dev/null +++ b/docs/template-logstash-filter-guardium/README.md @@ -0,0 +1,644 @@ +# [DATASOURCE_NAME]-Guardium Logstash Filter Plugin Template + +> **Important Note**: This template is located in the `docs/` directory for reference. To use this template and build your plugin, you must copy it to the `filter-plugin/` directory: +> ```bash +> cp -r docs/template-logstash-filter-guardium filter-plugin/logstash-filter-YOURDATASOURCE-guardium +> cd filter-plugin/logstash-filter-YOURDATASOURCE-guardium +> ``` +> The build system requires the plugin to be under `filter-plugin/` to access shared resources and dependencies. + +# [DATASOURCE_NAME]-Guardium Logstash Filter Plugin Template + +### Meet [DATASOURCE_NAME] +* Tested versions: [VERSION_NUMBER] +* Environment: [ENVIRONMENT] (e.g., On-Premise, AWS, Azure, GCP) +* Supported inputs: [INPUT_TYPE] (e.g., Filebeat, Kafka, HTTP, Pub/Sub) +* Supported Guardium versions: + * Guardium Data Protection: [MIN_VERSION] and above + * Guardium Insights: [SUPPORTED_VERSION] (if applicable) + +This is a [Logstash](https://github.com/elastic/logstash) filter plug-in for the universal connector that is featured in IBM Security Guardium. It parses events and messages from the [DATASOURCE_NAME] audit log into a [Guardium record](https://github.com/IBM/universal-connectors/blob/main/common/src/main/java/com/ibm/guardium/universalconnector/commons/structures/Record.java) instance (which is a standard structure made out of several parts). The information is then sent over to Guardium. Guardium records include the accessor (the person who tried to access the data), the session, data, and exceptions. If there are no errors, the data contains details about the query "construct". The construct details the main action (verb) and collections (objects) involved. + +The plug-in is free and open-source (Apache 2.0). It can be used as a starting point to develop additional filter plug-ins for the Guardium universal connector. + +## 1. Configuring the [DATASOURCE_NAME] Service + +### Prerequisites +[List any prerequisites needed before configuration] + +### Procedure: +1. [Step-by-step instructions for setting up your data source] +2. [Include screenshots or code examples where helpful] +3. [Be specific about versions, settings, and configurations] + +## 2. Enabling Audit Logging + +### Enable Audit Logs +1. [Detailed steps to enable audit logging in your data source] +2. [Specify what types of operations are logged] +3. [Include any performance considerations] + +**Note:** [Any important notes about audit logging behavior] + +### Audit Log Configuration +[Describe the audit log configuration options] +- **Option 1**: [Description] +- **Option 2**: [Description] + +Example configuration: +``` +[Configuration example in appropriate format - JSON, YAML, SQL, etc.] +``` + +## 3. Viewing the Audit Logs + +### Accessing Logs +[Describe how to access and view the audit logs] + +1. [Step 1] +2. [Step 2] +3. [Step 3] + +### Log Format +The audit logs are in [FORMAT] format. Example: +```json +{ + "timestamp": "2024-01-01T12:00:00Z", + "user": "admin@example.com", + "operation": "SELECT", + "database": "mydb", + "table": "users", + "query": "SELECT * FROM users WHERE id = 1", + "status": "SUCCESS" +} +``` + +### Supported Audit Log Types +* [LOG_TYPE_1] - [Description] +* [LOG_TYPE_2] - [Description] +* [LOG_TYPE_3] - [Description] + +## 6. Limitations + +1. The following important fields couldn't be mapped with [DATASOURCE_NAME] audit logs: + - **Source Program**: [Explanation - e.g., "Not available in audit logs, left blank"] + - **Server IP**: [Explanation - e.g., "Not provided, defaults to 0.0.0.0"] + - **Client HostName**: [Explanation] + - **OS User**: [Explanation] + - [Add other unmapped fields] + +2. [Specific limitation related to your data source] +3. [Another limitation] + +## 4. Query Parsing and Sniffer Parsers + +Parsing query statements is one of the most complicated parts of parsing which could hurt performance. There are a list of available high-performance sniffer parsers that can be used in universal connectors to improve performance. Here is a list of the available ones. This list could get updated if more sniffer parsers get created. + +### Available Sniffer Parser Languages: + +| Language Mark | Database/Technology | +|---------------|---------------------| +| -tg | TigerGraph | +| -mi | Milvus | +| -b | BigQuery | +| -B | BigQuery SQL | +| -ca | Cassandra | +| -C | Couchbase | +| -c | Cypher | +| -d | DB2 | +| -e | ElsSql | +| -hi | Hive | +| -ha | Hana | +| -i | Informix | +| -im | Impala | +| -m | MySql | +| -x | MySql X | +| -M | MemSql | +| -mo | MongoDB | +| -N | N1ql | +| -n | Neo4j | +| -o | Oracle | +| -os | OpenSearch | +| -p | Postgres | +| -r | Redis | +| -s | Sybase | +| -S | Snowflake | +| -t | TSql/MsSql | +| -td | Teradata | +| -ad | AWS DynamoDB | +| -ae | AWS Elastic Search | +| -as | AWS S3 | +| -ck | Cockroach | +| -nz | Netezza | +| -xq | Xquery | +| -cd | CouchDB | +| -ns | NoSQL | + +### Using Sniffer Parsers + +If the queries in your database are compatible with any of the above parsers, you can use the corresponding parser in your universal connector. + +#### With Sniffer Parser: + +In the **Accessor** field: +- Set `language` to the database mark (e.g., `-m` for MySQL, `-p` for Postgres) +- Set `dataType` to `"TEXT"` + +In the **Data** field: +- Populate `originalSqlCommand` with the query text + +Example: +```java +accessor.setLanguage("-p"); // For PostgreSQL +accessor.setDataType(Accessor.DATA_TYPE_GUARDIUM_SHOULD_PARSE_SQL); +data.setOriginalSqlCommand(queryText); +``` + +#### Without Sniffer Parser: + +In the **Accessor** field: +- Set `language` to `FREE_TEXT` +- Set `dataType` to `"CONSTRUCT"` +- Set `serverType` to the database type + +In the **Data** field: +- Populate the `construct` object: + - **Verb**: The action (for non-SQL DBs) or SQL operation (SELECT, INSERT, DELETE, UPDATE, etc.) + - **Object**: May have more than one object (DB, table, index, etc.). Set the type for each one. + +Example: +```java +accessor.setLanguage(Accessor.LANGUAGE_FREE_TEXT_STRING); +accessor.setDataType(Accessor.DATA_TYPE_GUARDIUM_SHOULD_NOT_PARSE_SQL); +accessor.setServerType("YourDatabaseType"); + +Construct construct = new Construct(); +Sentence sentence = new Sentence("SELECT"); +SentenceObject tableObject = new SentenceObject("users"); +tableObject.setType("table"); +sentence.getObjects().add(tableObject); +construct.sentences.add(sentence); +data.setConstruct(construct); +``` + +4. [Performance considerations] +5. [Known issues or workarounds] + +## 5. Configuring the [DATASOURCE_NAME] Filter in Guardium + +The Guardium universal connector is the Guardium entry point for native audit logs. The Guardium universal connector identifies and parses the received events, and converts them to a standard Guardium format. The output of the Guardium universal connector is forwarded to the Guardium sniffer on the collector, for policy and auditing enforcements. Configure Guardium to read the native audit logs by customizing the [DATASOURCE_NAME] template. + +### Before you begin +* Configure the policies you require. See [policies](https://github.com/IBM/universal-connectors/#policies) for more information. +* You must have permission for the S-Tap Management role. The admin user includes this role by default. +* Download the [guardium_logstash-offline-plugin-[DATASOURCE_PLACEHOLDER].zip](./package/guardium_logstash-offline-plugin-[DATASOURCE_PLACEHOLDER].zip) plug-in. (Do not unzip the offline-package file throughout the procedure). + +## 5. Important Implementation Notes + +### Critical Guidelines + +1. **Missing Fields in Audit Logs** + - If a field is not available in the audit logs and you leave it empty, you **MUST** document this in the Limitations section of this README. + - Example: "Client HostName: Not available in audit logs, left blank" + +2. **Session ID Usage** + - `session_id` in Universal Connectors has a specific meaning. It creates a grouping based on user name, database name, and other fields. + - Setting this from a wrong value could result in incorrect database names or user names in reports. + - **Solution**: Do not set `session_id` at all if you are not sure about the correct value. + +3. **Guard Record Requirements** + - **NEVER** return `NULL` as a Guard Record + - In Guard Record, you must have **either** Data **or** Exception (never both, never neither) + +4. **Data and Construct Requirements** + - If you have Data and you **DO NOT** use sniffer for parsing SQL statements: + - Construct **CANNOT** be NULL (will cause NullPointerException) + - You must create the construct object with verb and objects + + - If sniffer parses the SQL statement (language is database mark and dataType is TEXT): + - You only need to set `originalSqlCommand` on the Data field + - Sniffer will handle construct creation + + - Otherwise, it is your responsibility to create the construct object (verb and objects) + +5. **S-TAP Identification** + - S-TAP is identified by: + - If `serverHostName` is available: `serverHostName:serverPort` + - Otherwise: `serverIp:serverPort` + - **Note**: Default values are still being standardized. Use either `""` or `"N.A."` for serverHostName and `"0.0.0.0"` for serverIp + +6. **Unit Test Security** + - **Always** check your unit tests to ensure they do not contain any private or sensitive information + - Remove any real usernames, passwords, IP addresses, or company-specific data + - Use placeholder values like "testuser", "testdb", "192.168.1.1", etc. + +### Implementation Checklist + +Before submitting your plugin, verify: + +- [ ] All missing fields are documented in Limitations section +- [ ] Session ID is only set when you have a reliable value +- [ ] Parser never returns NULL as Guard Record +- [ ] Data always has either Construct (if not using sniffer) or originalSqlCommand (if using sniffer) +- [ ] Exception is set for error cases +- [ ] S-TAP identification uses correct serverHostName/serverIp and port +- [ ] Unit tests contain no sensitive information +- [ ] All TODO comments in template code have been addressed +- [ ] README is updated with data source-specific information + +### Example: Proper Data/Exception Handling + +```java +public static Record parseRecord(JsonObject input) { + Record record = new Record(); + + // ... set other fields ... + + // Check for errors + if (isError(input)) { + // Set exception for error cases + ExceptionRecord exception = new ExceptionRecord(); + exception.setExceptionTypeId("SQL_ERROR"); + exception.setDescription(getErrorMessage(input)); + exception.setSqlString(getQueryIfAvailable(input)); + record.setException(exception); + } else { + // Set data for success cases + Data data = new Data(); + + if (useSnifferParser()) { + // Using sniffer - only set originalSqlCommand + data.setOriginalSqlCommand(getQuery(input)); + } else { + // Not using sniffer - must create construct + Construct construct = new Construct(); + Sentence sentence = new Sentence(getVerb(input)); + // Add objects to sentence + sentence.getObjects().add(createSentenceObject(input)); + construct.sentences.add(sentence); + construct.setFullSql(getQuery(input)); + data.setConstruct(construct); // MUST NOT be null + } + + record.setData(data); + } + + return record; // NEVER return null +} +``` + +* Download the plug-in filter configuration file [[DATASOURCE_PLACEHOLDER].conf](.//[DATASOURCE_PLACEHOLDER].conf). + +### Procedure +1. On the collector, go to **Setup** > **Tools and Views** > **Configure Universal Connector**. +2. Enable the universal connector if it is disabled. +3. Click **Upload File** and select the offline [guardium_logstash-offline-plugin-[DATASOURCE_PLACEHOLDER].zip](./package/guardium_logstash-offline-plugin-[DATASOURCE_PLACEHOLDER].zip) plug-in. After it is uploaded, click **OK**. +4. Click the Plus sign to open the Connector Configuration dialog box. +5. Type a name in the **Connector name** field. +6. Update the input section to add the details from the [[DATASOURCE_PLACEHOLDER].conf](.//[DATASOURCE_PLACEHOLDER].conf) file's input part, omitting the keyword "input{" at the beginning and its corresponding "}" at the end. +7. Update the filter section to add the details from the [[DATASOURCE_PLACEHOLDER].conf](.//[DATASOURCE_PLACEHOLDER].conf) file's filter part, omitting the keyword "filter{" at the beginning and its corresponding "}" at the end. +8. The **type** fields should match in the input and filter configuration sections. This field should be unique for every individual connector added. +9. Click **Save**. Guardium validates the new connector and displays it in the Configure Universal Connector page. +10. After the offline plug-in is installed and the configuration is uploaded and saved in the Guardium machine, restart the Universal Connector using the **Disable/Enable** button. + +## 8. Supported Operations + +The following operations are supported by this filter plugin: + +### Data Manipulation +- **SELECT/READ**: Query operations +- **INSERT/CREATE**: Insert operations +- **UPDATE/MODIFY**: Update operations +- **DELETE/DROP**: Delete operations + +### Data Definition +- **CREATE DATABASE**: Database creation +- **CREATE TABLE**: Table/Collection creation +- **ALTER TABLE**: Schema modifications +- **DROP TABLE**: Table/Collection deletion + +### Access Control +- **GRANT**: Permission grants +- **REVOKE**: Permission revocations +- **CREATE USER**: User creation +- **DROP USER**: User deletion + +[Customize this list based on your data source's capabilities] + +## 9. Input Plugin Configuration + +The input plugin determines how audit logs are collected and sent to Logstash. Choose the appropriate input plugin based on your data source's log delivery method. + +### Common Input Plugin Options + +#### Option 1: File Input (Reading from Log Files) +Use when audit logs are written to files on disk. + +```ruby +input { + file { + path => "/var/log/[datasource]/audit*.log" + start_position => "beginning" + sincedb_path => "/var/lib/logstash/sincedb_[datasource]" + type => "[DATASOURCE_PLACEHOLDER]" + codec => json # Use "plain" for text logs + tags => ["[DATASOURCE_NAME]"] + } +} +``` + +**Key Parameters:** +- `path`: Location of audit log files (supports wildcards) +- `start_position`: "beginning" or "end" (where to start reading) +- `sincedb_path`: Tracks file reading position (for resuming after restart) +- `codec`: "json" for JSON logs, "plain" for text logs + +#### Option 2: Kafka Input (Message Queue) +Use when audit logs are published to Kafka topics. + +```ruby +input { + kafka { + bootstrap_servers => "kafka1:9092,kafka2:9092" + topics => ["[datasource]-audit-logs"] + group_id => "logstash-[datasource]-consumer" + consumer_threads => 4 + type => "[DATASOURCE_PLACEHOLDER]" + codec => json + auto_offset_reset => "earliest" + } +} +``` + +**Key Parameters:** +- `bootstrap_servers`: Kafka broker addresses +- `topics`: Kafka topics to consume from +- `group_id`: Consumer group identifier +- `consumer_threads`: Number of parallel consumers + +#### Option 3: HTTP Input (Webhook/API) +Use when audit logs are pushed via HTTP/HTTPS. + +```ruby +input { + http { + port => 8080 + type => "[DATASOURCE_PLACEHOLDER]" + codec => json + ssl => true + ssl_certificate => "/path/to/cert.pem" + ssl_key => "/path/to/key.pem" + additional_codecs => { "application/json" => "json" } + } +} +``` + +**Key Parameters:** +- `port`: Port to listen on +- `ssl`: Enable HTTPS +- `ssl_certificate` / `ssl_key`: SSL/TLS certificates + +#### Option 4: Beats Input (Filebeat/Metricbeat) +Use when using Elastic Beats to ship logs. + +```ruby +input { + beats { + port => 5044 + type => "[DATASOURCE_PLACEHOLDER]" + ssl => true + ssl_certificate => "/path/to/cert.pem" + ssl_key => "/path/to/key.pem" + } +} +``` + +**Key Parameters:** +- `port`: Port for Beats to connect to (default: 5044) +- `ssl`: Enable encrypted communication + +#### Option 5: Cloud-Specific Inputs + +**AWS CloudWatch Logs:** +```ruby +input { + cloudwatch_logs { + log_group => "/aws/[datasource]/audit" + region => "us-east-1" + type => "[DATASOURCE_PLACEHOLDER]" + access_key_id => "${AWS_ACCESS_KEY}" + secret_access_key => "${AWS_SECRET_KEY}" + } +} +``` + +**Azure Event Hub:** +```ruby +input { + azure_event_hubs { + event_hub_connections => ["Endpoint=sb://...;EntityPath=..."] + threads => 4 + decorate_events => true + consumer_group => "$Default" + storage_connection => "DefaultEndpointsProtocol=https;..." + type => "[DATASOURCE_PLACEHOLDER]" + } +} +``` + +**GCP Pub/Sub:** +```ruby +input { + google_pubsub { + project_id => "your-project-id" + topic => "[datasource]-audit-logs" + subscription => "logstash-subscription" + json_key_file => "/path/to/service-account-key.json" + type => "[DATASOURCE_PLACEHOLDER]" + codec => json + } +} +``` + +### Filter Plugin Configuration + +The filter plugin processes the audit logs and converts them to Guardium format. + +**Basic Configuration:** +```ruby +filter { + if [type] == "[DATASOURCE_PLACEHOLDER]" { + [DATASOURCE_PLACEHOLDER]_guardium_filter { + source => "message" + } + } +} +``` + +**With Pre-processing:** +```ruby +filter { + if [type] == "[DATASOURCE_PLACEHOLDER]" { + # Optional: Parse timestamp if needed + date { + match => ["timestamp", "ISO8601", "yyyy-MM-dd HH:mm:ss"] + target => "@timestamp" + } + + # Optional: Add custom fields + mutate { + add_field => { + "environment" => "production" + "data_center" => "dc1" + } + } + + # Main filter plugin + [DATASOURCE_PLACEHOLDER]_guardium_filter { + source => "message" + } + } +} +``` + +### Complete Configuration Example + +See [datasource_sample.conf](datasource_sample.conf) for a complete configuration file with: +- Multiple input plugin options (commented) +- Filter configuration with pre/post-processing +- Output configuration for Guardium +- Error handling examples +- Performance tuning options + +### Configuration Best Practices + +1. **Type Field**: Always set a unique `type` field in the input to identify logs from this data source +2. **Codec Selection**: Use `json` codec for JSON logs, `plain` for text logs +3. **Error Handling**: Add error tags and separate outputs for failed events +4. **Performance**: + - Adjust `pipeline.workers` for parallel processing + - Set appropriate `pipeline.batch.size` (default: 125) + - Configure persistent queues for reliability +5. **Security**: + - Use SSL/TLS for network inputs + - Store credentials in environment variables or keystore + - Implement proper access controls +6. **Monitoring**: + - Enable Logstash monitoring + - Set up alerts for processing failures + - Monitor queue sizes and throughput + +### Testing Your Configuration + +```bash +# Test configuration syntax +/path/to/logstash/bin/logstash -f your-config.conf --config.test_and_exit + +# Run with debug output +/path/to/logstash/bin/logstash -f your-config.conf --log.level=debug + +# Run in foreground for testing +/path/to/logstash/bin/logstash -f your-config.conf +``` + +## 10. Troubleshooting + +### Common Issues + +#### Issue 1: Events not being processed +**Symptoms**: No Guardium records are generated +**Solution**: +- Check that the `type` field matches in input and filter sections +- Verify audit logs are in the expected format +- Check Logstash logs for parsing errors + +#### Issue 2: Missing fields in Guardium +**Symptoms**: Some fields appear as "UNKNOWN" or empty +**Solution**: +- Verify audit logging is configured correctly +- Check if the data source provides the missing information +- Review the limitations section + +#### Issue 3: Performance issues +**Symptoms**: High CPU or memory usage +**Solution**: +- Adjust Logstash heap size +- Consider filtering logs before processing +- Review batch size settings + +### Debug Mode +To enable debug logging, add this to your Logstash configuration: +``` +filter { + if [type] == "[DATASOURCE_PLACEHOLDER]" { + mutate { + add_field => { "[@metadata][debug]" => "true" } + } + [DATASOURCE_PLACEHOLDER]_guardium_filter {} + } +} +``` + +## 11. Development and Testing + +### Building from Source +```bash +# Clone the repository +git clone https://github.com/IBM/universal-connectors.git +cd universal-connectors/filter-plugin/logstash-filter-[DATASOURCE_PLACEHOLDER]-guardium + +# Build the plugin +./gradlew gem + +# The gem file will be created in the project root +``` + +### Running Tests +```bash +# Run all tests +./gradlew test + +# Generate coverage report +./gradlew jacocoTestReport + +# View coverage report +open build/reports/jacoco/index.html +``` + +### Local Testing +```bash +# Install the plugin locally +/path/to/logstash/bin/logstash-plugin install /path/to/logstash-filter-[DATASOURCE_PLACEHOLDER]_guardium_filter-X.X.X.gem + +# Run Logstash with your configuration +/path/to/logstash/bin/logstash -f [DATASOURCE_PLACEHOLDER].conf +``` + +## 12. Contributing + +Contributions are welcome! Please: +1. Fork the repository +2. Create a feature branch +3. Make your changes +4. Add tests for new functionality +5. Ensure all tests pass +6. Submit a pull request + +## 13. License + +This plugin is licensed under the Apache 2.0 License. See [LICENSE](LICENSE) for details. + +## 14. Support + +For issues, questions, or contributions: +- GitHub Issues: [Link to issues page] +- Documentation: [Link to additional documentation] +- Universal Connectors: https://github.com/IBM/universal-connectors + +## 15. Version History + +See [CHANGELOG.md](CHANGELOG.md) for version history and release notes. + +--- + +**Note**: This is a template. Replace all placeholders (marked with [BRACKETS] or DATASOURCE_PLACEHOLDER) with your actual data source information. \ No newline at end of file diff --git a/docs/template-logstash-filter-guardium/TEMPLATE_SUMMARY.md b/docs/template-logstash-filter-guardium/TEMPLATE_SUMMARY.md new file mode 100644 index 000000000..47cc4ab63 --- /dev/null +++ b/docs/template-logstash-filter-guardium/TEMPLATE_SUMMARY.md @@ -0,0 +1,237 @@ +# Template Summary - Logstash Filter Plugin for Guardium + +## Overview + +This template provides a complete starting point for creating custom Logstash filter plugins for IBM Security Guardium Universal Connector. It was created by analyzing successful implementations from: +- Azure Cosmos DB filter plugin +- Google Cloud BigTable filter plugin + +## Template Structure + +``` +template-logstash-filter-guardium/ +├── GETTING_STARTED.md # Comprehensive guide for using this template +├── README.md # Template README with placeholders +├── TEMPLATE_SUMMARY.md # This file +├── LICENSE # Apache 2.0 License +├── VERSION # Version file (starts at 0.0.1) +├── CHANGELOG.md # Changelog template +├── build.gradle # Gradle build configuration +├── settings.gradle # Gradle settings +├── datasource_sample.conf # Sample Logstash configuration +├── gradle/ +│ └── wrapper/ # Gradle wrapper files (copy from samples) +├── src/ +│ ├── main/ +│ │ └── java/ +│ │ └── com/ +│ │ └── ibm/ +│ │ └── guardium/ +│ │ └── DATASOURCE_PLACEHOLDER/ +│ │ ├── DataSourceGuardiumFilter.java # Main filter class +│ │ ├── Parser.java # Parser for both JSON and text logs +│ │ └── ApplicationConstants.java # Constants +│ └── test/ +│ ├── java/ +│ │ └── com/ +│ │ └── ibm/ +│ │ └── guardium/ +│ │ └── DATASOURCE_PLACEHOLDER/ +│ │ └── DataSourceGuardiumFilterTest.java # Unit tests +│ └── resources/ # Test resources (add sample logs here) +``` + +## Key Features + +### 1. Dual Format Support +The template supports both JSON and text-based log formats: +- **JSON logs**: Direct parsing using Gson +- **Text logs**: Regex-based extraction +- Flexible architecture allows mixing both formats + +### 2. Complete Guardium Record Mapping +Implements all required Guardium structures: +- **Record**: Top-level container +- **Accessor**: User and system information +- **SessionLocator**: Network details (IPv4/IPv6 support) +- **Data/Construct**: Query and operation details +- **ExceptionRecord**: Error handling +- **Time**: Timestamp with timezone support + +### 3. Comprehensive Documentation +- **GETTING_STARTED.md**: Step-by-step guide (368 lines) +- **README.md**: Template with all standard sections +- **Inline comments**: Extensive TODO markers and examples +- **Configuration examples**: Multiple input/output scenarios + +### 4. Testing Framework +- Unit test template with multiple test cases +- Success and error scenarios +- Edge case handling +- Test helper classes included + +### 5. Build System +- Gradle-based build system +- Shadow JAR for dependencies +- JaCoCo for code coverage +- Ruby gem generation for Logstash + +## Placeholders to Replace + +When using this template, replace the following placeholders throughout all files: + +| Placeholder | Description | Example | +|-------------|-------------|---------| +| `DATASOURCE_PLACEHOLDER` | Lowercase data source identifier | `mongodb`, `postgresql` | +| `DATASOURCE_NAME` | Display name of data source | `MongoDB`, `PostgreSQL` | +| `DataSourceGuardiumFilter` | Main filter class name | `MongoDbGuardiumFilter` | +| `your_filter_name` | Logstash plugin name | `mongodb_guardium_filter` | +| `[DATASOURCE_NAME]` | Bracketed display name in docs | `[MongoDB]` | +| `YOUR_PACKAGE_NAME` | Java package name | `mongodb` | + +## Quick Start Checklist + +- [ ] Copy template to new directory: `logstash-filter-YOURDATASOURCE-guardium` +- [ ] Replace all placeholders (use find & replace) +- [ ] Rename directory: `DATASOURCE_PLACEHOLDER` → your data source name +- [ ] Update `settings.gradle` with correct project name +- [ ] Update `build.gradle` with correct group, plugin info +- [ ] Collect sample audit logs from your data source +- [ ] Implement parsing logic in `Parser.java` +- [ ] Update `ApplicationConstants.java` with your field names +- [ ] Implement validation in `DataSourceGuardiumFilter.java` +- [ ] Write unit tests with real log samples +- [ ] Update `README.md` with data source-specific information +- [ ] Test build: `./gradlew gem` +- [ ] Test locally with Logstash +- [ ] Create package for Guardium deployment + +## Implementation Patterns + +### Pattern 1: JSON-Only Data Source +If your data source only produces JSON logs: +1. Remove text parsing methods from `Parser.java` +2. Simplify `parseRecord()` to only handle JsonObject +3. Focus on JSON field extraction + +### Pattern 2: Text-Only Data Source +If your data source only produces text logs: +1. Remove JSON parsing methods from `Parser.java` +2. Define regex patterns in `ApplicationConstants.java` +3. Implement comprehensive regex extraction + +### Pattern 3: Mixed Format Data Source +If your data source produces both formats: +1. Keep both parsing methods +2. Add format detection logic in filter +3. Route to appropriate parser method + +### Pattern 4: Complex Query Parsing +If you need to parse SQL/NoSQL queries: +1. Consider using parser libraries (Parboiled, JSqlParser) +2. Add dependencies to `build.gradle` +3. Implement query parsing in separate class +4. Extract verbs and objects for Sentence/SentenceObject + +## Common Customizations + +### Adding Custom Fields +```java +// In ApplicationConstants.java +public static final String CUSTOM_FIELD = "customField"; + +// In Parser.java +String customValue = inputJSON.get(ApplicationConstants.CUSTOM_FIELD).getAsString(); +``` + +### Adding Error Codes +```java +// Create ErrorCodes.java +public enum ErrorCodes { + SUCCESS(0), + AUTH_FAILED(401), + NOT_FOUND(404); + + private final int code; + // ... implementation +} +``` + +### Adding Utility Methods +```java +// Create CommonUtils.java +public class CommonUtils { + public static boolean isValidJSON(String str) { + // ... implementation + } +} +``` + +## Testing Strategy + +### Unit Tests +- Test each operation type (SELECT, INSERT, UPDATE, DELETE, etc.) +- Test success and failure scenarios +- Test edge cases (null, empty, malformed) +- Test different user types +- Test IPv4 and IPv6 addresses + +### Integration Tests +- Test with actual Logstash +- Test with sample log files +- Test with Guardium collector +- Verify reports in Guardium UI + +### Performance Tests +- Test with high volume logs +- Monitor memory usage +- Check CPU utilization +- Measure throughput + +## Deployment + +### Building the Plugin +```bash +./gradlew clean gem +``` + +### Installing Locally +```bash +/path/to/logstash/bin/logstash-plugin install /path/to/plugin.gem +``` + +### Creating Guardium Package +``` +your-datasource-package/ +├── logstash-filter-your_datasource_guardium_filter.zip +├── your_datasource.conf +└── gi_templates.json (optional, for Guardium Insights) +``` + +## Support and Resources + +- **Universal Connectors Repo**: https://github.com/IBM/universal-connectors +- **Guardium Record Structure**: Check common module in repo +- **Sample Plugins**: Review existing filter plugins for patterns +- **Logstash Plugin Development**: https://www.elastic.co/guide/en/logstash/current/contributing-to-logstash.html + +## Version History + +- **0.0.1** (Template Creation): Initial template with dual format support + +## Contributing + +When improving this template: +1. Test changes with multiple data source types +2. Update documentation +3. Add examples for new patterns +4. Maintain backward compatibility +5. Update this summary document + +## License + +Apache 2.0 License - See LICENSE file for details + +--- + +**Note**: This template is designed to be flexible and comprehensive. Not all features may be needed for your specific data source. Feel free to remove unused code and simplify where appropriate. \ No newline at end of file diff --git a/docs/template-logstash-filter-guardium/VERSION b/docs/template-logstash-filter-guardium/VERSION new file mode 100644 index 000000000..8a9ecc2ea --- /dev/null +++ b/docs/template-logstash-filter-guardium/VERSION @@ -0,0 +1 @@ +0.0.1 \ No newline at end of file diff --git a/docs/template-logstash-filter-guardium/build.gradle b/docs/template-logstash-filter-guardium/build.gradle new file mode 100644 index 000000000..32d8ff9fd --- /dev/null +++ b/docs/template-logstash-filter-guardium/build.gradle @@ -0,0 +1,174 @@ +import java.nio.file.Files +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING + +apply plugin: 'java' +def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + + +apply plugin: 'jacoco' + + +apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" + +// =========================================================================== +// plugin info +// =========================================================================== +group 'com.ibm.guardium.DATASOURCE_PLACEHOLDER' // must match the package of the main plugin class +version "${file("VERSION").text.trim()}" // read from required VERSION file +description = "[DATASOURCE_NAME]-Guardium filter" +pluginInfo.licenses = ['Apache-2.0'] // list of SPDX license IDs +pluginInfo.longDescription = "This gem is a Logstash [DATASOURCE_NAME] filter plugin required to be installed as part of IBM Security Guardium, Guardium Universal connector configuration. This gem is not a stand-alone program." +pluginInfo.authors = ['IBM'] +pluginInfo.email = [''] +pluginInfo.homepage = "https://github.com/IBM/universal-connectors" +pluginInfo.pluginType = "filter" +pluginInfo.pluginClass = "[YourFilterClass]" // e.g., "MongoDbGuardiumFilter" +pluginInfo.pluginName = "[your_filter_name]" // e.g., "mongodb_guardium_filter" - must match the @LogstashPlugin annotation in the main plugin class +// =========================================================================== + +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + + dependencies { + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' + classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' + } +} + + +repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } +} + +task copyDependencyLibs(type: Copy) { + into "dependenciesLib" + from configurations.compileClasspath + from configurations.runtimeClasspath + from configurations.testCompileClasspath + from configurations.testRuntimeClasspath +} + + +apply plugin: 'com.github.johnrengelman.shadow' + +shadowJar { + archiveClassifier = null + transform(com.github.jengelman.gradle.plugins.shadow.transformers.Log4j2PluginsCacheFileTransformer) +} + +dependencies { + // Core dependencies - required for all plugins + implementation group: 'commons-validator', name: 'commons-validator', version: versions.dependencies.commonsValidator + implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: versions.dependencies.log4jCore + implementation 'org.apache.commons:commons-lang3:' + versions.dependencies.commonsLang + implementation 'com.google.code.gson:gson:' + versions.dependencies.gson + implementation group: 'commons-beanutils', name: 'commons-beanutils', version: versions.dependencies.commonsBeanutils + implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "build/libs/logstash-core-*.*.*.jar") + implementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "common-*.*.*.jar") + + // Optional dependencies - uncomment if needed for your data source + // implementation group: 'org.parboiled', name: 'parboiled-java', version: versions.dependencies.parboiledJava + // implementation group: 'org.json', name: 'json', version: versions.dependencies.json + // implementation group: 'org.glassfish', name: 'javax.json', version: versions.dependencies.javaxJson + + // Test dependencies + testImplementation'junit:junit:' + versions.dependencies.junit + testImplementation 'org.jruby:jruby-complete:' + versions.dependencies.jrubyComplete + testImplementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "common-*.*.*.jar") + + // Add any additional dependencies specific to your data source here + // Example: + // implementation 'com.example:library:1.0.0' +} + +clean { + delete "${projectDir}/Gemfile" + delete "${projectDir}/" + pluginInfo.pluginFullName() + ".gemspec" + delete "${projectDir}/lib/" + delete "${projectDir}/dependenciesLib/" + delete "${projectDir}/vendor/" + new FileNameFinder().getFileNames(projectDir.toString(), pluginInfo.pluginFullName() + "-?.?.?.gem").each { filename -> + delete filename + } +} + +tasks.withType(JavaCompile) { + options.encoding = 'UTF-8' +} + +tasks.register("vendor"){ + dependsOn shadowJar + doLast { + String vendorPathPrefix = "vendor/jar-dependencies" + String projectGroupPath = project.group.replaceAll('\\.', '/') + File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") + projectJarFile.mkdirs() + Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) + validatePluginJar(projectJarFile, project.group) + } +} + +tasks.register("generateRubySupportFiles") { + doLast { + generateRubySupportFilesForPlugin(project.description, project.group, version) + } +} + +tasks.register("removeObsoleteJars") { + doLast { + new FileNameFinder().getFileNames( + projectDir.toString(), + "vendor/**/" + pluginInfo.pluginFullName() + "*.jar", + "vendor/**/" + pluginInfo.pluginFullName() + "-" + version + ".jar").each { f -> + delete f + } + } +} + +tasks.register("gem"){ + dependsOn ([downloadAndInstallJRuby, removeObsoleteJars, vendor, generateRubySupportFiles]) + doLast { + buildGem(projectDir, buildDir, pluginInfo.pluginFullName() + ".gemspec") + } +} + +jacocoTestReport { + reports { + xml.required = true + html.required = true + } + + afterEvaluate { // (optional) : to exclude classes / packages from coverage + classDirectories.from = files(classDirectories.files.collect { + fileTree( + dir: it, + exclude: [ + // Add any classes to exclude from coverage here + // Example: '**/ApplicationConstants.class' + ]) + }) + } +} + +test.finalizedBy jacocoTestReport diff --git a/docs/template-logstash-filter-guardium/datasource_sample.conf b/docs/template-logstash-filter-guardium/datasource_sample.conf new file mode 100644 index 000000000..54d67d7dd --- /dev/null +++ b/docs/template-logstash-filter-guardium/datasource_sample.conf @@ -0,0 +1,178 @@ +# Sample Logstash Configuration for [DATASOURCE_NAME] Guardium Filter Plugin +# +# INSTRUCTIONS: +# 1. Replace [DATASOURCE_NAME] with your data source name +# 2. Replace DATASOURCE_PLACEHOLDER with your data source identifier +# 3. Update the input section based on how you receive logs +# 4. Customize filter conditions as needed +# 5. Save this file as: datasource_name.conf (e.g., mongodb.conf, postgresql.conf) + +input { + # ========== OPTION 1: File Input ========== + # Use this if reading logs from files + # file { + # path => "/path/to/your/audit/logs/*.log" + # start_position => "beginning" + # sincedb_path => "/dev/null" + # type => "DATASOURCE_PLACEHOLDER" + # codec => "json" # Use "plain" for text logs + # } + + # ========== OPTION 2: Kafka Input ========== + # Use this if receiving logs from Kafka + # kafka { + # bootstrap_servers => "localhost:9092" + # topics => ["datasource-audit-logs"] + # type => "DATASOURCE_PLACEHOLDER" + # codec => "json" + # } + + # ========== OPTION 3: HTTP Input ========== + # Use this if receiving logs via HTTP + # http { + # port => 8080 + # type => "DATASOURCE_PLACEHOLDER" + # codec => "json" + # } + + # ========== OPTION 4: Beats Input (Filebeat) ========== + # Use this if using Filebeat to ship logs + # beats { + # port => 5044 + # type => "DATASOURCE_PLACEHOLDER" + # } + + # ========== OPTION 5: Cloud-specific inputs ========== + # For AWS CloudWatch: + # cloudwatch_logs { + # log_group => "/aws/datasource/audit" + # region => "us-east-1" + # type => "DATASOURCE_PLACEHOLDER" + # } + + # For Azure Event Hub: + # azure_event_hubs { + # event_hub_connections => ["Endpoint=sb://..."] + # threads => 2 + # decorate_events => true + # consumer_group => "$Default" + # storage_connection => "DefaultEndpointsProtocol=https;..." + # type => "DATASOURCE_PLACEHOLDER" + # } + + # For GCP Pub/Sub: + # google_pubsub { + # project_id => "your-project-id" + # topic => "datasource-audit-logs" + # subscription => "logstash-subscription" + # json_key_file => "/path/to/key.json" + # type => "DATASOURCE_PLACEHOLDER" + # } + + # TODO: Uncomment and configure the appropriate input for your use case + # Example: Reading from a file + file { + path => "/var/log/datasource/audit.log" + start_position => "beginning" + type => "DATASOURCE_PLACEHOLDER" + codec => "json" + } +} + +filter { + # Only process events of this type + if [type] == "DATASOURCE_PLACEHOLDER" { + + # ========== Optional: Pre-processing ========== + # Add any pre-processing steps here + + # Example: Parse timestamp if not already in ISO format + # date { + # match => ["timestamp", "yyyy-MM-dd HH:mm:ss", "ISO8601"] + # target => "@timestamp" + # } + + # Example: Add custom fields + # mutate { + # add_field => { + # "environment" => "production" + # "data_source" => "[DATASOURCE_NAME]" + # } + # } + + # Example: Remove unnecessary fields + # mutate { + # remove_field => ["@version", "host"] + # } + + # ========== Main Filter Plugin ========== + # This is where the Guardium filter plugin is applied + DATASOURCE_PLACEHOLDER_guardium_filter { + source => "message" + } + + # ========== Optional: Post-processing ========== + # Add any post-processing steps here + + # Example: Add tags for monitoring + # if "_guardiumrecord" in [tags] { + # mutate { + # add_tag => ["processed_successfully"] + # } + # } + } +} + +output { + # ========== Guardium Output ========== + # Send processed events to Guardium + if [GuardRecord] { + # This output is automatically configured by Guardium Universal Connector + # No additional configuration needed here + } + + # ========== Optional: Debug Output ========== + # Uncomment for debugging - writes to stdout + # stdout { + # codec => rubydebug + # } + + # ========== Optional: File Output for Debugging ========== + # Uncomment to write processed events to a file + # file { + # path => "/var/log/logstash/datasource-guardium-output.log" + # codec => json_lines + # } + + # ========== Optional: Error Handling ========== + # Send failed events to a separate output + # if "_DATASOURCE_PLACEHOLDER_json_parse_error" in [tags] { + # file { + # path => "/var/log/logstash/datasource-errors.log" + # codec => json_lines + # } + # } +} + +# ========== Configuration Notes ========== +# +# 1. The 'type' field must match in input and filter sections +# 2. The 'type' field should be unique for each connector +# 3. Adjust codec based on your log format (json, plain, multiline, etc.) +# 4. For production, consider: +# - Setting appropriate buffer sizes +# - Configuring persistent queues +# - Setting up monitoring and alerting +# - Implementing proper error handling +# +# 5. Performance tuning: +# - Adjust worker threads: pipeline.workers +# - Adjust batch size: pipeline.batch.size +# - Configure memory: -Xms and -Xmx in jvm.options +# +# 6. Security considerations: +# - Use SSL/TLS for network inputs +# - Secure credential storage +# - Implement proper access controls +# - Enable audit logging for Logstash itself + diff --git a/docs/template-logstash-filter-guardium/gradle/wrapper/gradle-wrapper.jar b/docs/template-logstash-filter-guardium/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 000000000..62d4c0535 Binary files /dev/null and b/docs/template-logstash-filter-guardium/gradle/wrapper/gradle-wrapper.jar differ diff --git a/docs/template-logstash-filter-guardium/gradle/wrapper/gradle-wrapper.properties b/docs/template-logstash-filter-guardium/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 000000000..81aa1c044 --- /dev/null +++ b/docs/template-logstash-filter-guardium/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,5 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/docs/template-logstash-filter-guardium/gradlew b/docs/template-logstash-filter-guardium/gradlew new file mode 100755 index 000000000..fbd7c5158 --- /dev/null +++ b/docs/template-logstash-filter-guardium/gradlew @@ -0,0 +1,185 @@ +#!/usr/bin/env sh + +# +# Copyright 2015 the original author or authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin or MSYS, switch paths to Windows format before running java +if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=`expr $i + 1` + done + case $i in + 0) set -- ;; + 1) set -- "$args0" ;; + 2) set -- "$args0" "$args1" ;; + 3) set -- "$args0" "$args1" "$args2" ;; + 4) set -- "$args0" "$args1" "$args2" "$args3" ;; + 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=`save "$@"` + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +exec "$JAVACMD" "$@" diff --git a/docs/template-logstash-filter-guardium/gradlew.bat b/docs/template-logstash-filter-guardium/gradlew.bat new file mode 100644 index 000000000..5093609d5 --- /dev/null +++ b/docs/template-logstash-filter-guardium/gradlew.bat @@ -0,0 +1,104 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto init + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto init + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:init +@rem Get command-line arguments, handling Windows variants + +if not "%OS%" == "Windows_NT" goto win9xME_args + +:win9xME_args +@rem Slurp the command line arguments. +set CMD_LINE_ARGS= +set _SKIP=2 + +:win9xME_args_slurp +if "x%~1" == "x" goto execute + +set CMD_LINE_ARGS=%* + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/docs/template-logstash-filter-guardium/settings.gradle b/docs/template-logstash-filter-guardium/settings.gradle new file mode 100644 index 000000000..3a95481a1 --- /dev/null +++ b/docs/template-logstash-filter-guardium/settings.gradle @@ -0,0 +1,11 @@ +/* + * This file was generated by the Gradle 'init' task. + * + * The settings file is used to specify which projects to include in your build. + * + * Detailed information about configuring a multi-project build in Gradle can be found + * in the user manual at https://docs.gradle.org/6.5.1/userguide/multi_project_builds.html + */ + +rootProject.name = 'logstash-filter-DATASOURCE_PLACEHOLDER-guardium' + diff --git a/docs/template-logstash-filter-guardium/src/main/java/com/ibm/guardium/DATASOURCE_PLACEHOLDER/ApplicationConstants.java b/docs/template-logstash-filter-guardium/src/main/java/com/ibm/guardium/DATASOURCE_PLACEHOLDER/ApplicationConstants.java new file mode 100644 index 000000000..52f2a1d68 --- /dev/null +++ b/docs/template-logstash-filter-guardium/src/main/java/com/ibm/guardium/DATASOURCE_PLACEHOLDER/ApplicationConstants.java @@ -0,0 +1,224 @@ +/* +Copyright IBM Corp. 2024 All rights reserved. +SPDX-License-Identifier: Apache-2.0 +*/ +package com.ibm.guardium.DATASOURCE_PLACEHOLDER; + +/** + * ApplicationConstants class contains all constant values used throughout the plugin. + * + * INSTRUCTIONS: + * 1. Replace DATASOURCE_PLACEHOLDER with your data source name + * 2. Update field name constants to match your audit log structure + * 3. Add any additional constants specific to your data source + * 4. Update default values as needed + */ +public class ApplicationConstants { + + // ========== Field Names ========== + // TODO: Update these constants to match your audit log field names + + /** + * Message field name in Logstash event + */ + public static final String MESSAGE = "message"; + + /** + * Timestamp field name in audit log + * Common alternatives: "time", "eventTime", "@timestamp", "timestamp" + */ + public static final String TIMESTAMP = "timestamp"; + + /** + * User field name in audit log + * Common alternatives: "user", "username", "dbUser", "principal", "userId" + */ + public static final String USER = "user"; + + /** + * Database name field in audit log + * Common alternatives: "database", "db", "schema", "databaseName" + */ + public static final String DATABASE = "database"; + + /** + * Table/Collection name field in audit log + * Common alternatives: "table", "collection", "tableName", "object" + */ + public static final String TABLE = "table"; + + /** + * Operation/Action field in audit log + * Common alternatives: "operation", "action", "command", "verb", "operationType" + */ + public static final String OPERATION = "operation"; + + /** + * Query/Statement field in audit log + * Common alternatives: "query", "statement", "sql", "command", "queryText" + */ + public static final String QUERY = "query"; + + /** + * Client IP address field in audit log + * Common alternatives: "clientIp", "client_ip", "sourceIp", "remoteAddress" + */ + public static final String CLIENT_IP = "clientIp"; + + /** + * Server hostname field in audit log + * Common alternatives: "serverHost", "server", "hostname", "host" + */ + public static final String SERVER_HOST = "serverHost"; + + /** + * Status code field in audit log + * Common alternatives: "statusCode", "status", "resultCode", "returnCode" + */ + public static final String STATUS_CODE = "statusCode"; + + /** + * Error message field in audit log + * Common alternatives: "errorMessage", "error", "message", "errorDescription" + */ + public static final String ERROR_MESSAGE = "errorMessage"; + + /** + * Session ID field in audit log + * Common alternatives: "sessionId", "session", "connectionId", "activityId" + */ + public static final String SESSION_ID = "sessionId"; + + // ========== Default Values ========== + + /** + * Default IPv4 address when not available + */ + public static final String DEFAULT_IP = "0.0.0.0"; + + /** + * Default IPv6 address when not available + */ + public static final String DEFAULT_IPV6 = "0000:0000:0000:0000:0000:FFFF:0000:0000"; + + /** + * Default PORT when not available + */ + public static final String DEFAULT_IP = "0"; + + /** + * Unknown string value + */ + public static final String UNKNOWN_STRING = ""; + + /** + * Not available string value + */ + public static final String NOT_AVAILABLE = "N.A."; + + // ========== Server and Protocol Information ========== + + /** + * Server type identifier + * TODO: Update with your data source name (e.g., "MongoDB", "PostgreSQL", "MySQL") + */ + public static final String SERVER_TYPE = "[DATASOURCE_NAME]"; + + /** + * Data protocol identifier + * TODO: Update with your data source protocol (e.g., "MongoDB", "PostgreSQL", "MySQL") + */ + public static final String DATA_PROTOCOL = "[DATASOURCE_NAME]"; + + // ========== Object Types ========== + + /** + * Collection/Table object type + */ + public static final String COLLECTION = "collection"; + + /** + * Database object type + */ + public static final String DATABASE_TYPE = "database"; + + /** + * Schema object type + */ + public static final String SCHEMA = "schema"; + + /** + * Table object type + */ + public static final String TABLE_TYPE = "table"; + + // ========== Exception Types ========== + + /** + * Authorization exception type + */ + public static final String EXCEPTION_TYPE_AUTHORIZATION_STRING = "SQL_ERROR"; + + /** + * Authentication exception type + */ + public static final String EXCEPTION_TYPE_AUTHENTICATION_STRING = "LOGIN_FAILED"; + + /** + * SQL error exception type + */ + public static final String SQL_ERROR = "SQL_ERROR"; + + // ========== Logstash Tags ========== + + /** + * Tag for JSON parsing errors + */ + public static final String LOGSTASH_TAG_JSON_PARSE_ERROR = "DATASOURCE_PLACEHOLDER_json_parse_error"; + + /** + * Tag for events that are not from this data source + */ + public static final String LOGSTASH_TAG_SKIP_NOT_DATASOURCE = "DATASOURCE_PLACEHOLDER_skip_not_datasource"; + + /** + * Tag for invalid events + */ + public static final String LOGSTASH_TAG_INVALID_EVENT = "DATASOURCE_PLACEHOLDER_invalid_event"; + + // ========== Additional Constants ========== + // TODO: Add any additional constants specific to your data source + + /** + * Example: Minimum supported version + */ + // public static final String MIN_VERSION = "1.0.0"; + + /** + * Example: Maximum log size + */ + // public static final int MAX_LOG_SIZE = 10000; + + /** + * Example: Specific operation types + */ + // public static final String OPERATION_SELECT = "SELECT"; + // public static final String OPERATION_INSERT = "INSERT"; + // public static final String OPERATION_UPDATE = "UPDATE"; + // public static final String OPERATION_DELETE = "DELETE"; + + /** + * Example: Log categories + */ + // public static final String CATEGORY_DATA_ACCESS = "DATA_ACCESS"; + // public static final String CATEGORY_ADMIN = "ADMIN"; + // public static final String CATEGORY_DDL = "DDL"; + + /** + * Private constructor to prevent instantiation + */ + private ApplicationConstants() { + throw new IllegalStateException("Constants class"); + } +} + diff --git a/docs/template-logstash-filter-guardium/src/main/java/com/ibm/guardium/DATASOURCE_PLACEHOLDER/DataSourceGuardiumFilter.java b/docs/template-logstash-filter-guardium/src/main/java/com/ibm/guardium/DATASOURCE_PLACEHOLDER/DataSourceGuardiumFilter.java new file mode 100644 index 000000000..231058ec5 --- /dev/null +++ b/docs/template-logstash-filter-guardium/src/main/java/com/ibm/guardium/DATASOURCE_PLACEHOLDER/DataSourceGuardiumFilter.java @@ -0,0 +1,137 @@ +/* +Copyright IBM Corp. 2024 All rights reserved. +SPDX-License-Identifier: Apache-2.0 +*/ +package com.ibm.guardium.DATASOURCE_PLACEHOLDER; + +import java.util.Collection; +import java.util.Collections; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonObject; +import com.ibm.guardium.universalconnector.commons.GuardConstants; +import com.ibm.guardium.universalconnector.commons.structures.Record; + +import co.elastic.logstash.api.Configuration; +import co.elastic.logstash.api.Context; +import co.elastic.logstash.api.Event; +import co.elastic.logstash.api.Filter; +import co.elastic.logstash.api.FilterMatchListener; +import co.elastic.logstash.api.LogstashPlugin; +import co.elastic.logstash.api.PluginConfigSpec; + +/** + * This class is the main filter plugin for [DATASOURCE_NAME]. + * It receives log events from Logstash, validates them, parses them into + * Guardium Record format, and returns the processed events. + * + * INSTRUCTIONS: + * 1. Replace DATASOURCE_PLACEHOLDER with your data source name (e.g., mongodb, postgresql) + * 2. Replace DataSourceGuardiumFilter with your actual class name (e.g., MongoDbGuardiumFilter) + * 3. Update the @LogstashPlugin name to match your plugin name + * 4. Implement the validation logic specific to your data source + * 5. Update the error tag constants + */ +@LogstashPlugin(name = "DATASOURCE_PLACEHOLDER_guardium_filter") +public class DataSourceGuardiumFilter implements Filter { + + // Configuration for the source field (usually "message") + public static final PluginConfigSpec SOURCE_CONFIG = PluginConfigSpec.stringSetting("source", "message"); + + // Error tag for JSON parsing errors - customize for your data source + public static final String LOGSTASH_TAG_JSON_PARSE_ERROR = "DATASOURCE_PLACEHOLDER_json_parse_error"; + + private String id; + private static Logger log = LogManager.getLogger(DataSourceGuardiumFilter.class); + + /** + * Constructor for the filter plugin + * + * @param id Unique identifier for this filter instance + * @param config Configuration object + * @param context Logstash context + */ + public DataSourceGuardiumFilter(String id, Configuration config, Context context) { + this.id = id; + } + + @Override + public Collection> configSchema() { + return Collections.singletonList(SOURCE_CONFIG); + } + + /** + * Main filter method that processes events + * + * @param events Collection of events to process + * @param matchListener Listener to notify when events match + * @return Collection of processed events + */ + @Override + public Collection filter(Collection events, FilterMatchListener matchListener) { + + for (Event event : events) { + // Get the message field from the event + if (event.getField("message") instanceof String && event.getField("message") != null) { + String messageString = event.getField("message").toString(); + + try { + // TODO: Add validation logic specific to your data source + // Example: Check if message contains specific keywords or patterns + // if (!isValidDataSourceMessage(messageString)) { + // event.tag("DATASOURCE_PLACEHOLDER_skip_not_datasource"); + // continue; + // } + + // Parse the JSON message if it is in the JSON format + JsonObject inputJSON = new Gson().fromJson(messageString, JsonObject.class); + + // TODO: Add additional validation on the parsed JSON + // Example: Check for required fields + // if (!inputJSON.has("required_field")) { + // event.tag("DATASOURCE_PLACEHOLDER_missing_required_field"); + // continue; + // } + + // Parse the record using the Parser class. Pass the JSO object or the message itself + Record record = Parser.parseRecord(inputJSON); + + // Convert the record to JSON and add it to the event + final Gson gson = new GsonBuilder().disableHtmlEscaping().serializeNulls().create(); + event.setField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME, gson.toJson(record)); + + // Notify that this event matched + matchListener.filterMatched(event); + + } catch (Exception exception) { + log.error("[DATASOURCE_NAME] filter: Error parsing event ", exception); + event.tag(LOGSTASH_TAG_JSON_PARSE_ERROR); + } + } + } + return events; + } + + @Override + public String getId() { + return this.id; + } + + /** + * TODO: Implement validation logic for your data source + * This method should check if the message is from your data source + * + * @param message The message string to validate + * @return true if the message is from your data source, false otherwise + */ + private boolean isValidDataSourceMessage(String message) { + // Example implementation: + // return message.contains("your_datasource_identifier"); + return true; // Replace with actual validation + } +} + diff --git a/docs/template-logstash-filter-guardium/src/main/java/com/ibm/guardium/DATASOURCE_PLACEHOLDER/Parser.java b/docs/template-logstash-filter-guardium/src/main/java/com/ibm/guardium/DATASOURCE_PLACEHOLDER/Parser.java new file mode 100644 index 000000000..859583138 --- /dev/null +++ b/docs/template-logstash-filter-guardium/src/main/java/com/ibm/guardium/DATASOURCE_PLACEHOLDER/Parser.java @@ -0,0 +1,445 @@ +/* +Copyright IBM Corp. 2024 All rights reserved. +SPDX-License-Identifier: Apache-2.0 +*/ +package com.ibm.guardium.DATASOURCE_PLACEHOLDER; + +import com.ibm.guardium.universalconnector.commons.Util; +import com.ibm.guardium.universalconnector.commons.structures.Accessor; +import com.ibm.guardium.universalconnector.commons.structures.Construct; +import com.ibm.guardium.universalconnector.commons.structures.Data; +import com.ibm.guardium.universalconnector.commons.structures.ExceptionRecord; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import com.ibm.guardium.universalconnector.commons.structures.Sentence; +import com.ibm.guardium.universalconnector.commons.structures.SentenceObject; +import com.ibm.guardium.universalconnector.commons.structures.SessionLocator; +import com.ibm.guardium.universalconnector.commons.structures.Time; + +import java.time.ZonedDateTime; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.google.gson.JsonObject; + +/** + * Parser class for converting [DATASOURCE_NAME] audit logs into Guardium Record format. + * + * This parser supports both JSON and text-based log formats. + * + * INSTRUCTIONS: + * 1. Replace DATASOURCE_PLACEHOLDER with your data source name + * 2. Determine if your logs are JSON, text, or mixed format + * 3. Implement appropriate parsing methods for your log format + * 4. Update field extraction logic to match your log structure + * 5. Add helper methods as needed for your specific data source + */ +public class Parser { + private static Logger log = LogManager.getLogger(Parser.class); + + /** + * Main method to parse audit log into a Guardium Record + * This method handles both JSON and text-based formats + * + * @param input The input object (JsonObject for JSON logs, String for text logs) + * @return A Guardium Record object + * @throws Exception if parsing fails + */ + public static Record parseRecord(final Object input) throws Exception { + + if (log.isDebugEnabled()) { + log.debug("Parsing event: {}", input); + } + + Record record = new Record(); + + try { + // Determine input type and parse accordingly + if (input instanceof JsonObject) { + return parseJsonRecord((JsonObject) input); + } else if (input instanceof String) { + return parseTextRecord((String) input); + } else { + throw new Exception("Unsupported input type: " + input.getClass().getName()); + } + + } catch (Exception e) { + log.error("Exception occurred while parsing event in parseRecord method: ", e); + throw e; + } + } + + /** + * Parse JSON-formatted audit log + * + * @param inputJSON The JSON object containing the audit log + * @return A Guardium Record object + * @throws Exception if parsing fails + */ + private static Record parseJsonRecord(final JsonObject inputJSON) throws Exception { + Record record = new Record(); + + // TODO: Extract and validate required fields from your JSON audit log + // Example: + // if (!inputJSON.has("timestamp")) { + // throw new Exception("Missing required field: timestamp"); + // } + + // 1. Parse and set the timestamp + record.setTime(parseTimeFromJson(inputJSON)); + + // 2. Extract and set database name + String databaseName = extractDatabaseNameFromJson(inputJSON); + record.setDbName(databaseName); + + // 3. Extract and set session ID + String sessionId = extractSessionIdFromJson(inputJSON); + record.setSessionId(sessionId); + + // 4. Extract and set app user name + String appUserName = extractAppUserNameFromJson(inputJSON); + record.setAppUserName(appUserName); + + // 5. Parse accessor (user and client information) + record.setAccessor(parseAccessorFromJson(inputJSON)); + + // 6. Parse session locator (network information) + record.setSessionLocator(parseSessionLocatorFromJson(inputJSON)); + + // 7. Check if this is an error/exception or successful operation + if (isErrorFromJson(inputJSON)) { + record.setException(parseExceptionFromJson(inputJSON)); + } else { + record.setData(parseDataFromJson(inputJSON)); + } + + return record; + } + + /** + * Parse text-formatted audit log + * + * @param inputText The text string containing the audit log + * @return A Guardium Record object + * @throws Exception if parsing fails + */ + private static Record parseTextRecord(final String inputText) throws Exception { + Record record = new Record(); + + // TODO: Define regex patterns to extract fields from text logs + // Example patterns: + // Pattern timestampPattern = Pattern.compile("timestamp=(\\S+)"); + // Pattern userPattern = Pattern.compile("user=(\\S+)"); + // Pattern dbPattern = Pattern.compile("database=(\\S+)"); + + // 1. Parse and set the timestamp + record.setTime(parseTimeFromText(inputText)); + + // 2. Extract and set database name + String databaseName = extractDatabaseNameFromText(inputText); + record.setDbName(databaseName); + + // 3. Extract and set session ID + String sessionId = extractSessionIdFromText(inputText); + record.setSessionId(sessionId); + + // 4. Extract and set app user name + String appUserName = extractAppUserNameFromText(inputText); + record.setAppUserName(appUserName); + + // 5. Parse accessor (user and client information) + record.setAccessor(parseAccessorFromText(inputText)); + + // 6. Parse session locator (network information) + record.setSessionLocator(parseSessionLocatorFromText(inputText)); + + // 7. Check if this is an error/exception or successful operation + if (isErrorFromText(inputText)) { + record.setException(parseExceptionFromText(inputText)); + } else { + record.setData(parseDataFromText(inputText)); + } + + return record; + } + + // ========== JSON Parsing Methods ========== + + private static Time parseTimeFromJson(JsonObject inputJSON) { + // TODO: Extract timestamp field from your JSON audit log + String dateString = inputJSON.get("timestamp").getAsString(); // Replace with your field name + ZonedDateTime date = ZonedDateTime.parse(dateString); + long millis = date.toInstant().toEpochMilli(); + int minOffset = date.getOffset().getTotalSeconds() / 60; + return new Time(millis, minOffset, 0); + } + + private static String extractDatabaseNameFromJson(JsonObject inputJSON) { + // TODO: Implement logic to extract database name from JSON + // The default value is N.A. + return ApplicationConstants.NOT_AVAILABLE; + } + + private static String extractSessionIdFromJson(JsonObject inputJSON) { + // TODO: Implement logic to extract or generate session ID from JSON + return ""; + } + + private static String extractAppUserNameFromJson(JsonObject inputJSON) { + // TODO: Implement logic to extract app user name from JSON + return ""; + } + + private static boolean isErrorFromJson(JsonObject inputJSON) { + // TODO: Implement logic to detect errors in JSON + return false; + } + + private static ExceptionRecord parseExceptionFromJson(JsonObject inputJSON) { + ExceptionRecord exception = new ExceptionRecord(); + // We have 2 types of exceptions here; SQL_ERROR and LOGIN_FAILED + exception.setExceptionTypeId("EXCEPTION TYPE"); + exception.setDescription("Error DESCRIPTION"); + // If there is a query statement we need to set it here + exception.setSqlString(ApplicationConstants.NOT_AVAILABLE); + return exception; + } + + private static Data parseDataFromJson(JsonObject inputJSON) throws Exception { + // The goal of this method is to parse the query statement and extract useful information from it + // For some of the query languages like MySql or MsSql there are available high performance Sniffer parsers available that we can use here instead of parsing the query in the universal connector + // In the case of using a Sniffer parser, please read the README section about how to use sniffer parsers + Data data = new Data(); + Construct construct = parseConstructFromJson(inputJSON); + if (construct != null) { + data.setConstruct(construct); + } + return data; + } + + private static Construct parseConstructFromJson(JsonObject inputJSON) throws Exception { + final Construct construct = new Construct(); + Sentence sentence = parseSentenceFromJson(inputJSON); + construct.sentences.add(sentence); + String fullSql = inputJSON.toString(); // Replace with actual query extraction + construct.setFullSql(fullSql); + construct.setRedactedSensitiveDataSql(fullSql); + return construct; + } + + private static Sentence parseSentenceFromJson(JsonObject inputJSON) { + // Verb is the action in your query eg. SELECT or INSERT + // OBJECTS are the objects you have in your statement eg. table1 + String verb = "VERB"; // TODO: Extract verb from JSON + Sentence sentence = new Sentence(verb); + SentenceObject sentenceObject = new SentenceObject("OBJECT"); + sentenceObject.setType(ApplicationConstants.COLLECTION); + sentence.getObjects().add(sentenceObject); + return sentence; + } + + private static SessionLocator parseSessionLocatorFromJson(JsonObject inputJSON) { + SessionLocator sessionLocator = new SessionLocator(); + String clientIp = ApplicationConstants.DEFAULT_IP; // TODO: Extract from JSON + + sessionLocator.setIpv6(Boolean.FALSE); + if (Util.isIPv6(clientIp)) { + sessionLocator.setIpv6(true); + sessionLocator.setServerIpv6(ApplicationConstants.DEFAULT_IPV6); + sessionLocator.setClientIpv6(clientIp); + sessionLocator.setClientIp(clientIp); + } else { + sessionLocator.setServerIp(ApplicationConstants.DEFAULT_IP); + sessionLocator.setClientIp(clientIp); + sessionLocator.setClientIpv6(ApplicationConstants.UNKNOWN_STRING); + } + + // Set the ports if possible + sessionLocator.setServerPort(ApplicationConstants.PORT_DEFAULT); + sessionLocator.setClientPort(ApplicationConstants.PORT_DEFAULT); + return sessionLocator; + } + + private static Accessor parseAccessorFromJson(JsonObject inputJSON) { + Accessor accessor = new Accessor(); + accessor.setDbUser(ApplicationConstants.NOT_AVAILABLE); // TODO: Extract from JSON + accessor.setServerHostName("DATASOURCE_SERVER"); // TODO: Extract from JSON + accessor.setServerType(ApplicationConstants.SERVER_TYPE); + accessor.setDbProtocol(ApplicationConstants.DATA_PROTOCOL); + accessor.setDbProtocolVersion(ApplicationConstants.UNKNOWN_STRING); + accessor.setSourceProgram(ApplicationConstants.UNKNOWN_STRING); + accessor.setServerDescription(ApplicationConstants.UNKNOWN_STRING); + accessor.setLanguage(Accessor.LANGUAGE_FREE_TEXT_STRING); + accessor.setDataType(Accessor.DATA_TYPE_GUARDIUM_SHOULD_NOT_PARSE_SQL); + accessor.setClient_mac(ApplicationConstants.UNKNOWN_STRING); + accessor.setClientHostName(ApplicationConstants.UNKNOWN_STRING); + accessor.setCommProtocol(ApplicationConstants.UNKNOWN_STRING); + accessor.setOsUser(ApplicationConstants.UNKNOWN_STRING); + accessor.setClientOs(ApplicationConstants.UNKNOWN_STRING); + accessor.setServerOs(ApplicationConstants.UNKNOWN_STRING); + accessor.setServiceName(ApplicationConstants.UNKNOWN_STRING); + return accessor; + } + + // ========== Text Parsing Methods ========== + + private static Time parseTimeFromText(String inputText) { + // TODO: Extract timestamp from text using regex + // Example: + // Pattern pattern = Pattern.compile("timestamp=(\\S+)"); + // Matcher matcher = pattern.matcher(inputText); + // if (matcher.find()) { + // String dateString = matcher.group(1); + // ZonedDateTime date = ZonedDateTime.parse(dateString); + // long millis = date.toInstant().toEpochMilli(); + // int minOffset = date.getOffset().getTotalSeconds() / 60; + // return new Time(millis, minOffset, 0); + // } + + // Default: use current time + long millis = System.currentTimeMillis(); + return new Time(millis, 0, 0); + } + + private static String extractDatabaseNameFromText(String inputText) { + // TODO: Extract database name using regex + // Example: + // Pattern pattern = Pattern.compile("database=(\\S+)"); + // Matcher matcher = pattern.matcher(inputText); + // if (matcher.find()) { + // return matcher.group(1); + // } + // Default value is N.A. + return ApplicationConstants.NOT_AVAILABLE; + } + + private static String extractSessionIdFromText(String inputText) { + // TODO: Extract or generate session ID from text + return ""; + } + + private static String extractAppUserNameFromText(String inputText) { + // TODO: Extract app user name using regex + return ""; + } + + private static boolean isErrorFromText(String inputText) { + // TODO: Detect errors in text logs + // Example: check for keywords like "ERROR", "FAILED", "DENIED" + return inputText.contains("ERROR") || inputText.contains("FAILED"); + } + + private static ExceptionRecord parseExceptionFromText(String inputText) { + ExceptionRecord exception = new ExceptionRecord(); + // Exception type can be SQL_ERROR or LOGIN_FAILED + exception.setExceptionTypeId("EXCEPTION TYPE"); + exception.setDescription("Error description"); + // Set the query statement if available + exception.setSqlString(""); + return exception; + } + + private static Data parseDataFromText(String inputText) throws Exception { + Data data = new Data(); + Construct construct = parseConstructFromText(inputText); + if (construct != null) { + data.setConstruct(construct); + } + return data; + } + + private static Construct parseConstructFromText(String inputText) throws Exception { + final Construct construct = new Construct(); + Sentence sentence = parseSentenceFromText(inputText); + construct.sentences.add(sentence); + construct.setFullSql(inputText); + construct.setRedactedSensitiveDataSql(inputText); + return construct; + } + + private static Sentence parseSentenceFromText(String inputText) { + // TODO: Extract verb (operation) from text + // Example: look for SQL keywords or operation names + String verb = "OBJECT"; + Sentence sentence = new Sentence(verb); + SentenceObject sentenceObject = new SentenceObject("VERB"); + sentenceObject.setType(ApplicationConstants.COLLECTION); + sentence.getObjects().add(sentenceObject); + return sentence; + } + + private static SessionLocator parseSessionLocatorFromText(String inputText) { + SessionLocator sessionLocator = new SessionLocator(); + + // TODO: Extract client IP using regex + // Example: + // Pattern pattern = Pattern.compile("client_ip=(\\S+)"); + // Matcher matcher = pattern.matcher(inputText); + // String clientIp = matcher.find() ? matcher.group(1) : ApplicationConstants.DEFAULT_IP; + String clientIp = ApplicationConstants.DEFAULT_IP; + + sessionLocator.setIpv6(Boolean.FALSE); + if (Util.isIPv6(clientIp)) { + sessionLocator.setIpv6(true); + sessionLocator.setServerIpv6(ApplicationConstants.DEFAULT_IPV6); + sessionLocator.setClientIpv6(clientIp); + sessionLocator.setClientIp(ApplicationConstants.UNKNOWN_STRING); + } else { + sessionLocator.setServerIp(ApplicationConstants.DEFAULT_IP); + sessionLocator.setClientIp(clientIp); + sessionLocator.setClientIpv6(ApplicationConstants.UNKNOWN_STRING); + } + + sessionLocator.setServerPort(SessionLocator.PORT_DEFAULT); + sessionLocator.setClientPort(SessionLocator.PORT_DEFAULT); + return sessionLocator; + } + + private static Accessor parseAccessorFromText(String inputText) { + Accessor accessor = new Accessor(); + + // TODO: Extract user from text using regex + accessor.setDbUser(ApplicationConstants.NOT_AVAILABLE); + accessor.setServerHostName("DATASOURCE_SERVER"); + accessor.setServerType(ApplicationConstants.SERVER_TYPE); + accessor.setDbProtocol(ApplicationConstants.DATA_PROTOCOL); + accessor.setDbProtocolVersion(ApplicationConstants.UNKNOWN_STRING); + accessor.setSourceProgram(ApplicationConstants.UNKNOWN_STRING); + accessor.setServerDescription(ApplicationConstants.UNKNOWN_STRING); + accessor.setLanguage(Accessor.LANGUAGE_FREE_TEXT_STRING); + accessor.setDataType(Accessor.DATA_TYPE_GUARDIUM_SHOULD_NOT_PARSE_SQL); + accessor.setClient_mac(ApplicationConstants.UNKNOWN_STRING); + accessor.setClientHostName(ApplicationConstants.UNKNOWN_STRING); + accessor.setCommProtocol(ApplicationConstants.UNKNOWN_STRING); + accessor.setOsUser(ApplicationConstants.UNKNOWN_STRING); + accessor.setClientOs(ApplicationConstants.UNKNOWN_STRING); + accessor.setServerOs(ApplicationConstants.UNKNOWN_STRING); + accessor.setServiceName(ApplicationConstants.UNKNOWN_STRING); + return accessor; + } + + /** + * Helper method to extract value using regex pattern + * + * @param input The input string + * @param pattern The regex pattern + * @param groupIndex The capture group index (usually 1) + * @return Extracted value or empty string if not found + */ + protected static String extractWithRegex(String input, String pattern, int groupIndex) { + try { + Pattern p = Pattern.compile(pattern); + Matcher m = p.matcher(input); + if (m.find()) { + return m.group(groupIndex); + } + } catch (Exception e) { + log.error("Error extracting with regex pattern: " + pattern, e); + } + return StringUtils.EMPTY; + } +} + diff --git a/docs/template-logstash-filter-guardium/src/test/java/com/ibm/guardium/DATASOURCE_PLACEHOLDER/DataSourceGuardiumFilterTest.java b/docs/template-logstash-filter-guardium/src/test/java/com/ibm/guardium/DATASOURCE_PLACEHOLDER/DataSourceGuardiumFilterTest.java new file mode 100644 index 000000000..16fef92a2 --- /dev/null +++ b/docs/template-logstash-filter-guardium/src/test/java/com/ibm/guardium/DATASOURCE_PLACEHOLDER/DataSourceGuardiumFilterTest.java @@ -0,0 +1,245 @@ +/* +Copyright IBM Corp. 2024 All rights reserved. +SPDX-License-Identifier: Apache-2.0 +*/ +package com.ibm.guardium.DATASOURCE_PLACEHOLDER; + +import static org.junit.Assert.*; + +import java.util.Collection; +import java.util.Collections; +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.Test; +import org.logstash.plugins.ContextImpl; + +import com.ibm.guardium.universalconnector.commons.GuardConstants; + +import co.elastic.logstash.api.Context; +import co.elastic.logstash.api.Event; +import co.elastic.logstash.api.FilterMatchListener; + +/** + * Unit tests for DataSourceGuardiumFilter + * + * INSTRUCTIONS: + * 1. Replace DATASOURCE_PLACEHOLDER with your data source name + * 2. Replace DataSourceGuardiumFilter with your actual filter class name + * 3. Add test cases for different log types from your data source + * 4. Include tests for success cases, error cases, and edge cases + * 5. Use actual sample logs from your data source + */ +public class DataSourceGuardiumFilterTest { + + final static Context context = new ContextImpl(null, null); + final static DataSourceGuardiumFilter filter = new DataSourceGuardiumFilter("test-id", null, context); + + /** + * Test helper class to count matched events + */ + static class TestMatchListener implements FilterMatchListener { + private AtomicInteger matchCount = new AtomicInteger(0); + + @Override + public void filterMatched(Event event) { + matchCount.incrementAndGet(); + } + + public int getMatchCount() { + return matchCount.get(); + } + } + + /** + * Test successful SELECT query + * TODO: Replace with actual log from your data source + */ + @Test + public void testSuccessfulSelectQuery() { + // TODO: Replace with actual JSON or text log from your data source + final String auditLog = "{ \"timestamp\": \"2024-01-01T12:00:00Z\", \"user\": \"testuser\", \"database\": \"testdb\", \"operation\": \"SELECT\", \"query\": \"SELECT * FROM users\", \"status\": \"SUCCESS\" }"; + + Event e = new org.logstash.Event(); + TestMatchListener matchListener = new TestMatchListener(); + e.setField("message", auditLog); + + Collection results = filter.filter(Collections.singletonList(e), matchListener); + + assertEquals(1, results.size()); + assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); + assertEquals(1, matchListener.getMatchCount()); + } + + /** + * Test successful INSERT operation + * TODO: Replace with actual log from your data source + */ + @Test + public void testSuccessfulInsert() { + final String auditLog = "{ \"timestamp\": \"2024-01-01T12:00:00Z\", \"user\": \"testuser\", \"database\": \"testdb\", \"operation\": \"INSERT\", \"query\": \"INSERT INTO users VALUES (1, 'John')\", \"status\": \"SUCCESS\" }"; + + Event e = new org.logstash.Event(); + TestMatchListener matchListener = new TestMatchListener(); + e.setField("message", auditLog); + + Collection results = filter.filter(Collections.singletonList(e), matchListener); + + assertEquals(1, results.size()); + assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); + assertEquals(1, matchListener.getMatchCount()); + } + + /** + * Test successful UPDATE operation + * TODO: Replace with actual log from your data source + */ + @Test + public void testSuccessfulUpdate() { + final String auditLog = "{ \"timestamp\": \"2024-01-01T12:00:00Z\", \"user\": \"testuser\", \"database\": \"testdb\", \"operation\": \"UPDATE\", \"query\": \"UPDATE users SET name='Jane' WHERE id=1\", \"status\": \"SUCCESS\" }"; + + Event e = new org.logstash.Event(); + TestMatchListener matchListener = new TestMatchListener(); + e.setField("message", auditLog); + + Collection results = filter.filter(Collections.singletonList(e), matchListener); + + assertEquals(1, results.size()); + assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); + assertEquals(1, matchListener.getMatchCount()); + } + + /** + * Test successful DELETE operation + * TODO: Replace with actual log from your data source + */ + @Test + public void testSuccessfulDelete() { + final String auditLog = "{ \"timestamp\": \"2024-01-01T12:00:00Z\", \"user\": \"testuser\", \"database\": \"testdb\", \"operation\": \"DELETE\", \"query\": \"DELETE FROM users WHERE id=1\", \"status\": \"SUCCESS\" }"; + + Event e = new org.logstash.Event(); + TestMatchListener matchListener = new TestMatchListener(); + e.setField("message", auditLog); + + Collection results = filter.filter(Collections.singletonList(e), matchListener); + + assertEquals(1, results.size()); + assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); + assertEquals(1, matchListener.getMatchCount()); + } + + /** + * Test failed operation (error case) + * TODO: Replace with actual error log from your data source + */ + @Test + public void testFailedOperation() { + final String auditLog = "{ \"timestamp\": \"2024-01-01T12:00:00Z\", \"user\": \"testuser\", \"database\": \"testdb\", \"operation\": \"SELECT\", \"query\": \"SELECT * FROM nonexistent\", \"status\": \"ERROR\", \"errorMessage\": \"Table not found\" }"; + + Event e = new org.logstash.Event(); + TestMatchListener matchListener = new TestMatchListener(); + e.setField("message", auditLog); + + Collection results = filter.filter(Collections.singletonList(e), matchListener); + + assertEquals(1, results.size()); + assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); + assertEquals(1, matchListener.getMatchCount()); + } + + /** + * Test DDL operation (CREATE TABLE) + * TODO: Replace with actual DDL log from your data source + */ + @Test + public void testCreateTable() { + final String auditLog = "{ \"timestamp\": \"2024-01-01T12:00:00Z\", \"user\": \"admin\", \"database\": \"testdb\", \"operation\": \"CREATE\", \"query\": \"CREATE TABLE users (id INT, name VARCHAR(100))\", \"status\": \"SUCCESS\" }"; + + Event e = new org.logstash.Event(); + TestMatchListener matchListener = new TestMatchListener(); + e.setField("message", auditLog); + + Collection results = filter.filter(Collections.singletonList(e), matchListener); + + assertEquals(1, results.size()); + assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); + assertEquals(1, matchListener.getMatchCount()); + } + + /** + * Test invalid JSON + */ + @Test + public void testInvalidJson() { + final String auditLog = "This is not valid JSON"; + + Event e = new org.logstash.Event(); + TestMatchListener matchListener = new TestMatchListener(); + e.setField("message", auditLog); + + Collection results = filter.filter(Collections.singletonList(e), matchListener); + + assertEquals(1, results.size()); + // Should be tagged with error + assertTrue(e.getTags().contains(DataSourceGuardiumFilter.LOGSTASH_TAG_JSON_PARSE_ERROR)); + assertEquals(0, matchListener.getMatchCount()); + } + + /** + * Test null message + */ + @Test + public void testNullMessage() { + Event e = new org.logstash.Event(); + TestMatchListener matchListener = new TestMatchListener(); + e.setField("message", null); + + Collection results = filter.filter(Collections.singletonList(e), matchListener); + + assertEquals(1, results.size()); + assertEquals(0, matchListener.getMatchCount()); + } + + /** + * Test empty message + */ + @Test + public void testEmptyMessage() { + Event e = new org.logstash.Event(); + TestMatchListener matchListener = new TestMatchListener(); + e.setField("message", ""); + + Collection results = filter.filter(Collections.singletonList(e), matchListener); + + assertEquals(1, results.size()); + assertEquals(0, matchListener.getMatchCount()); + } + + /** + * Test with missing required fields + * TODO: Customize based on your required fields + */ + @Test + public void testMissingRequiredFields() { + final String auditLog = "{ \"timestamp\": \"2024-01-01T12:00:00Z\" }"; // Missing other required fields + + Event e = new org.logstash.Event(); + TestMatchListener matchListener = new TestMatchListener(); + e.setField("message", auditLog); + + Collection results = filter.filter(Collections.singletonList(e), matchListener); + + assertEquals(1, results.size()); + // Depending on your implementation, this might be tagged with an error + } + + // TODO: Add more test cases specific to your data source: + // - Different user types (admin, regular user, service account) + // - Different operation types specific to your data source + // - Edge cases (very long queries, special characters, etc.) + // - Different timestamp formats if applicable + // - IPv4 and IPv6 addresses + // - Multiple databases/schemas + // - Transactions + // - Stored procedures/functions + // - etc. +} diff --git a/filter-plugin/logstash-filter-adabas-guardium/.gitignore b/filter-plugin/logstash-filter-adabas-guardium/.gitignore new file mode 100644 index 000000000..371b42cb5 --- /dev/null +++ b/filter-plugin/logstash-filter-adabas-guardium/.gitignore @@ -0,0 +1,28 @@ +# Ignore Gradle build output directory +build +bin +*.idea +*.iml +lib/ +vendor/ +.bundle/ +build/ +out/ +.idea +.gradle +.vscode +.classpath +.project +*.code-workspace +*.DS_Store +*.iml +*.class +*.ipr +*.iws +*.gemspec +Gemfile* +.gitattributes +.settings/ +.settings +gradle.properties +*.gem \ No newline at end of file diff --git a/filter-plugin/logstash-filter-adabas-guardium/IMPLEMENTATION_GUIDE.md b/filter-plugin/logstash-filter-adabas-guardium/IMPLEMENTATION_GUIDE.md new file mode 100644 index 000000000..97b8790f6 --- /dev/null +++ b/filter-plugin/logstash-filter-adabas-guardium/IMPLEMENTATION_GUIDE.md @@ -0,0 +1,1262 @@ +# Adabas Universal Connector Implementation Guide for Guardium Data Protection + +## Table of Contents +1. [Overview](#overview) +2. [Architecture](#architecture) +3. [Prerequisites](#prerequisites) +4. [Implementation Steps](#implementation-steps) + - [Step 1: Configure Adabas Auditing Server](#step-1-configure-adabas-auditing-server) + - [Step 2: Obtain or Build the Plugins](#step-2-obtain-or-build-the-plugins) + - [Step 3: Install Plugins on Guardium](#step-3-install-plugins-on-guardium) + - [Step 4: Configure Universal Connector](#step-4-configure-universal-connector) + - [Step 5: Enable Universal Connector](#step-5-enable-universal-connector) + - [Step 6: Verify Data Flow](#step-6-verify-data-flow) +5. [Configuration Reference](#configuration-reference) +6. [Troubleshooting](#troubleshooting) +7. [Appendix](#appendix) + +--- + +## Overview + +This guide provides step-by-step instructions for implementing the Adabas Universal Connector with IBM Guardium Data Protection. The Adabas connector enables Guardium to monitor and audit Adabas database activity without requiring traditional S-TAP agents. + +### What is the Universal Connector? + +The Guardium Universal Connector is a framework that allows Guardium to receive and process audit data from various data sources through their native audit logs. It uses a Logstash-based pipeline with three components: + +1. **Input Plugin** - Receives audit data from the data source +2. **Filter Plugin** - Parses and transforms the data into Guardium format +3. **Output Plugin** - Sends processed data to Guardium (internal component) + +### Adabas-Specific Architecture + +The Adabas implementation uses a **custom input plugin** that connects directly to the Adabas Auditing Server via EntireX Broker, rather than using standard log forwarding methods like Syslog or Filebeat. + +**Key Components:** +- **Adabas Auditing Server** - Generates audit events from Adabas database activity +- **EntireX Broker** - Messaging middleware that facilitates communication +- **Adabas Input Plugin** (`logstash-input-adabas_auditing_input`) - Connects to the broker and retrieves audit messages +- **Adabas Filter Plugin** (`logstash-filter-adabas_guardium_filter`) - Parses audit data into Guardium format +- **Guardium Universal Connector** - Hosts the plugins and forwards data to Guardium + +--- + +## Architecture + +``` +┌─────────────────────────┐ +│ Adabas Database │ +│ │ +└───────────┬─────────────┘ + │ + │ Audit Events + ▼ +┌─────────────────────────┐ +│ Adabas Auditing Server │ +│ │ +└───────────┬─────────────┘ + │ + │ EntireX Protocol + ▼ +┌─────────────────────────┐ +│ EntireX Broker │ +│ (Messaging Layer) │ +└───────────┬─────────────┘ + │ + │ Broker Messages + ▼ +┌─────────────────────────────────────────────┐ +│ Guardium Universal Connector │ +│ │ +│ ┌─────────────────────────────────────┐ │ +│ │ Input Plugin │ │ +│ │ (adabas_auditing_input) │ │ +│ │ - Connects to EntireX Broker │ │ +│ │ - Retrieves audit messages │ │ +│ │ - Parses binary format │ │ +│ └──────────────┬──────────────────────┘ │ +│ │ │ +│ ▼ │ +│ ┌─────────────────────────────────────┐ │ +│ │ Filter Plugin │ │ +│ │ (adabas_guardium_filter) │ │ +│ │ - Transforms to Guardium format │ │ +│ │ - Extracts metadata │ │ +│ │ - Handles errors │ │ +│ └──────────────┬──────────────────────┘ │ +│ │ │ +│ ▼ │ +│ ┌─────────────────────────────────────┐ │ +│ │ Output Plugin (Internal) │ │ +│ │ - Sends to Guardium Sniffer │ │ +│ └─────────────────────────────────────┘ │ +└─────────────────┬───────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────┐ +│ Guardium Data Protection │ +│ - Policy Enforcement │ +│ - Reporting & Analytics │ +│ - Alerting │ +└─────────────────────────────────────────────┘ +``` + +--- + +## Prerequisites + +### System Requirements + +#### Guardium System +- **Guardium Data Protection Version:** 12.0 or later +- **Deployment Type:** Standalone system or Collector +- **User Permissions:** S-TAP Management Application role +- **Network Access:** Connectivity to Adabas Auditing Server's EntireX Broker + +#### Adabas Environment +- **Adabas Version:** Compatible version with Auditing Server support +- **Adabas Auditing Server:** Installed and configured +- **EntireX Broker:** Running and accessible +- **Network Configuration:** Firewall rules allowing Guardium to connect to broker port (default: 3000) + +### Software Requirements + +#### For Using Pre-Built Plugins (Recommended) +- Pre-built plugin files (`.gem` files) provided by IBM or Software AG: + - `logstash-input-adabas_auditing_input--java.gem` + - `logstash-filter-adabas_guardium_filter--java.gem` + +#### For Building Plugins from Source (Optional) +- **Java Development Kit (JDK):** Version 11 or later +- **Gradle:** Version 6.x or later (included via wrapper) +- **Logstash:** Version 7.5.2 or compatible +- **Git:** For cloning repositories +- **Build Dependencies:** + - Guardium Universal Connector Commons library + - Adabas SDK libraries (provided by Software AG) + - EntireX libraries (provided by Software AG) + +### Required Information + +Before starting, gather the following information: + +| Information | Description | Example | +|------------|-------------|---------| +| **Broker Host** | Hostname or IP of EntireX Broker | `adabas-broker.company.com` | +| **Broker Port** | Port number for broker connection | `3000` | +| **Broker Class** | Broker class identifier | `ADABAS-AUDIT` | +| **Broker Server** | Broker server name | `AUDIT-SERVER` | +| **Broker Service** | Service name for audit data | `AUDIT-SERVICE` | +| **User** | Authentication user for broker | `guardium_user` | +| **Token** | Authentication token/password | `` | +| **Guardium Collector IP** | IP address of Guardium system | `10.0.1.100` | + +### Network Requirements + +Ensure the following network connectivity: + +``` +Guardium Collector → EntireX Broker +- Protocol: TCP +- Port: 3000 (default, may vary) +- Direction: Outbound from Guardium +``` + +--- + +## Implementation Steps + +### Step 1: Configure Adabas Auditing Server + +The Adabas Auditing Server must be properly configured to generate and publish audit events. + +#### 1.1 Enable Adabas Auditing + +**Note:** This step is typically performed by your Adabas administrator. Consult Software AG documentation for detailed Adabas configuration. + +1. **Enable Auditing on Adabas Database:** + - Configure the Adabas database to generate audit records + - Specify which operations to audit (reads, writes, DDL, etc.) + - Set audit detail level + +2. **Configure Adabas Auditing Server:** + - Install the Adabas Auditing Server component + - Configure connection to the Adabas database + - Set up audit event collection parameters + +#### 1.2 Configure EntireX Broker + +The EntireX Broker acts as the messaging middleware between Adabas and Guardium. + +1. **Verify Broker is Running:** + ```bash + # Check broker status (command may vary by platform) + etbinfo -b + ``` + +2. **Configure Broker Service:** + - Create or identify the service that will publish audit events + - Note the broker class, server, and service names + - Configure authentication if required + +3. **Test Broker Connectivity:** + ```bash + # Test connection from Guardium server + telnet + ``` + +#### 1.3 Verify Audit Data Flow + +Before proceeding, verify that audit data is being generated: + +1. Perform some database operations on Adabas +2. Check that audit events are being published to the broker +3. Use EntireX tools to verify message flow + +--- + +### Step 2: Obtain or Build the Plugins + +You have two options: use pre-built plugins or build from source. + +#### Option A: Using Pre-Built Plugins (Recommended) + +If IBM or Software AG has provided pre-built plugin files: + +1. **Obtain the Plugin Files:** + - `logstash-input-adabas_auditing_input--java.gem` + - `logstash-filter-adabas_guardium_filter--java.gem` + +2. **Transfer to Guardium System:** + ```bash + # Copy files to Guardium (from your local machine) + scp logstash-input-adabas_auditing_input-*.gem guardium@:/tmp/ + scp logstash-filter-adabas_guardium_filter-*.gem guardium@:/tmp/ + ``` + +3. **Skip to Step 3** (Installation) + +#### Option B: Building Plugins from Source + +If you need to build the plugins yourself: + +##### 2.1 Prepare Build Environment + +1. **Install Java 11:** + ```bash + # Verify Java installation + java -version + # Should show Java 11 or later + ``` + +2. **Download Logstash:** + ```bash + # Download Logstash 7.5.2 or compatible version + wget https://artifacts.elastic.co/downloads/logstash/logstash-7.5.2.tar.gz + tar -xzf logstash-7.5.2.tar.gz + export LOGSTASH_HOME=/path/to/logstash-7.5.2 + ``` + +3. **Obtain Required Files:** + - Download `rubyUtils.gradle` and `versions.yml` from Logstash GitHub + - Copy to your Logstash installation directory + +4. **Fix rubyUtils.gradle Issues:** + + Edit `rubyUtils.gradle` and make these changes: + + **Issue 1 - Fix JRuby Version:** + ```gradle + // Find this line (around line 20): + classpath "org.jruby:jruby-core:${gradle.ext.versions.jruby.version}" + + // Replace with actual version from versions.yml: + classpath "org.jruby:jruby-core:9.4.13.0" + ``` + + **Issue 2 - Add YAML Parsing:** + ```gradle + // Add this code after the Ruby variables section: + + // Ruby variables + def versionsPath = project.hasProperty("LOGSTASH_CORE_PATH") ? LOGSTASH_CORE_PATH + "/../versions.yml" : "${projectDir}/versions.yml" + + // Add YAML parsing code below: + def versionsFile = new File(versionsPath) + if (!versionsFile.exists()) { + throw new GradleException("versions.yml file not found at: ${versionsPath}") + } + + def versionsData = [:] + def currentSection = null + versionsFile.eachLine { line -> + def trimmed = line.trim() + if (trimmed && !trimmed.startsWith('#')) { + if (!trimmed.startsWith(' ') && trimmed.endsWith(':')) { + currentSection = trimmed.replaceAll(':', '') + versionsData[currentSection] = [:] + } else if (trimmed.startsWith('version:') || trimmed.startsWith('sha256:')) { + def parts = trimmed.split(':', 2) + if (parts.length == 2 && currentSection) { + versionsData[currentSection][parts[0].trim()] = parts[1].trim() + } + } + } + } + + gradle.ext.versions = versionsData + versionMap = gradle.ext.versions + ``` + +##### 2.2 Build Guardium Commons Library + +1. **Clone the Commons Repository:** + ```bash + git clone https://github.com/IBM/guardium-universalconnector-commons.git + cd guardium-universalconnector-commons + ``` + +2. **Build the Commons JARs:** + ```bash + # Follow the README instructions to build + ./gradlew build + # Note the location of generated JAR files + ``` + +##### 2.3 Build Input Plugin + +1. **Clone or Navigate to Input Plugin:** + ```bash + cd /path/to/universal-connectors/input-plugin/logstash-input-adabas + ``` + +2. **Create gradle.properties:** + ```bash + cat > gradle.properties << EOF + LOGSTASH_CORE_PATH=/path/to/logstash-7.5.2/logstash-core + EOF + ``` + +3. **Build the Plugin:** + ```bash + ./gradlew assemble gem + ``` + +4. **Verify Build:** + ```bash + # Check for generated .gem file + ls -l logstash-input-adabas_auditing_input-*.gem + ``` + +##### 2.4 Build Filter Plugin + +1. **Navigate to Filter Plugin:** + ```bash + cd /path/to/universal-connectors/filter-plugin/logstash-filter-adabas-guardium + ``` + +2. **Create gradle.properties:** + ```bash + cat > gradle.properties << EOF + LOGSTASH_CORE_PATH=/path/to/logstash-7.5.2/logstash-core + GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH=/path/to/guardium-universalconnector-commons/build/libs + EOF + ``` + +3. **Build the Plugin:** + ```bash + ./gradlew assemble gem + ``` + +4. **Verify Build:** + ```bash + # Check for generated .gem file + ls -l logstash-filter-adabas_guardium_filter-*.gem + ``` + +##### 2.5 Transfer Built Plugins to Guardium + +```bash +# Copy both .gem files to Guardium +scp logstash-input-adabas_auditing_input-*.gem guardium@:/tmp/ +scp logstash-filter-adabas_guardium_filter-*.gem guardium@:/tmp/ +``` + +--- + +### Step 3: Install Plugins on Guardium + +Now install the plugins on your Guardium system. + +#### 3.1 Access Guardium CLI + +1. **SSH to Guardium:** + ```bash + ssh guardium@ + ``` + +2. **Switch to CLI Mode:** + ```bash + # If prompted, enter CLI mode + cli + ``` + +#### 3.2 Install Input Plugin + +1. **Install the Input Plugin:** + ```bash + grdapi install_universal_connector_plugin \ + plugin_file=/tmp/logstash-input-adabas_auditing_input--java.gem + ``` + +2. **Verify Installation:** + ```bash + # List installed plugins + grdapi list_universal_connector_plugins + ``` + + Expected output should include: + ``` + adabas_auditing_input + ``` + +#### 3.3 Install Filter Plugin + +1. **Install the Filter Plugin:** + ```bash + grdapi install_universal_connector_plugin \ + plugin_file=/tmp/logstash-filter-adabas_guardium_filter--java.gem + ``` + +2. **Verify Installation:** + ```bash + # List installed plugins + grdapi list_universal_connector_plugins + ``` + + Expected output should include: + ``` + adabas_auditing_input + adabas_guardium_filter + ``` + +#### 3.4 Clean Up + +```bash +# Remove temporary .gem files +rm /tmp/logstash-input-adabas_auditing_input-*.gem +rm /tmp/logstash-filter-adabas_guardium_filter-*.gem +``` + +--- + +### Step 4: Configure Universal Connector + +Configure the Universal Connector to use the Adabas plugins. + +#### 4.1 Access Universal Connector Configuration + +1. **Log in to Guardium Web UI:** + - Open browser: `https://:8443` + - Enter credentials + +2. **Navigate to Universal Connector:** + - Go to: **Setup** → **Tools and Views** → **Configure Universal Connector** + +#### 4.2 Create Connector Configuration + +1. **Click "Add" to Create New Connector** + +2. **Enter Connector Name:** + - Name: `Adabas_Production` (or your preferred name) + - Description: `Adabas database auditing via EntireX Broker` + +3. **Configure Input Section:** + + Click on the **Input** tab and enter: + + ```ruby + input { + adabas_auditing_input { + # EntireX Broker connection details + host => "adabas-broker.company.com" + port => 3000 + + # Broker service identification + brokerClass => "ADABAS-AUDIT" + brokerServer => "AUDIT-SERVER" + brokerService => "AUDIT-SERVICE" + + # Authentication + user => "guardium_user" + token => "your_secure_token" + + # Optional: Connection parameters + retryInterval => 5 + retryCount => 10 + waitTime => 30 + receiveLength => 32767 + compression => 0 + + # Optional: Metadata REST server URL + # restURL => "http://metadata-server:8080" + } + } + ``` + + **Parameter Descriptions:** + + | Parameter | Required | Description | Default | + |-----------|----------|-------------|---------| + | `host` | Yes | EntireX Broker hostname or IP | `localhost` | + | `port` | Yes | EntireX Broker port | `3000` | + | `brokerClass` | Yes | Broker class identifier | `class` | + | `brokerServer` | Yes | Broker server name | `server` | + | `brokerService` | Yes | Service name for audit data | `service` | + | `user` | Yes | Authentication username | `user` | + | `token` | Yes | Authentication token/password | `token` | + | `retryInterval` | No | Retry interval in seconds | `5` | + | `retryCount` | No | Number of retry attempts | `10` | + | `waitTime` | No | Wait time in seconds | `30` | + | `receiveLength` | No | Maximum message receive length | `32767` | + | `compression` | No | Compression level (0=none) | `0` | + | `restURL` | No | Metadata REST server URL | `""` | + +4. **Configure Filter Section:** + + Click on the **Filter** tab and enter: + + ```ruby + filter { + adabas_guardium_filter { + # Source field containing audit data + source => "adabas-auditing" + } + } + ``` + + **Note:** The `source` parameter should match the field name used by the input plugin (default: `adabas-auditing`). + +5. **Review Configuration:** + - Verify all parameters are correct + - Check for syntax errors (Guardium will validate) + +6. **Save Configuration:** + - Click **Save** + - Guardium will validate the configuration + - If validation fails, review error messages and correct + +#### 4.3 Using Secrets for Sensitive Data (Recommended) + +For production environments, use Guardium's keystore for sensitive information: + +1. **Create Secrets via CLI:** + ```bash + # SSH to Guardium + ssh guardium@ + + # Add broker credentials to keystore + grdapi universal_connector_keystore_add \ + key=ADABAS_BROKER_USER \ + password=guardium_user + + grdapi universal_connector_keystore_add \ + key=ADABAS_BROKER_TOKEN \ + password=your_secure_token + ``` + +2. **Verify Secrets:** + ```bash + grdapi universal_connector_keystore_list + ``` + +3. **Update Input Configuration to Use Secrets:** + ```ruby + input { + adabas_auditing_input { + host => "adabas-broker.company.com" + port => 3000 + brokerClass => "ADABAS-AUDIT" + brokerServer => "AUDIT-SERVER" + brokerService => "AUDIT-SERVICE" + + # Use environment variables from keystore + user => "${ADABAS_BROKER_USER}" + token => "${ADABAS_BROKER_TOKEN}" + + retryInterval => 5 + retryCount => 10 + waitTime => 30 + } + } + ``` + +4. **Save Updated Configuration** + +--- + +### Step 5: Enable Universal Connector + +#### 5.1 Enable via Web UI + +1. **In Configure Universal Connector Page:** + - Verify your connector configuration is listed + - Click **Enable** button + +2. **Wait for Startup:** + - Universal Connector will start (takes 1-2 minutes) + - Status will change from "Disabled" to "Enabled" + +#### 5.2 Enable via CLI (Alternative) + +```bash +# SSH to Guardium +ssh guardium@ + +# Enable Universal Connector +grdapi run_universal_connector +``` + +#### 5.3 Enable with Debug Logging (For Troubleshooting) + +```bash +# Enable with debug level logging +grdapi run_universal_connector debug_level=2 +``` + +**Debug Levels:** +- `0` - Errors only (default) +- `1` - Warnings and errors +- `2` - Info, warnings, and errors +- `3` - Debug (verbose) + +#### 5.4 Verify Universal Connector Status + +1. **Check Status in Web UI:** + - The **Enable** button should change to **Disable** + - Status indicator should be green + +2. **Check Status via CLI:** + ```bash + grdapi get_universal_connector_status + ``` + + Expected output: + ``` + Status: Running + Connectors: 1 + ``` + +--- + +### Step 6: Verify Data Flow + +#### 6.1 Generate Test Activity + +1. **Perform Operations on Adabas:** + - Connect to your Adabas database + - Execute some queries (SELECT, INSERT, UPDATE) + - Perform administrative operations if applicable + +2. **Wait for Processing:** + - Allow 1-2 minutes for data to flow through the pipeline + +#### 6.2 Check S-TAP Status Page + +1. **Navigate to S-TAP Status:** + - Go to: **Monitor** → **S-TAP Status** + +2. **Look for Adabas Connector:** + - S-TAP Host format: `::UC` + - S-TAP Version: `Universal connector V` + - Status should be **Green** (active) + +3. **Verify Data Flow:** + - Check "Last Contact" timestamp (should be recent) + - Check "Messages" count (should be increasing) + +#### 6.3 Check Guardium Logs + +1. **Access Universal Connector Logs:** + ```bash + # SSH to Guardium + ssh guardium@ + + # View recent log entries + grdapi tail_universal_connector_log lines=100 + ``` + +2. **Look for Success Indicators:** + - Connection to broker established + - Messages being received + - No error messages + +3. **Check for Errors:** + ```bash + # Search for errors in logs + grdapi tail_universal_connector_log lines=500 | grep -i error + ``` + +#### 6.4 Verify Data in Guardium Reports + +1. **Run Activity Report:** + - Go to: **Reports** → **Activity** → **Activity Report** + - Set time range to last hour + - Filter by database type: Adabas + +2. **Check for Audit Records:** + - Verify records are appearing + - Check that user names are populated + - Verify SQL/commands are captured + - Confirm timestamps are correct + +3. **Review Session Details:** + - Go to: **Monitor** → **Sessions** + - Look for Adabas sessions + - Verify session information is complete + +#### 6.5 Test Policy Enforcement + +1. **Create Test Policy:** + - Go to: **Policy** → **Policy Builder** + - Create a simple policy (e.g., alert on SELECT statements) + - Apply to Adabas data source + +2. **Trigger Policy:** + - Execute operations that match the policy + - Wait for policy evaluation + +3. **Check Alerts:** + - Go to: **Monitor** → **Alerts** + - Verify alerts are generated for Adabas activity + +--- + +## Configuration Reference + +### Complete Configuration Example + +Here's a complete, production-ready configuration: + +```ruby +# ============================================ +# INPUT SECTION +# ============================================ +input { + adabas_auditing_input { + # Broker Connection + host => "adabas-broker.company.com" + port => 3000 + + # Service Identification + brokerClass => "ADABAS-AUDIT" + brokerServer => "AUDIT-SERVER" + brokerService => "AUDIT-SERVICE" + + # Authentication (using keystore) + user => "${ADABAS_BROKER_USER}" + token => "${ADABAS_BROKER_TOKEN}" + + # Connection Tuning + retryInterval => 5 # Retry every 5 seconds on failure + retryCount => 10 # Retry up to 10 times + waitTime => 30 # Wait 30 seconds for messages + receiveLength => 32767 # Maximum message size + compression => 0 # No compression + + # Optional: Metadata Server + # restURL => "http://metadata-server:8080" + } +} + +# ============================================ +# FILTER SECTION +# ============================================ +filter { + adabas_guardium_filter { + source => "adabas-auditing" + } + + # Optional: Add custom fields + mutate { + add_field => { + "environment" => "production" + "data_center" => "DC1" + } + } +} +``` + +### Environment Variables Reference + +When using the keystore, these environment variables are available: + +| Variable Name | Purpose | Example Value | +|--------------|---------|---------------| +| `ADABAS_BROKER_USER` | Broker authentication username | `guardium_user` | +| `ADABAS_BROKER_TOKEN` | Broker authentication token | `` | +| `ADABAS_BROKER_HOST` | Broker hostname (optional) | `adabas-broker.company.com` | +| `ADABAS_METADATA_URL` | Metadata REST server URL (optional) | `http://metadata:8080` | + +### Guardium Record Structure + +The filter plugin transforms Adabas audit data into this Guardium structure: + +```json +{ + "sessionId": "mV20eHvvRha2ELTeqJxQJg==", + "dbName": "ADABAS-DB-001", + "appUserName": "APP_USER", + "time": { + "timestamp": 1705751051070, + "minOffsetFromGMT": -240, + "minDst": 0 + }, + "accessor": { + "dbUser": "NATUID_VALUE", + "serverType": "Adabas", + "serverHostName": "LPAR_NAME", + "sourceProgram": "NATPROG_NAME", + "language": "FREE_TEXT", + "dataType": "CONSTRUCT", + "dbProtocol": "Adabas native audit" + }, + "data": { + "construct": { + "sentences": [ + { + "verb": "READ", + "fields": ["FIELD1", "FIELD2", "FIELD3"] + } + ] + } + }, + "exception": { + "exceptionTypeId": "SQL_ERROR", + "description": "Response Code 3 (0) received.", + "sqlString": "READ with ISN 12345" + } +} +``` + +--- + +## Troubleshooting + +### Common Issues and Solutions + +#### Issue 1: Universal Connector Won't Start + +**Symptoms:** +- Status remains "Disabled" after clicking Enable +- Error in logs: "Failed to start Universal Connector" + +**Solutions:** + +1. **Check Plugin Installation:** + ```bash + grdapi list_universal_connector_plugins + ``` + Verify both `adabas_auditing_input` and `adabas_guardium_filter` are listed. + +2. **Check Configuration Syntax:** + - Review input and filter sections for typos + - Ensure all required parameters are present + - Verify quotes and brackets are balanced + +3. **Check Logs for Details:** + ```bash + grdapi tail_universal_connector_log lines=200 + ``` + +4. **Restart with Overwrite:** + ```bash + grdapi run_universal_connector overwrite_old_instance="true" + ``` + +#### Issue 2: Cannot Connect to EntireX Broker + +**Symptoms:** +- Log message: "Failed to connect to broker" +- Log message: "Connection refused" +- No data flowing to Guardium + +**Solutions:** + +1. **Verify Network Connectivity:** + ```bash + # From Guardium CLI + telnet + ``` + +2. **Check Firewall Rules:** + - Ensure Guardium can reach broker port (default: 3000) + - Check both outbound (Guardium) and inbound (broker) rules + +3. **Verify Broker is Running:** + ```bash + # On broker server + etbinfo -b + ``` + +4. **Check Broker Configuration:** + - Verify broker class, server, and service names + - Ensure service is registered and active + +5. **Test with Broker Tools:** + - Use EntireX tools to verify broker accessibility + - Test authentication credentials + +#### Issue 3: Authentication Failures + +**Symptoms:** +- Log message: "Authentication failed" +- Log message: "Invalid credentials" +- Connection established but no data received + +**Solutions:** + +1. **Verify Credentials:** + - Check username and token are correct + - Ensure no extra spaces or special characters + +2. **Check Keystore Values:** + ```bash + grdapi universal_connector_keystore_list + ``` + Verify keys exist and are spelled correctly. + +3. **Test Credentials Manually:** + - Use EntireX tools to test authentication + - Verify user has permissions to access audit service + +4. **Update Credentials:** + ```bash + # Remove old key + grdapi universal_connector_keystore_remove key=ADABAS_BROKER_TOKEN + + # Add new key + grdapi universal_connector_keystore_add \ + key=ADABAS_BROKER_TOKEN \ + password=new_token + + # Restart UC + grdapi run_universal_connector overwrite_old_instance="true" + ``` + +#### Issue 4: No Audit Data Appearing + +**Symptoms:** +- Universal Connector is running +- Connection to broker successful +- No records in Guardium reports + +**Solutions:** + +1. **Verify Audit Data is Being Generated:** + - Check Adabas Auditing Server is running + - Perform database operations + - Verify audit events are published to broker + +2. **Check Filter Processing:** + ```bash + # Enable debug logging + grdapi run_universal_connector debug_level=3 + + # Check logs for filter activity + grdapi tail_universal_connector_log lines=500 | grep -i filter + ``` + +3. **Look for Skipped Events:** + ```bash + # Check for skip tags + grdapi tail_universal_connector_log lines=500 | grep -i skip + ``` + +4. **Verify Source Field:** + - Ensure filter `source` parameter matches input plugin output + - Default is `adabas-auditing` + +5. **Check for Parsing Errors:** + ```bash + # Look for parsing errors + grdapi tail_universal_connector_log lines=500 | grep -i "parse\|error" + ``` + +#### Issue 5: Incomplete or Missing Data Fields + +**Symptoms:** +- Records appear in Guardium but missing information +- User names showing as empty +- Database names not populated + +**Solutions:** + +1. **Check Adabas Audit Configuration:** + - Verify audit level captures required fields + - Ensure user context is included in audit events + +2. **Review Audit Event Structure:** + - Check that CLNT (client) data is present + - Verify ACBX (control block) data is included + +3. **Check Metadata Configuration:** + - If using metadata REST server, verify it's accessible + - Check `restURL` parameter is correct + +4. **Review Parser Logic:** + - Check filter plugin logs for warnings + - Verify all expected fields are being extracted + +#### Issue 6: Performance Issues + +**Symptoms:** +- High CPU usage on Guardium +- Slow report generation +- Delayed data processing + +**Solutions:** + +1. **Adjust Connection Parameters:** + ```ruby + input { + adabas_auditing_input { + # Increase wait time to reduce polling frequency + waitTime => 60 + + # Increase receive length for batch processing + receiveLength => 65535 + } + } + ``` + +2. **Enable Compression:** + ```ruby + input { + adabas_auditing_input { + compression => 1 # Enable compression + } + } + ``` + +3. **Filter Audit Events at Source:** + - Configure Adabas to audit only necessary operations + - Reduce audit detail level if appropriate + +4. **Check Guardium Resources:** + - Monitor CPU and memory usage + - Consider adding more collectors for load balancing + +#### Issue 7: Universal Connector Stops After Reboot + +**Symptoms:** +- Universal Connector not running after Guardium restart +- Must manually enable after reboot + +**Solution:** + +After each Guardium reboot, restart the Universal Connector: + +```bash +# SSH to Guardium +ssh guardium@ + +# Start Universal Connector +grdapi run_universal_connector +``` + +**Note:** This is expected behavior. Universal Connector must be manually started after system reboots. + +### Diagnostic Commands + +#### Check Universal Connector Status +```bash +grdapi get_universal_connector_status +``` + +#### View Recent Logs +```bash +# Last 100 lines +grdapi tail_universal_connector_log lines=100 + +# Last 500 lines with timestamps +grdapi tail_universal_connector_log lines=500 +``` + +#### List Installed Plugins +```bash +grdapi list_universal_connector_plugins +``` + +#### List Connector Configurations +```bash +# Via Web UI: Setup → Tools and Views → Configure Universal Connector +``` + +#### Check Keystore Contents +```bash +grdapi universal_connector_keystore_list +``` + +#### Restart Universal Connector +```bash +# Normal restart +grdapi run_universal_connector + +# Force restart (overwrite existing instance) +grdapi run_universal_connector overwrite_old_instance="true" + +# Restart with debug logging +grdapi run_universal_connector debug_level=3 +``` + +#### Stop Universal Connector +```bash +grdapi stop_universal_connector +``` + +### Log File Locations + +When creating a MustGather, these log files are included: + +- **Universal Connector Log:** `uc-logstash.log` +- **Logstash Standard Output:** `logstash_stdout_stderr.log` +- **Guardium System Logs:** Various system logs + +### Getting Help + +If you continue to experience issues: + +1. **Create a MustGather:** + - Go to: **Setup** → **Tools and Views** → **MustGather** + - Select all relevant options + - Include Universal Connector logs + +2. **Contact IBM Support:** + - Provide MustGather output + - Include configuration details (sanitize sensitive data) + - Describe the issue and steps to reproduce + +3. **Check Documentation:** + - [IBM Guardium Documentation](https://www.ibm.com/docs/en/guardium) + - [Universal Connectors GitHub](https://github.com/IBM/universal-connectors) + - Software AG Adabas documentation + +--- + +## Appendix + +### A. Glossary + +| Term | Definition | +|------|------------| +| **Adabas** | A high-performance database management system by Software AG | +| **Adabas Auditing Server** | Component that captures and publishes Adabas audit events | +| **EntireX Broker** | Messaging middleware for communication between Adabas and Guardium | +| **Universal Connector** | Guardium framework for ingesting data from various sources | +| **Input Plugin** | Component that receives data from external sources | +| **Filter Plugin** | Component that parses and transforms data | +| **S-TAP** | Software Tap - traditional Guardium monitoring agent | +| **Guardium Record** | Standardized data structure for audit events in Guardium | +| **ACBX** | Adabas Control Block Extended - contains command details | +| **CLNT** | Client information block in Adabas audit data | + +### B. Port Reference + +| Port | Protocol | Purpose | Direction | +|------|----------|---------|-----------| +| 3000 | TCP | EntireX Broker (default) | Guardium → Broker | +| 8443 | HTTPS | Guardium Web UI | Admin → Guardium | +| 22 | SSH | Guardium CLI access | Admin → Guardium | + +### C. File Locations + +| File/Directory | Purpose | +|----------------|---------| +| `/tmp/` | Temporary location for plugin .gem files | +| Guardium internal | Universal Connector logs (accessed via grdapi) | +| Guardium internal | Plugin installation directory (managed by Guardium) | + +### D. Related Documentation + +- [Guardium Data Protection Documentation](https://www.ibm.com/docs/en/guardium) +- [Universal Connectors GitHub Repository](https://github.com/IBM/universal-connectors) +- [Developing Plugins for Guardium](https://github.com/IBM/universal-connectors/blob/main/docs/Guardium%20Data%20Protection/developing_plugins_gdp.md) +- [Configuring Universal Connector](https://github.com/IBM/universal-connectors/blob/main/docs/Guardium%20Data%20Protection/uc_config_gdp.md) +- Software AG Adabas Documentation +- Software AG EntireX Documentation + +### E. Quick Reference Card + +#### Essential Commands + +```bash +# Enable Universal Connector +grdapi run_universal_connector + +# Check status +grdapi get_universal_connector_status + +# View logs +grdapi tail_universal_connector_log lines=100 + +# List plugins +grdapi list_universal_connector_plugins + +# Stop Universal Connector +grdapi stop_universal_connector + +# Restart with overwrite +grdapi run_universal_connector overwrite_old_instance="true" +``` + +#### Configuration Template + +```ruby +input { + adabas_auditing_input { + host => "BROKER_HOST" + port => BROKER_PORT + brokerClass => "BROKER_CLASS" + brokerServer => "BROKER_SERVER" + brokerService => "BROKER_SERVICE" + user => "${ADABAS_BROKER_USER}" + token => "${ADABAS_BROKER_TOKEN}" + } +} + +filter { + adabas_guardium_filter { + source => "adabas-auditing" + } +} +``` + +### F. Version History + +| Version | Date | Changes | +|---------|------|---------| +| 1.0 | 2025-01 | Initial implementation guide | + +--- + +## Support and Feedback + +For questions, issues, or feedback regarding this implementation guide: + +- **IBM Support:** Contact your IBM support representative +- **GitHub Issues:** [Universal Connectors Issues](https://github.com/IBM/universal-connectors/issues) +- **Documentation Updates:** Submit pull requests to improve this guide + +--- + +**Document Information:** +- **Title:** Adabas Universal Connector Implementation Guide +- **Audience:** Guardium administrators, Database administrators +- **Prerequisites:** Basic knowledge of Guardium, Adabas, and networking +- **Estimated Implementation Time:** 2-4 hours (excluding Adabas configuration) + +--- + +*This guide is provided as-is and may be updated as new versions of Guardium or the Adabas plugins are released. Always refer to the latest official IBM and Software AG documentation for the most current information.* \ No newline at end of file diff --git a/filter-plugin/logstash-filter-adabas-guardium/LICENSE b/filter-plugin/logstash-filter-adabas-guardium/LICENSE new file mode 100644 index 000000000..a80a3fd53 --- /dev/null +++ b/filter-plugin/logstash-filter-adabas-guardium/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2020 Elastic and contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/filter-plugin/logstash-filter-adabas-guardium/README.md b/filter-plugin/logstash-filter-adabas-guardium/README.md new file mode 100644 index 000000000..998a77b73 --- /dev/null +++ b/filter-plugin/logstash-filter-adabas-guardium/README.md @@ -0,0 +1,132 @@ +# Logstash Adabas Auditing Filter Plugin + +This is a Java plugin for [Logstash](https://github.com/elastic/logstash). + +## Build +The build of this plugin requires the access to an installation of Logstash. + +1. Download Logstash from https://www.elastic.co/downloads/logstash +2. Copy the files **rubyUtils.gradle** and **versions.yml** from Github repository https://github.com/elastic/logstash to directory where you installed Logstash + + **Note:** We've identified issues with the `rubyUtils.gradle` file from the Logstash GitHub repository that may cause build failures for this project. Please make the following modifications to the `rubyUtils.gradle` file: + + - Issue 1: JRuby Version Resolution + + **Problem:** Dynamic version reference fails during build + ```gradle + // Original (causes build failure) + classpath "org.jruby:jruby-core:${gradle.ext.versions.jruby.version}" + ``` + **Solution:** Use the actual version number of jruby from versions.yml, for example: + ```gradle + // Fixed version + classpath "org.jruby:jruby-core:9.4.13.0" + ``` + + - Issue 2: YAML Parsing + + **Problem:** Missing YAML parsing logic causes version resolution to fail + + **Solution:** Add the following YAML parsing code after the Ruby variables section: + + ```gradle + // Ruby variables + def versionsPath = project.hasProperty("LOGSTASH_CORE_PATH") ? LOGSTASH_CORE_PATH + "/../versions.yml" : "${projectDir}/versions.yml" + + // ⚠️Add this YAML parsing code below: + // Read and parse versions.yml without external dependencies + def versionsFile = new File(versionsPath) + if (!versionsFile.exists()) { + throw new GradleException("versions.yml file not found at: ${versionsPath}") + } + + // Simple YAML parsing for versions.yml structure + def versionsData = [:] + def currentSection = null + versionsFile.eachLine { line -> + def trimmed = line.trim() + if (trimmed && !trimmed.startsWith('#')) { + if (!trimmed.startsWith(' ') && trimmed.endsWith(':')) { + // Top level section + currentSection = trimmed.replaceAll(':', '') + versionsData[currentSection] = [:] + } else if (trimmed.startsWith('version:') || trimmed.startsWith('sha256:')) { + // Property in current section + def parts = trimmed.split(':', 2) + if (parts.length == 2 && currentSection) { + versionsData[currentSection][parts[0].trim()] = parts[1].trim() + } + } + } + } + + // Set gradle.ext.versions + gradle.ext.versions = versionsData + versionMap = gradle.ext.versions + ``` +3. Clone the [guardium-universalconnector-commons project](https://github.com/IBM/guardium-universalconnector-commons) from GitHub to get helper classes for creating a Guardium record. +4. Build the Guardium jars following the instructions of the README.md. (Java 11 was required to build the jars) +5. Clone this repository +6. Set the property variable **LOGSTASH_CORE_PATH**. This could be done in gradle.properties file +6. Set the property variable **GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH** to the directory of jars from step 4. This could be done in gradle.properties file +7. Assemble plugin with the command `./gradlew assemble gem` + +After that successful build a file **logstash-input-adabas_guardium_filter--java.gem** is created in the root directory of the project. + +See also [Developing new plug-ins for Guardium Data Protection](https://github.com/IBM/universal-connectors/blob/main/docs/Guardium%20Data%20Protection/developing_plugins_gdp.md). + +## Install Plugin +To install the plugin use the command +``` +logstash-plugin install --no-verify --local /logstash-input-adabas_guardium_filter--java.gem +``` + +## Run Logstash +Execute the command `logstash -f ` where ``is your Logstash configuration file. An example is below. + +## Plugin Configuration Example +This configuration reads the data from the Adabas Auditing Server and write the data to `elasticsearch` and `stdout`. + +``` +input { + adabas_auditing_input { + brokerClass => "class" + brokerServer => "server" + brokerService => "service" + host => "host" + port => 3000 + token => "token" + user => "user" + } +} +filter { + adabas_guardium_filter { + } +} +output { + stdout { + codec => rubydebug + } +} +``` + +## Plugin Parameter +| Parameter | Description | Type | Default Value | +| ------------- | --------------------------- | ------ | ---------------- | +| host | Broker host | String | "localhost" | +| port | Broker port | Number | 3000 | +| brokerClass | Broker class name | String | "class" | +| brokerServer | Broker server name | String | "server" | +| brokerService | Broker service name | String | "service" | +| user | User | String | "user" | +| token | Token | String | "token" | +| retryInterval | Retry interval in seconds | Number | 5 | +| retryCount | Retry count | Number | 10 | +| waitTime | Wait time in seconds | Number | 30 | +| receiveLength | Receive length | Number | 32767 | +| compression | Compression | Number | 0 | +| restURL | URL of metadata REST server | String | "" | +| Hosts | Elasticsearch host | String | "localhost:9200" | + +## Environment Variable +Use the environment variable `REST_PATH` set the directory for the metadata outside of Logstash. diff --git a/filter-plugin/logstash-filter-adabas-guardium/VERSION b/filter-plugin/logstash-filter-adabas-guardium/VERSION new file mode 100644 index 000000000..afaf360d3 --- /dev/null +++ b/filter-plugin/logstash-filter-adabas-guardium/VERSION @@ -0,0 +1 @@ +1.0.0 \ No newline at end of file diff --git a/filter-plugin/logstash-filter-adabas-guardium/build.gradle b/filter-plugin/logstash-filter-adabas-guardium/build.gradle new file mode 100644 index 000000000..c75a7329d --- /dev/null +++ b/filter-plugin/logstash-filter-adabas-guardium/build.gradle @@ -0,0 +1,129 @@ +import java.nio.file.Files +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + +plugins { + id 'com.github.johnrengelman.shadow' version '8.1.1' + id 'java' +} + +ext { + snakeYamlVersion = '2.2' + shadowGradlePluginVersion = '8.1.1' +} + +def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + + +apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" + +// =========================================================================== +// plugin info +// =========================================================================== +group 'com.softwareag.adabas.auditing.logstash' // must match the package of the main plugin class +version "${file("VERSION").text.trim()}" // read from required VERSION file +description = "Adabas Auditing Filter Plugin for Logstash" +pluginInfo.licenses = ['Apache-2.0'] // list of SPDX license IDs +pluginInfo.longDescription = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using \$LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" +pluginInfo.authors = ['Software GmbH'] +pluginInfo.email = ['support@softwareag.com'] +pluginInfo.homepage = "https://www.softwareag.com" +pluginInfo.pluginType = "filter" +pluginInfo.pluginClass = "AdabasGuardiumFilter" +pluginInfo.pluginName = "adabas_guardium_filter" // must match the @LogstashPlugin annotation in the main plugin class +// =========================================================================== + +java { + sourceCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.VERSION_1_8 +} + +repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } +} + +shadowJar { + archiveClassifier.set('') +} + +dependencies { + implementation 'org.apache.commons:commons-lang3:3.7' + implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "**/logstash-core.jar") + implementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "guardium-universalconnector-commons-?.?.?.jar") + implementation 'com.google.code.gson:gson:2.8.6' + implementation 'org.apache.logging.log4j:log4j-api:2.17.0' // provided by Logstash + implementation 'org.apache.logging.log4j:log4j-core:2.17.0' // provided by Logstash + + testImplementation 'junit:junit:4.12' + testImplementation 'org.jruby:jruby-complete:9.2.20.1' +} + +clean { + delete "${projectDir}/Gemfile" + delete "${projectDir}/" + pluginInfo.pluginFullName() + ".gemspec" + delete "${projectDir}/lib/" + delete "${projectDir}/vendor/" + new FileNameFinder().getFileNames(projectDir.toString(), pluginInfo.pluginFullName() + "-?.?.?.gem").each { filename -> + delete filename + } +} + +tasks.withType(JavaCompile) { + options.encoding = 'UTF-8' +} + +tasks.register("vendor"){ + dependsOn shadowJar + doLast { + String vendorPathPrefix = "vendor/jar-dependencies" + String projectGroupPath = project.group.replaceAll('\\.', '/') + File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") + projectJarFile.mkdirs() + Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) + validatePluginJar(projectJarFile, project.group) + } +} + +tasks.register("generateRubySupportFiles") { + doLast { + generateRubySupportFilesForPlugin(project.description, project.group, version) + } +} + +tasks.register("removeObsoleteJars") { + doLast { + new FileNameFinder().getFileNames( + projectDir.toString(), + "vendor/**/" + pluginInfo.pluginFullName() + "*.jar", + "vendor/**/" + pluginInfo.pluginFullName() + "-" + version + ".jar").each { f -> + delete f + } + } +} + +tasks.register("gem"){ + dependsOn = [downloadAndInstallJRuby, removeObsoleteJars, vendor, generateRubySupportFiles] + doLast { + buildGem(projectDir, buildDir, pluginInfo.pluginFullName() + ".gemspec") + } +} diff --git a/filter-plugin/logstash-filter-adabas-guardium/docs/index.asciidoc b/filter-plugin/logstash-filter-adabas-guardium/docs/index.asciidoc new file mode 100644 index 000000000..fca3730ac --- /dev/null +++ b/filter-plugin/logstash-filter-adabas-guardium/docs/index.asciidoc @@ -0,0 +1,87 @@ +:plugin: example +:type: filter +// Update header with plugin name + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: %VERSION% +:release_date: %RELEASE_DATE% +:changelog_url: %CHANGELOG_URL% +:include_path: ../../../../logstash/docs/include +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +[id="plugins-{type}s-{plugin}"] + +=== Example filter plugin + +include::{include_path}/plugin_header.asciidoc[] + +==== Description + +Add plugin description here + +// Format anchors and links to support generated ids for versioning +// Sample anchor: [id="plugins-{type}s-{plugin}-setting_name"] +// Sample link: <> + +[id="plugins-{type}s-{plugin}-options"] +==== Example Filter Configuration Options + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +| <> |<>|No +|======================================================================= + +[id="plugins-{type}s-{plugin}-a_setting_name"] +===== `a_setting_name` + + * Value type is <> + * Default value is `true` + +Add description here + +[id="plugins-{type}s-{plugin}-another_setting_name"] +===== `another_setting_name` + + * Value type is <> + * Default value is `{}` + +Add description here + +[id="plugins-{type}s-{plugin}-setting_name_3"] +===== `setting_name_3` + + * Value type is <> + * Default value is `{}` + +Add description here + +[id="plugins-{type}s-{plugin}-setting_name_4"] +===== `setting_name_4` + + * Value type is <> + * Default value is `0` + +Add description here + +[id="plugins-{type}s-{plugin}-setting_name_5"] +===== `setting_name_5` + + * Value type is <> + * Default value is {} + +Add description here + +// The full list of Value Types is here: +// https://www.elastic.co/guide/en/logstash/current/configuration-file-structure.html + +[id="plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/filter-plugin/logstash-filter-adabas-guardium/gradle/wrapper/gradle-wrapper.jar b/filter-plugin/logstash-filter-adabas-guardium/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 000000000..e6441136f Binary files /dev/null and b/filter-plugin/logstash-filter-adabas-guardium/gradle/wrapper/gradle-wrapper.jar differ diff --git a/filter-plugin/logstash-filter-adabas-guardium/gradle/wrapper/gradle-wrapper.properties b/filter-plugin/logstash-filter-adabas-guardium/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 000000000..b82aa23a4 --- /dev/null +++ b/filter-plugin/logstash-filter-adabas-guardium/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,7 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-bin.zip +networkTimeout=10000 +validateDistributionUrl=true +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/filter-plugin/logstash-filter-adabas-guardium/gradlew b/filter-plugin/logstash-filter-adabas-guardium/gradlew new file mode 100755 index 000000000..1aa94a426 --- /dev/null +++ b/filter-plugin/logstash-filter-adabas-guardium/gradlew @@ -0,0 +1,249 @@ +#!/bin/sh + +# +# Copyright © 2015-2021 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# +############################################################################## + +# Attempt to set APP_HOME + +# Resolve links: $0 may be a link +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/filter-plugin/logstash-filter-adabas-guardium/gradlew.bat b/filter-plugin/logstash-filter-adabas-guardium/gradlew.bat new file mode 100644 index 000000000..7101f8e46 --- /dev/null +++ b/filter-plugin/logstash-filter-adabas-guardium/gradlew.bat @@ -0,0 +1,92 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%"=="" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if %ERRORLEVEL% equ 0 goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if %ERRORLEVEL% equ 0 goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/filter-plugin/logstash-filter-adabas-guardium/src/main/java/com/softwareag/adabas/auditing/logstash/AdabasGuardiumFilter.java b/filter-plugin/logstash-filter-adabas-guardium/src/main/java/com/softwareag/adabas/auditing/logstash/AdabasGuardiumFilter.java new file mode 100644 index 000000000..234875981 --- /dev/null +++ b/filter-plugin/logstash-filter-adabas-guardium/src/main/java/com/softwareag/adabas/auditing/logstash/AdabasGuardiumFilter.java @@ -0,0 +1,97 @@ +/* + * Copyright © 2025 Software GmbH, Darmstadt, Germany and/or its licensors + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.softwareag.adabas.auditing.logstash; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.ibm.guardium.universalconnector.commons.GuardConstants; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import co.elastic.logstash.api.Configuration; +import co.elastic.logstash.api.Context; +import co.elastic.logstash.api.Event; +import co.elastic.logstash.api.Filter; +import co.elastic.logstash.api.FilterMatchListener; +import co.elastic.logstash.api.LogstashPlugin; +import co.elastic.logstash.api.PluginConfigSpec; + +// class name must match plugin name +@LogstashPlugin(name = "adabas_guardium_filter") +public class AdabasGuardiumFilter implements Filter { + + private static final Logger logger = LogManager.getLogger(AdabasGuardiumFilter.class); + + public static final PluginConfigSpec SOURCE_CONFIG = PluginConfigSpec.stringSetting("source", + "adabas-auditing"); + + private final String id; + private final String sourceField; + private final Parser parser; + + public AdabasGuardiumFilter(final String id, final Configuration config, final Context context) { + this.id = id; + this.sourceField = config.get(SOURCE_CONFIG); + this.parser = new Parser(); + logger.debug("Adabas Auditing Filter created with id: {} and source field: {}", id, sourceField); + } + + @SuppressWarnings("unchecked") + @Override + public Collection filter(final Collection events, final FilterMatchListener matchListener) { + for (final Event e : events) { + final Object f = e.getField(sourceField); + if (f instanceof HashMap) { + final HashMap map = (HashMap) f; + logger.debug("Event map: {}", map); + + Record record = parser.parseRecord(map); + + if (record.getAccessor() != null) { + final GsonBuilder builder = new GsonBuilder(); + builder.serializeNulls(); + final Gson gson = builder.create(); + e.setField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME, gson.toJson(record)); + e.setField("type", sourceField); + matchListener.filterMatched(e); + } else { + e.tag(Constants.LOGSTASH_TAG_SKIP_NOT_COMMAND); + } + } + } + return events; + } + + @Override + public Collection> configSchema() { + // should return a list of all configuration options for this plugin + return Collections.singletonList(SOURCE_CONFIG); + } + + @Override + public String getId() { + return this.id; + } +} diff --git a/filter-plugin/logstash-filter-adabas-guardium/src/main/java/com/softwareag/adabas/auditing/logstash/Constants.java b/filter-plugin/logstash-filter-adabas-guardium/src/main/java/com/softwareag/adabas/auditing/logstash/Constants.java new file mode 100644 index 000000000..8d96013a5 --- /dev/null +++ b/filter-plugin/logstash-filter-adabas-guardium/src/main/java/com/softwareag/adabas/auditing/logstash/Constants.java @@ -0,0 +1,50 @@ +/* + * Copyright © 2025 Software GmbH, Darmstadt, Germany and/or its licensors + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.softwareag.adabas.auditing.logstash; + +public final class Constants { + private Constants() {} + + public static final String LOGSTASH_TAG_SKIP_NOT_COMMAND = "_adabasguardium_skip_not_command"; + + public static final String DATA_PROTOCOL_STRING = "Adabas native audit"; + public static final String SERVER_TYPE_STRING = "Adabas"; + public static final String UNKNOWN_STRING = ""; + + // hashmap mappings + public static final String RECORD_SESSION_ID = "CMDID"; + public static final String RECORD_DB_NAME = "UABIDBID"; + public static final String RECORD_TIME = "UABHTIME"; + public static final String RECORD_APP_USER_NAME = "TPUSERID"; + + public static final String ACCESSOR_DB_USER = "NATUID"; + public static final String ACCESSOR_SERVER_HOST_NAME = "LPARNAME"; + public static final String ACCESSOR_SOURCE_PROGRAM = "NATPROG"; + + // ACBX fields + public static final String ACBX_RSP_CODE = "RSPCODE"; + public static final String ACBX_RSP_SUB_CODE = "RSPSUBCODE"; + public static final String ACBX_CMD_CODE = "CMDCODE"; + public static final String ACBX_ISN = "ISN"; + + // Error type + public static final String SQL_ERROR = "SQL_ERROR"; + +} diff --git a/filter-plugin/logstash-filter-adabas-guardium/src/main/java/com/softwareag/adabas/auditing/logstash/Parser.java b/filter-plugin/logstash-filter-adabas-guardium/src/main/java/com/softwareag/adabas/auditing/logstash/Parser.java new file mode 100644 index 000000000..31e0fdf55 --- /dev/null +++ b/filter-plugin/logstash-filter-adabas-guardium/src/main/java/com/softwareag/adabas/auditing/logstash/Parser.java @@ -0,0 +1,195 @@ +/* + * Copyright © 2025 Software GmbH, Darmstadt, Germany and/or its licensors + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.softwareag.adabas.auditing.logstash; + +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.HashMap; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.ibm.guardium.universalconnector.commons.structures.Accessor; +import com.ibm.guardium.universalconnector.commons.structures.Construct; +import com.ibm.guardium.universalconnector.commons.structures.Data; +import com.ibm.guardium.universalconnector.commons.structures.ExceptionRecord; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import com.ibm.guardium.universalconnector.commons.structures.Sentence; +import com.ibm.guardium.universalconnector.commons.structures.Time; + +public class Parser { + + private static final Logger logger = LogManager.getLogger(Parser.class); + + @SuppressWarnings("unchecked") + public Record parseRecord(HashMap map) { + final Record record = new Record(); + Accessor accessor = null; + Data data = null; + + if (map.containsKey("UABI_ITEMS")) { + // extract data from the map + final HashMap items = (HashMap) map.get("UABI_ITEMS"); + // data for the record + if (items.containsKey(Constants.RECORD_DB_NAME)) { + record.setDbName(items.get(Constants.RECORD_DB_NAME).toString()); + } + // extract data from ACBX or CLNT + if (items.containsKey("UABD_ITEMS")) { + final Object item = items.get("UABD_ITEMS"); + if (item instanceof ArrayList) { + final ArrayList> list = (ArrayList>) item; + HashMap clnt = null; + HashMap acbx = null; + HashMap fbuf = null; + for (final HashMap itemMap : list) { + final String type = itemMap.get("UABDTY").toString(); + if (type.equals("ACBX") || type.equals("CLNT") || type.equals("FBUF")) { + if (itemMap.containsKey("PAYLOAD_CLNT")) { + clnt = (HashMap) itemMap.get("PAYLOAD_CLNT"); + // cnlt data for the record + if (clnt.containsKey(Constants.RECORD_APP_USER_NAME)) { + record.setAppUserName(clnt.get(Constants.RECORD_APP_USER_NAME).toString()); + } + // fill accessor data + accessor = parseAccessor(clnt); + } + if (itemMap.containsKey("PAYLOAD_ACBX")) { + acbx = (HashMap) itemMap.get("PAYLOAD_ACBX"); + // acbx data for the record + if (acbx.containsKey(Constants.RECORD_SESSION_ID)) { + record.setSessionId(acbx.get(Constants.RECORD_SESSION_ID).toString()); + } + if (acbx.containsKey(Constants.ACBX_RSP_CODE)) { + int rspCode = (int) acbx.get(Constants.ACBX_RSP_CODE); + if (rspCode != 0) { + ExceptionRecord exceptionRecord = new ExceptionRecord(); + exceptionRecord.setExceptionTypeId(Constants.SQL_ERROR); + exceptionRecord + .setDescription("Response Code " + + acbx.get(Constants.ACBX_RSP_CODE).toString() + "(" + + acbx.get(Constants.ACBX_RSP_SUB_CODE).toString() + + ") received."); + exceptionRecord.setSqlString( + acbx.get(Constants.ACBX_CMD_CODE).toString() + " with ISN " + + acbx.get(Constants.ACBX_ISN).toString()); + record.setException(exceptionRecord); + } + } + } + if (itemMap.containsKey("PAYLOAD_FBUF")) { + fbuf = (HashMap) itemMap.get("PAYLOAD_FBUF"); + } + } + } + data = parseData(acbx, clnt, fbuf); + } + } + } + // extract time + if (map.containsKey(Constants.RECORD_TIME)) { + record.setTime(getTime(map.get(Constants.RECORD_TIME).toString())); + } + record.setAccessor(accessor); + record.setData(data); + return record; + } + + private Accessor parseAccessor(final HashMap clnt) { + logger.debug("Client info: {}", clnt); + + final Accessor accessor = new Accessor(); + accessor.setDbProtocol(Constants.DATA_PROTOCOL_STRING); + accessor.setServerType(Constants.SERVER_TYPE_STRING); + + accessor.setDbUser( + clnt.containsKey(Constants.ACCESSOR_DB_USER) ? clnt.get(Constants.ACCESSOR_DB_USER).toString() + : Constants.UNKNOWN_STRING); + accessor.setServerHostName( + clnt.containsKey(Constants.ACCESSOR_SERVER_HOST_NAME) + ? clnt.get(Constants.ACCESSOR_SERVER_HOST_NAME).toString() + : Constants.UNKNOWN_STRING); + accessor.setSourceProgram( + clnt.containsKey(Constants.ACCESSOR_SOURCE_PROGRAM) + ? clnt.get(Constants.ACCESSOR_SOURCE_PROGRAM).toString() + : Constants.UNKNOWN_STRING); + accessor.setLanguage(Accessor.LANGUAGE_FREE_TEXT_STRING); + accessor.setDataType(Accessor.DATA_TYPE_GUARDIUM_SHOULD_NOT_PARSE_SQL); + + accessor.setClient_mac(Constants.UNKNOWN_STRING); + accessor.setClientHostName(Constants.UNKNOWN_STRING); + accessor.setClientOs(Constants.UNKNOWN_STRING); + accessor.setCommProtocol(Constants.UNKNOWN_STRING); + accessor.setDbProtocolVersion(Constants.UNKNOWN_STRING); + accessor.setOsUser(Constants.UNKNOWN_STRING); + accessor.setServerDescription(Constants.UNKNOWN_STRING); + accessor.setServerOs(Constants.UNKNOWN_STRING); + accessor.setServiceName(Constants.UNKNOWN_STRING); + return accessor; + } + + private Data parseData(final HashMap acbx, final HashMap clnt, + final HashMap fbuf) { + final Data data = new Data(); + data.setConstruct(parseConstruct(acbx, fbuf)); + return data; + } + + private Construct parseConstruct(HashMap acbx, HashMap fbuf) { + final Construct construct = new Construct(); + construct.sentences.add(parseSentence(acbx, fbuf)); + return construct; + } + + private Sentence parseSentence(HashMap acbx, HashMap fbuf) { + if (acbx == null) { + return null; + } + final Sentence sentence = new Sentence(acbx.get("CMDCODE").toString()); + if (fbuf != null) { + final String s = fbuf.get("FORMATBUFFER").toString(); + if (s != null) { + final String[] fb = s.substring(0, s.length() - 1).split(","); + // very simple formatbuffer parser + ArrayList fields = new ArrayList(); + int i = 0; + for (String f : fb) { + if (i == 0) { + fields.add(f); + } + i++; + if (i == 3) { + i = 0; + } + } + sentence.setFields(fields); + } + } + return sentence; + } + + private Time getTime(final String dateString) { + final ZonedDateTime date = ZonedDateTime.parse(dateString, DateTimeFormatter.ISO_ZONED_DATE_TIME); + final long millis = date.toInstant().toEpochMilli(); + final int minOffset = date.getOffset().getTotalSeconds() / 60; + return new Time(millis, minOffset, 0); + } +} \ No newline at end of file diff --git a/filter-plugin/logstash-filter-adabas-guardium/src/test/java/org/logstashplugins/AdabasGuardiumFilterTest.java b/filter-plugin/logstash-filter-adabas-guardium/src/test/java/org/logstashplugins/AdabasGuardiumFilterTest.java new file mode 100644 index 000000000..7cd01d66c --- /dev/null +++ b/filter-plugin/logstash-filter-adabas-guardium/src/test/java/org/logstashplugins/AdabasGuardiumFilterTest.java @@ -0,0 +1,218 @@ +package org.logstashplugins; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.Assert; +import org.junit.Test; +import org.logstash.plugins.ConfigurationImpl; +import org.logstash.plugins.ContextImpl; + +import com.ibm.guardium.universalconnector.commons.GuardConstants; +import com.softwareag.adabas.auditing.logstash.AdabasGuardiumFilter; + +import co.elastic.logstash.api.Configuration; +import co.elastic.logstash.api.Context; +import co.elastic.logstash.api.Event; +import co.elastic.logstash.api.FilterMatchListener; + +public class AdabasGuardiumFilterTest { + + @Test + public void testAdabasReadFilter() throws ParseException { + HashMap testData = getTestData(); + + String sourceField = "adabas_auditing"; + Configuration config = new ConfigurationImpl(Collections.singletonMap("source", sourceField)); + + Context context = new ContextImpl(null, null); + AdabasGuardiumFilter filter = new AdabasGuardiumFilter("test-id", config, context); + + Event e = new org.logstash.Event(); + TestMatchListener matchListener = new TestMatchListener(); + + e.setField(sourceField, testData); + Collection results = filter.filter(Collections.singletonList(e), matchListener); + + Assert.assertEquals(1, results.size()); + Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); + // Assert.assertEquals( + // "{\"sessionId\":\"14800101\",\"dbName\":\"22131\",\"appUserName\":\"GER\",\"time\":{\"timstamp\":1738675401420,\"minOffsetFromGMT\":0,\"minDst\":0},\"sessionLocator\":null,\"accessor\":{\"dbUser\":\"GER\",\"serverType\":\"Adabas\",\"serverOs\":\"\",\"clientOs\":\"\",\"clientHostName\":\"\",\"serverHostName\":\"DA3F\",\"commProtocol\":\"\",\"dbProtocol\":\"Adabas + // native + // audit\",\"dbProtocolVersion\":\"\",\"osUser\":\"\",\"sourceProgram\":\"EMPLAPP\",\"client_mac\":\"\",\"serverDescription\":\"\",\"serviceName\":\"\",\"language\":\"FREE_TEXT\",\"dataType\":\"CONSTRUCT\"},\"data\":null,\"exception\":null}", + // e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME).toString()); + System.out.println(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME).toString()); + Assert.assertEquals(1, matchListener.getMatchCount()); + } + + private HashMap getTestData() throws ParseException { + SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); + + HashMap testData = new HashMap<>(); + + HashMap uabiItems = new HashMap<>(); + uabiItems.put("UABIDBID", 22131); + uabiItems.put("messageType", "CMD"); + uabiItems.put("UABIITIM", formatter.parse("2025-02-04T14:23:16.176Z")); + uabiItems.put("UABISNAM", "EMPL"); + uabiItems.put("UABINUCI", 0); + uabiItems.put("UABITY", "CMD"); + + HashMap uaicm = new HashMap<>(); + uaicm.put("UAICMAUD", "DE00AC140A6D2607"); + uaicm.put("UAICMSID", "GER"); + uaicm.put("UAICMISN", 109425); + uaicm.put("UAICMGID", + "000498E885620001404040404040404000F72580C7C5D940404040F1"); + uaicm.put("UAICMFNR", 1); + uaicm.put("UAICMISG", 0); + uaicm.put("UAICMCMD", "S1"); + + uabiItems.put("UAICM", uaicm); + + // // "UABD_ITEMS" => [ + ArrayList> uabdItems = new ArrayList<>(); + HashMap payloadData = new HashMap<>(); + payloadData.put("AREACODE", ""); + payloadData.put("JOBTITLE", ""); + payloadData.put("MIDDLE_NAME", "W."); + payloadData.put("CITY", "Auckland"); + payloadData.put("SEX", ""); + payloadData.put("LEAVE_TAKEN", 0); + payloadData.put("COUNTRY", "NZ"); + payloadData.put("LEAVE_DUE", 0); + payloadData.put("MARSTAT", ""); + ArrayList addressLine = new ArrayList<>(); + addressLine.add("Line 1"); + addressLine.add("Line 2"); + addressLine.add("Line 3"); + addressLine.add("Line 4"); + addressLine.add("Line 5"); + payloadData.put("ADDRESS_LINE", addressLine); + payloadData.put("LANG", new ArrayList<>()); + payloadData.put("PERSONNEL_ID", "D2345678"); + payloadData.put("PHONE", ""); + payloadData.put("POSTCODE", ""); + payloadData.put("NAME", "Johnson"); + ArrayList> income = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + HashMap incomeItem = new HashMap<>(); + incomeItem.put("SALARY", 0); + ArrayList bonus = new ArrayList<>(); + for (int j = 0; j < 5; j++) { + bonus.add(0); + } + incomeItem.put("BONUS", bonus); + incomeItem.put("CURRCODE", ""); + income.add(incomeItem); + } + payloadData.put("INCOME", income); + payloadData.put("FIRST_NAME", "Max"); + payloadData.put("DEPT", "SALE01"); + ArrayList> leaveBooked = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + HashMap leaveBookedItem = new HashMap<>(); + leaveBookedItem.put("LEAVE_END", 0); + leaveBookedItem.put("LEAVE_START", 0); + leaveBooked.add(leaveBookedItem); + } + payloadData.put("LEAVE_BOOKED", leaveBooked); + + HashMap uabdItem = new HashMap<>(); + uabdItem.put("PAYLOAD_DATA", payloadData); + uabdItem.put("UABDISNG", 0); + uabdItem.put("UABDRETS", formatter.parse("2023-11-14T12:08:39.863Z")); + uabdItem.put("UABDSUBC", ""); + uabdItem.put("UABDRSP", 0); + uabdItem.put("UABDTY", "DATA"); + uabdItem.put("UABDISN", 109425); + uabdItems.add(uabdItem); + + uabdItem = new HashMap<>(); + uabdItem.put("UABDISNG", 0); + uabdItem.put("UABDRETS", formatter.parse("2023-11-13T11:58:04.861Z")); + uabdItem.put("UABDSUBC", ""); + uabdItem.put("UABDRSP", 0); + uabdItem.put("UABDTY", "ACBX"); + uabdItem.put("UABDISN", 0); + HashMap payloadACBX = new HashMap<>(); + payloadACBX.put("CMDID", "14800101"); + payloadACBX.put("USERFIELD", "00000000000000000000000000000000"); + payloadACBX.put("RSPCODE", 0); + payloadACBX.put("FNR", 1); + payloadACBX.put("RSPSUBCODE", 0); + payloadACBX.put("ISN", 109425); + payloadACBX.put("CMDCODE", "S1"); + payloadACBX.put("ISQ", 1); + payloadACBX.put("ADDITIONS1", "4040404040404040"); + payloadACBX.put("DBID", 0); + payloadACBX.put("RSPSUBCODECHAR", ""); + uabdItem.put("PAYLOAD_ACBX", payloadACBX); + uabdItems.add(uabdItem); + + uabdItem = new HashMap<>(); + uabdItem.put("UABDRETS", formatter.parse("2020-10-01T16:59:24.839Z")); + uabdItem.put("UABDSUBC", ""); + uabdItem.put("UABDRSP", 0); + uabdItem.put("UABDTY", "FBUF"); + uabdItem.put("UABDISN", 0); + HashMap payloadFBUF = new HashMap<>(); + payloadFBUF.put("FORMATBUFFER", "AI1-5,20,A,AC,20,A,AD,20,A,AE,20,A,AJ,20,A,AO,6,A,AL,3,A."); + uabdItem.put("PAYLOAD_FBUF", payloadFBUF); + uabdItems.add(uabdItem); + + uabdItem = new HashMap<>(); + uabdItem.put("UABDISNG", 0); + uabdItem.put("UABDRETS", formatter.parse("2021-07-27T14:54:05.440Z")); + uabdItem.put("UABDSUBC", ""); + uabdItem.put("UABDRSP", 0); + uabdItem.put("UABDTY", "CLNT"); + uabdItem.put("UABDISN", 0); + HashMap payloadCLNT = new HashMap<>(); + payloadCLNT.put("SECUID", "GER"); + payloadCLNT.put("LPARNAME", "DA3F"); + payloadCLNT.put("NATSTMT", "1480"); + payloadCLNT.put("USERTYPE", "COMPLETE"); + payloadCLNT.put("ACCTINFO", "ACCOUNT 2005"); + payloadCLNT.put("COM-PLETE-TID", 7); + payloadCLNT.put("TPUSERID", "GER"); + payloadCLNT.put("NATAPPL", "GER"); + payloadCLNT.put("NATPROG", "EMPLAPP"); + payloadCLNT.put("NATLIB", "GER"); + payloadCLNT.put("NATUID", "GER"); + payloadCLNT.put("CALLPGM", "NAT92"); + payloadCLNT.put("JOBID", "S0206859"); + uabdItem.put("PAYLOAD_CLNT", payloadCLNT); + uabdItems.add(uabdItem); + uabiItems.put("UABIDCNT", 3); + uabiItems.put("UABIPTIM", formatter.parse("2025-02-04T14:23:21.420Z")); + uabiItems.put("UABD_ITEMS", uabdItems); + + testData.put("UABI_ITEMS", uabiItems); + testData.put("UABHNAME", "ANSERVER"); + testData.put("UABHTIME", formatter.parse("2025-02-04T14:23:21.420Z")); + testData.put("UABHID", 22134); + testData.put("UABHNUCI", 0); + return testData; + } + +} + +class TestMatchListener implements FilterMatchListener { + + private AtomicInteger matchCount = new AtomicInteger(0); + + @Override + public void filterMatched(Event event) { + matchCount.incrementAndGet(); + } + + public int getMatchCount() { + return matchCount.get(); + } +} \ No newline at end of file diff --git a/filter-plugin/logstash-filter-alloydb-guardium/.gitignore b/filter-plugin/logstash-filter-alloydb-guardium/.gitignore new file mode 100644 index 000000000..eed8c4450 --- /dev/null +++ b/filter-plugin/logstash-filter-alloydb-guardium/.gitignore @@ -0,0 +1,5 @@ +.gradle +.idea +build +gradle.properties +vendor \ No newline at end of file diff --git a/filter-plugin/logstash-filter-alloydb-guardium/AlloyDBoverPubSubPackage/alloydb.conf b/filter-plugin/logstash-filter-alloydb-guardium/AlloyDBoverPubSubPackage/alloydb.conf new file mode 100644 index 000000000..133abca60 --- /dev/null +++ b/filter-plugin/logstash-filter-alloydb-guardium/AlloyDBoverPubSubPackage/alloydb.conf @@ -0,0 +1,31 @@ +input { + google_pubsub { + # Your GCP project id (name) + project_id => "" + + # The topic name below is currently hard-coded in the plugin. You + # must first create this topic by hand and ensure you are exporting + # logging to this pubsub topic. + topic => "" + + # The subscription name is customizeable. The plugin will attempt to + # create the subscription (but use the hard-coded topic name above). + subscription => "" + + # If you are running logstash within GCE, it will use + # Application Default Credentials and use GCE's metadata + # service to fetch tokens. However, if you are running logstash + # outside of GCE, you will need to specify the service account's + # JSON key file below. + json_key_file => "${THIRD_PARTY_PATH}/.json" + # Type should be populated with a unique ID per connector to provide data source isolation + # can be any string, preferably a meaningful one + type => "alloydb" +} +} + +filter{ + if [type] == "alloydb" { + alloydb_guardium_filter{} + } +} \ No newline at end of file diff --git a/filter-plugin/logstash-filter-alloydb-guardium/AlloyDBoverPubSubPackage/logstash-filter-alloydb_guardium_filter.zip b/filter-plugin/logstash-filter-alloydb-guardium/AlloyDBoverPubSubPackage/logstash-filter-alloydb_guardium_filter.zip new file mode 100644 index 000000000..a152d1a0d Binary files /dev/null and b/filter-plugin/logstash-filter-alloydb-guardium/AlloyDBoverPubSubPackage/logstash-filter-alloydb_guardium_filter.zip differ diff --git a/filter-plugin/logstash-filter-alloydb-guardium/CHANGELOG.md b/filter-plugin/logstash-filter-alloydb-guardium/CHANGELOG.md new file mode 100644 index 000000000..143dd1602 --- /dev/null +++ b/filter-plugin/logstash-filter-alloydb-guardium/CHANGELOG.md @@ -0,0 +1,10 @@ + +# Changelog +Notable changes will be documented in this file. + + + +## [] + +### Added +- Initial release, in parallel to Guardium . diff --git a/filter-plugin/logstash-filter-alloydb-guardium/LICENSE b/filter-plugin/logstash-filter-alloydb-guardium/LICENSE new file mode 100644 index 000000000..261eeb9e9 --- /dev/null +++ b/filter-plugin/logstash-filter-alloydb-guardium/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/filter-plugin/logstash-filter-alloydb-guardium/README.md b/filter-plugin/logstash-filter-alloydb-guardium/README.md new file mode 100644 index 000000000..519eb62d7 --- /dev/null +++ b/filter-plugin/logstash-filter-alloydb-guardium/README.md @@ -0,0 +1,90 @@ +# AlloyDB-Guardium Logstash filter plug-in + +### Meet AlloyDB + +* Tested versions: V1 +* Environment: Google Cloud Platform (GCP) +* Supported inputs: Google Pubsub input plugin +* Supported Guardium versions: Guardium Data Protection: 12.1 and later + +This is a [Logstash](https://github.com/elastic/logstash) filter plug-in for the universal connector that is featured in +IBM Security Guardium. It parses AlloyDB event logs into +a [Guardium record](https://github.com/IBM/universal-connectors/blob/main/common/src/main/java/com/ibm/guardium/universalconnector/commons/structures/Record.java) +instance, which is a standard structure made out of several parts. Then the information is sent to Guardium. +Guardium records include the accessor (the person who tries to access the data), the session, data, and exceptions. If +there are no errors, the data contains details about the query `construct`. The construct details the main action (verb) +and collections (objects) involved. The AlloyDB Logstash filter plug-in supports Guardium Data Protection. + +Note: This version is compliant with Guardium Data Protection v12.1 and later. For more information, see [input plug-in repository](https://github.com/IBM/universal-connectors/tree/main/input-plugin/logstash-input-google-pubsub). + +## Configuring AlloyDB on GCP + +1. [Create a cluster and its + primary instance](https://cloud.google.com/alloydb/docs/quickstart/create-and-connect?hl=en#create-cluster). +2. [Connect to your instance and create + a database](https://cloud.google.com/alloydb/docs/quickstart/create-and-connect?hl=en#run). +3. [Connect to the database that you created](https://cloud.google.com/alloydb/docs/quickstart/create-and-connect?hl=en#connect-to-guestbook). +4. [Verify your database + connection](https://cloud.google.com/alloydb/docs/quickstart/create-and-connect?hl=en#verify-connection). +5. [Create a log sink in Pub/Sub](https://cloud.google.com/logging/docs/export/configure_export_v2#creating_sink). + * Use the following inclusion filter for ```Choose logs to include in sink``` during log sink creation to specify which logs to route. The following filter captures relevant logs based on data access and activity logs: + + ((resource.type="alloydb.googleapis.com/Instance" logName="projects//logs/alloydb.googleapis.com%2Fpostgres.log" )) + +## Configuring GCP for the input plug-in +1. [Create a topic in Pub/Sub](https://cloud.google.com/pubsub/docs/create-topic#create_a_topic_2). +2. [Create a subscription in Pub/Sub](https://cloud.google.com/pubsub/docs/create-subscription#create_a_pull_subscription) +3. [Create service account credentials](https://developers.google.com/workspace/guides/create-credentials#create_a_service_account): + - To provide subscription access to the service account, select the **Pub/Sub Subscriber** role from the role selection list during the service account creation process. + - You do not need to grant users access to this service account. +4. [Create credentials for a service account](https://developers.google.com/workspace/guides/create-credentials#create_credentials_for_a_service_account). The key is used by the Logstash input plug-in configuration file. + +## Enabling audit logs: + +1. To view the detailed audit logs, enable the following flags on your database instance: + +* `log_statement: all` - View executed SQL statements in audit logs. + +2. To reduce the volume of audit logs, you can turn off the following flags, as they do not contain any details about the run queries: + +* `autovacuum: off` +* `log_checkpoints: off` +* `log_connections: off` +* `log_disconnections: off` + +## Configuring the AlloyDB filter in Guardium + +The Guardium universal connector is the Guardium entry point for native audit and data access logs. The Guardium universal +connector identifies and parses the received events, and converts them to a standard Guardium format. The output of the +Guardium universal connector is forwarded to the Guardium sniffer on the collector for policy and auditing +enforcements. Configure Guardium to read the native audit and data access logs by customizing the Capella template. + +### Before you begin + +* Configure the policies you need. For more information, see [Policies](/docs/#policies). +* You must have permissions for the S-Tap Management role. By default, the admin user is assigned the S-Tap Management role. +* Download + the [logstash-filter-alloydb_guardium_filter](https://github.com/IBM/universal-connectors/releases/download/v1.7.0/logstash-filter-alloydb_guardium_filter.zip) plug-in. + +### Procedure + +1. On the collector, go to **Setup** > **Tools and Views** > **Configure Universal Connector**. +2. Enable the universal connector if it is disabled. +3. Click **Upload File** and select the offline [logstash-filter-alloydb_guardium_filter](https://github.com/IBM/universal-connectors/releases/download/v1.7.0/logstash-filter-alloydb_guardium_filter.zip) plug-in. After it is uploaded, click **OK**. +4. Click **Upload File** and select the key.json file. After it is uploaded, click OK. +4. Click the **Plus** sign to open the Connector Configuration dialog. +5. In the **Connector name** field, enter a name. +6. Update the input section to add the details from + the [alloydb.conf](AlloyDBoverPubSubPackage/alloydb.conf) file's ``input`` section, omitting the keyword ``input{`` at the beginning and its corresponding ``}`` at the end. +7. Update the filter section to add the details from + the [alloydb.conf](AlloyDBoverPubSubPackage/alloydb.conf) file's ``filter`` section, omitting the keyword ``filter{`` at the beginning and its corresponding ``}`` at the end. +8. Make sure that the ``type`` fields in the ``input`` and ``filter`` configuration sections align. This field must be unique for each connector added to the system. +9. Click **Save**. Guardium validates the new connector and displays it in the Configure Universal Connector page. +10. When the offline plug-in is installed and the configuration is uploaded and saved in the Guardium machine, restart the universal connector by using the **Disable/Enable** button. + +## Limitations + +- Audit logs that contain SQL queries do not contain port and host information, so they are mapped to the default values. +- When you use GCP, duplicate entries can appear in both the reports and audit logs. + + diff --git a/filter-plugin/logstash-filter-alloydb-guardium/VERSION b/filter-plugin/logstash-filter-alloydb-guardium/VERSION new file mode 100644 index 000000000..afaf360d3 --- /dev/null +++ b/filter-plugin/logstash-filter-alloydb-guardium/VERSION @@ -0,0 +1 @@ +1.0.0 \ No newline at end of file diff --git a/filter-plugin/logstash-filter-alloydb-guardium/build.gradle b/filter-plugin/logstash-filter-alloydb-guardium/build.gradle new file mode 100644 index 000000000..fb9ae092e --- /dev/null +++ b/filter-plugin/logstash-filter-alloydb-guardium/build.gradle @@ -0,0 +1,215 @@ +import java.nio.file.Files +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING + +apply plugin: 'java' +apply plugin: 'jacoco' + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + + dependencies { + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' + classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.1.0" + classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' + } + + ext { + snakeYamlVersion = '2.2' + } +} + + +def universalConnectorsDir = project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load(new File("${universalConnectorsDir}/versions.yml").newInputStream()) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + +apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" +apply plugin: "com.github.johnrengelman.shadow" +apply plugin: "eclipse" + + +// =========================================================================== +// plugin info +// =========================================================================== +group "com.ibm.guardium.alloydb" // must match the package of the main plugin class +version "${file("VERSION").text.trim()}" // read from required VERSION file +description = "AlloyDB-Guardium filter plugin" +pluginInfo.licenses = ['Apache-2.0'] // list of SPDX license IDs +pluginInfo.longDescription = "This gem is a Logstash AlloyDB filter plugin required to be installed as part of IBM Security Guardium, Guardium Universal connector configuration. This gem is not a stand-alone program." +pluginInfo.authors = ['IBM', '', ''] +pluginInfo.email = [''] +pluginInfo.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html" +pluginInfo.pluginType = "filter" +pluginInfo.pluginClass = "AlloyDBGuardiumFilter" +pluginInfo.pluginName = "alloydb_guardium_filter" // must match the @LogstashPlugin annotation in the main plugin class +// =========================================================================== + +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 + +def jacocoVersion = '0.8.11' +// minimumCoverage can be set by Travis ENV +def minimumCoverageStr = System.getenv("MINIMUM_COVERAGE") ?: "50.0%" +if (minimumCoverageStr.endsWith("%")) { + minimumCoverageStr = minimumCoverageStr.substring(0, minimumCoverageStr.length() - 1) +} +def minimumCoverage = Float.valueOf(minimumCoverageStr) / 100 + + +repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } +} + +tasks.register("vendor") { + dependsOn shadowJar + doLast { + String vendorPathPrefix = "vendor/jar-dependencies" + String projectGroupPath = project.group.replaceAll('\\.', '/') + File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") + projectJarFile.mkdirs() + Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) + } +} + +apply plugin: 'com.github.johnrengelman.shadow' + +shadowJar { + archiveClassifier = null +} + + +dependencies { + implementation 'com.google.code.gson:gson:' + versions.dependencies.gson + implementation 'org.apache.commons:commons-lang3:' + versions.dependencies.commonsLang + implementation 'commons-validator:commons-validator:' + versions.dependencies.commonsValidator + implementation group: 'commons-beanutils', name: 'commons-beanutils', version: versions.dependencies.commonsBeanutils + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.17.2' + + testImplementation group: 'org.mockito', name: 'mockito-all', version: versions.dependencies.mockitoAll + testImplementation 'org.junit.jupiter:junit-jupiter:' + versions.dependencies.junitJupiter + testImplementation 'org.jruby:jruby-complete:' + versions.dependencies.jrubyComplete + implementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "common-*.*.*.jar") + implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "build/libs/logstash-core.jar") + implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: versions.dependencies.log4jCore + implementation group: 'org.json', name: 'json', version: versions.dependencies.json + implementation group: 'org.parboiled', name: 'parboiled-java', version: versions.dependencies.parboiledJava + implementation group: 'org.glassfish', name: 'javax.json', version: versions.dependencies.javaxJson + testImplementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "common-*.*.*.jar") +} + +clean { + delete "${projectDir}/Gemfile" + delete "${projectDir}/" + pluginInfo.pluginFullName() + ".gemspec" + delete "${projectDir}/lib/" + delete "${projectDir}/vendor/" + new FileNameFinder().getFileNames(projectDir.toString(), pluginInfo.pluginFullName() + "-*.*.*.gem").each { filename -> + delete filename + } +} + +tasks.withType(JavaCompile) { + options.encoding = 'UTF-8' +} +test { + useJUnitPlatform() +} +tasks.register("generateRubySupportFiles") { + doLast { + generateRubySupportFilesForPlugin(project.description, project.group, version) + } +} + +tasks.register("removeObsoleteJars") { + doLast { + new FileNameFinder().getFileNames( + projectDir.toString(), + "vendor/**/" + pluginInfo.pluginFullName() + "*.jar", + "vendor/**/" + pluginInfo.pluginFullName() + "-" + version + ".jar").each { f -> + delete f + } + } +} + +tasks.register("gem") { + dependsOn = [downloadAndInstallJRuby, removeObsoleteJars, vendor, generateRubySupportFiles] + doLast { + buildGem(projectDir, buildDir, pluginInfo.pluginFullName() + ".gemspec") + } +} + +tasks.register("copyDependencyLibs", Copy) { + into "dependenciesLib" + from configurations.compileClasspath + from configurations.runtimeClasspath + from configurations.testCompileClasspath + from configurations.testRuntimeClasspath +} + +apply plugin: 'jacoco' +//apply plugin: 'org.barfuin.gradle.jacocolog' version '2.0.0' +apply plugin: "org.barfuin.gradle.jacocolog" +// ------------------------------------ +// JaCoCo is a code coverage tool +// ------------------------------------ +jacoco { + toolVersion = "${jacocoVersion}" +} +jacocoTestReport { + // You will see "Report -> file://...." at the end of a JaCoCo build + // If no output, run this first: ./gradlew test + reports { + html.required = true + xml.required = true + csv.required = true + html.outputLocation = file("${buildDir}/reports/jacoco") + csv.outputLocation = file("${buildDir}/reports/jacoco/all.csv") + } + executionData.from fileTree(dir: "${buildDir}/jacoco/", includes: [ + '**/*.exec' + ]) + afterEvaluate { + // objective is to test TicketingService class + classDirectories.setFrom(files(classDirectories.files.collect { + fileTree(dir: it, exclude: []) + })) + } + doLast { + println "Report -> file://${buildDir}/reports/jacoco/index.html" + } +} +test.finalizedBy jacocoTestReport +jacocoTestCoverageVerification { + violationRules { + rule { + limit { + minimum = minimumCoverage + } + } + } + executionData.from fileTree(dir: "${buildDir}/jacoco/", includes: [ + '**/*.exec' + ]) + afterEvaluate { + // objective is to test TicketingService class + classDirectories.setFrom(files(classDirectories.files.collect { + fileTree(dir: it, exclude: []) + })) + } +} +project.tasks.check.dependsOn(jacocoTestCoverageVerification, jacocoTestReport) \ No newline at end of file diff --git a/filter-plugin/logstash-filter-alloydb-guardium/gradle/wrapper/gradle-wrapper.jar b/filter-plugin/logstash-filter-alloydb-guardium/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 000000000..7454180f2 Binary files /dev/null and b/filter-plugin/logstash-filter-alloydb-guardium/gradle/wrapper/gradle-wrapper.jar differ diff --git a/filter-plugin/logstash-filter-alloydb-guardium/gradle/wrapper/gradle-wrapper.properties b/filter-plugin/logstash-filter-alloydb-guardium/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 000000000..81aa1c044 --- /dev/null +++ b/filter-plugin/logstash-filter-alloydb-guardium/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,5 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/filter-plugin/logstash-filter-alloydb-guardium/gradlew b/filter-plugin/logstash-filter-alloydb-guardium/gradlew new file mode 100755 index 000000000..744e882ed --- /dev/null +++ b/filter-plugin/logstash-filter-alloydb-guardium/gradlew @@ -0,0 +1,185 @@ +#!/usr/bin/env sh + +# +# Copyright 2015 the original author or authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MSYS* | MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin or MSYS, switch paths to Windows format before running java +if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=`expr $i + 1` + done + case $i in + 0) set -- ;; + 1) set -- "$args0" ;; + 2) set -- "$args0" "$args1" ;; + 3) set -- "$args0" "$args1" "$args2" ;; + 4) set -- "$args0" "$args1" "$args2" "$args3" ;; + 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=`save "$@"` + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +exec "$JAVACMD" "$@" diff --git a/filter-plugin/logstash-filter-alloydb-guardium/gradlew.bat b/filter-plugin/logstash-filter-alloydb-guardium/gradlew.bat new file mode 100644 index 000000000..107acd32c --- /dev/null +++ b/filter-plugin/logstash-filter-alloydb-guardium/gradlew.bat @@ -0,0 +1,89 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto execute + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/filter-plugin/logstash-filter-alloydb-guardium/mainREADME.md b/filter-plugin/logstash-filter-alloydb-guardium/mainREADME.md new file mode 100644 index 000000000..01e969e36 --- /dev/null +++ b/filter-plugin/logstash-filter-alloydb-guardium/mainREADME.md @@ -0,0 +1,10 @@ +# AlloyDB Universal Connector + +## Follow this link to set up and use AlloyDB Universal Connector over PubSub Logstash Plugin + +[AlloyDBOverPubSub](./README.md) + +## Follow this link to set up and use AlloyDB Universal Connector over PubSub Connect + +[AlloyDBOverConnectPubSub](../../docs/KafkaBasedUCs/AlloyDBPubsubKafkaConnect.md) + diff --git a/filter-plugin/logstash-filter-alloydb-guardium/settings.gradle b/filter-plugin/logstash-filter-alloydb-guardium/settings.gradle new file mode 100644 index 000000000..63fb4e545 --- /dev/null +++ b/filter-plugin/logstash-filter-alloydb-guardium/settings.gradle @@ -0,0 +1,11 @@ +/* + * This file was generated by the Gradle 'init' task. + * + * The settings file is used to specify which projects to include in your build. + * + * Detailed information about configuring a multi-project build in Gradle can be found + * in the user manual at https://docs.gradle.org/7.1.1/userguide/multi_project_builds.html + */ + +rootProject.name = 'logstash-filter-alloydb-guardium' +include('lib') diff --git a/filter-plugin/logstash-filter-alloydb-guardium/src/main/java/com/ibm/guardium/alloydb/AlloyDBGuardiumFilter.java b/filter-plugin/logstash-filter-alloydb-guardium/src/main/java/com/ibm/guardium/alloydb/AlloyDBGuardiumFilter.java new file mode 100644 index 000000000..de64a9a0a --- /dev/null +++ b/filter-plugin/logstash-filter-alloydb-guardium/src/main/java/com/ibm/guardium/alloydb/AlloyDBGuardiumFilter.java @@ -0,0 +1,92 @@ +/* +Copyright IBM Corp. 2021, 2023 All rights reserved. +SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.guardium.alloydb; + +import co.elastic.logstash.api.*; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.ibm.guardium.universalconnector.commons.GuardConstants; +import com.ibm.guardium.universalconnector.commons.custom_parsing.ParserFactory; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; + +import static com.ibm.guardium.alloydb.Constants.*; + +@LogstashPlugin(name = "alloydb_guardium_filter") +public class AlloyDBGuardiumFilter implements Filter { + + private static Logger logger = LogManager.getLogger(AlloyDBGuardiumFilter.class); + + private String id; + private String filter; + private Parser parser; + public static final PluginConfigSpec SOURCE_CONFIG = + PluginConfigSpec.stringSetting("source", "message"); + + public AlloyDBGuardiumFilter(String id, String filter, Configuration config, Context context) { + this(id, config, context); + this.filter = filter; + } + + public AlloyDBGuardiumFilter(String id, Configuration config, Context context) { + this.id = id; + this.parser = new Parser(ParserFactory.ParserType.json); + } + + @Override + public Collection> configSchema() { + return Collections.singletonList(SOURCE_CONFIG); + } + + /** + * Returns the id + * + * @return id + */ + @Override + public String getId() { + return this.id; + } + + /** + * Filters the received events by skipping the invalid ones and normalizing them by parsing the + * provided payloads into Guardium Generic Records. + * + * @param events A list of received events + * @param filterMatchListener The listener for this plugin + * @return A list of normalized events + */ + public Collection filter( + Collection events, FilterMatchListener filterMatchListener) { + ArrayList skippedEvents = new ArrayList<>(); + for (Event e : events) { + if (logger.isDebugEnabled()) { + logger.debug("Event Now: {}", e.getData()); + } + if (!(e.getField(MESSAGE) instanceof String) + || (filter != null && !String.valueOf(e.getField(MESSAGE)).contains(filter))) { + e.tag(INVALID_MSG); + } else { + try { + Record record = this.parser.parseRecord(e.getField(MESSAGE).toString()); + final Gson gson = new GsonBuilder().disableHtmlEscaping().serializeNulls().create(); + e.setField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME, gson.toJson(record)); + filterMatchListener.filterMatched(e); + } catch (RuntimeException re) { + skippedEvents.add(e); + logger.debug(re.getMessage()); + } + } + } + events.removeAll(skippedEvents); + return events; + } +} diff --git a/filter-plugin/logstash-filter-alloydb-guardium/src/main/java/com/ibm/guardium/alloydb/ConfigurationGenerator.java b/filter-plugin/logstash-filter-alloydb-guardium/src/main/java/com/ibm/guardium/alloydb/ConfigurationGenerator.java new file mode 100644 index 000000000..2078a3c0a --- /dev/null +++ b/filter-plugin/logstash-filter-alloydb-guardium/src/main/java/com/ibm/guardium/alloydb/ConfigurationGenerator.java @@ -0,0 +1,22 @@ +package com.ibm.guardium.alloydb; + +public class ConfigurationGenerator { + static String getConfig() { + return "{\n" + + " \"db_protocol\": \"{ALLOYDB}\",\n" + + " \"text_payload\": \"/\\\"textPayload\\\"\",\n" + + " \"exception_type_id\": \"/\\\"severity\\\"\",\n" + + " \"db_name\": \"/\\\"protoPayload\\\"/\\\"request\\\"/\\\"database\\\"\",\n" + + " \"db_user\": \"/\\\"protoPayload\\\"/\\\"authenticationInfo\\\"/\\\"principalEmail\\\"\",\n" + + " \"parsing_type\": \"CUSTOM_PARSER\",\n" + + " \"client_ip\": \"/\\\"protoPayload\\\"/\\\"requestMetadata\\\"/\\\"callerIp\\\"\",\n" + + " \"client_ipv6\": \"/\\\"protoPayload\\\"/\\\"requestMetadata\\\"/\\\"callerIp\\\"\",\n" + + " \"server_type\": \"{ALLOYDB}\",\n" + + " \"session_id\": \"{N.A.}\",\n" + + " \"language\": \"{PGRS}\",\n" + + " \"data_type\": \"{TEXT}\",\n" + + " \"server_port\": \"{-1}\",\n" + + " \"timestamp\": \"/\\\"timestamp\\\"\"\n" + + "}"; + } +} diff --git a/filter-plugin/logstash-filter-alloydb-guardium/src/main/java/com/ibm/guardium/alloydb/Constants.java b/filter-plugin/logstash-filter-alloydb-guardium/src/main/java/com/ibm/guardium/alloydb/Constants.java new file mode 100644 index 000000000..43fa888dc --- /dev/null +++ b/filter-plugin/logstash-filter-alloydb-guardium/src/main/java/com/ibm/guardium/alloydb/Constants.java @@ -0,0 +1,21 @@ +package com.ibm.guardium.alloydb; + +public class Constants { + static final String MESSAGE = "message"; + static final String INVALID_MSG = "EVENT_IS_INVALID"; + static final String TEXT_PAYLOAD = "text_payload"; + static final String NOT_AVAILABLE = "N.A."; + static final String IP = "ip"; + static final String DB_NAME = "db_name"; + static final String DB_USER = "db_user"; + static final String PORT = "port"; + static final String SQL_STATEMENT = "sqlStatement"; + static final String ERROR = "error"; + static final String LANGUAGE = "language"; + static final String SERVER_TYPE = "server_type"; + static final String DATA_TYPE = "data_type"; + static final String EXCEPTION_TYPE_ID = "exception_type_id"; + static final String EXCEPTION_TYPE_AUTHENTICATION_STRING = "LOGIN_FAILED"; + static final String EXCEPTION_TYPE_AUTHORIZATION_STRING = "SQL_ERROR"; + static final String FREE_TEXT = "FREE_TEXT"; +} diff --git a/filter-plugin/logstash-filter-alloydb-guardium/src/main/java/com/ibm/guardium/alloydb/Parser.java b/filter-plugin/logstash-filter-alloydb-guardium/src/main/java/com/ibm/guardium/alloydb/Parser.java new file mode 100644 index 000000000..79b30dafc --- /dev/null +++ b/filter-plugin/logstash-filter-alloydb-guardium/src/main/java/com/ibm/guardium/alloydb/Parser.java @@ -0,0 +1,292 @@ +/* +Copyright IBM Corp. 2021, 2023 All rights reserved. +SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.guardium.alloydb; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.ibm.guardium.universalconnector.commons.custom_parsing.CustomParser; +import com.ibm.guardium.universalconnector.commons.custom_parsing.ParserFactory; +import static com.ibm.guardium.alloydb.Constants.*; +import static com.ibm.guardium.universalconnector.commons.custom_parsing.PropertyConstant.*; + +import com.ibm.guardium.universalconnector.commons.custom_parsing.SqlParser; +import com.ibm.guardium.universalconnector.commons.structures.Accessor; +import com.ibm.guardium.universalconnector.commons.structures.Construct; +import com.ibm.guardium.universalconnector.commons.structures.Data; +import com.ibm.guardium.universalconnector.commons.structures.ExceptionRecord; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import com.ibm.guardium.universalconnector.commons.structures.Sentence; +import com.ibm.guardium.universalconnector.commons.structures.SentenceObject; +import com.ibm.guardium.universalconnector.commons.structures.SessionLocator; +import com.ibm.guardium.universalconnector.commons.structures.Time; +import org.apache.commons.validator.routines.InetAddressValidator; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.IOException; +import java.util.Map; +import java.util.regex.*; +import java.util.*; + +/** + * Parser Class will perform operation on parsing events and messages from the AlloyDB audit logs + * into a Guardium record instance Guardium records include the accessor, the sessionLocator, data, + * and exceptions. If there are no errors, the data contains details about the query "construct" + * + * @className @Parser + */ +public class Parser extends CustomParser { + private static Logger logger = LogManager.getLogger(Parser.class); + private static final String HOST_REGEX = "host=([a-fA-F0-9:.]+)"; + private static final String PORT_REGEX = "port=(\\d+)"; + private static final String DB_REGEX = "db=([^,]+)"; + private static final String USER_REGEX = "user=([^ ]+)"; + private static final String SQL_STATEMENT_REGEX = "(?i)statement:\\s+(.*)"; + private static final String ALTERNATE_SQL_STATEMENT_REGEX = + "(?i)execute\\s*<\\s*unnamed\\s*>\\s*:\\s*(.*)$"; + private static final String ERROR_REGEX = "ERROR:(.*)"; + private static final String FATAL_ERROR_REGEX = "FATAL:(.*)"; + ; + private Map textPayloadFields; + private static final Pattern hostPattern = Pattern.compile(HOST_REGEX); + private static final Pattern portPattern = Pattern.compile(PORT_REGEX); + private static final Pattern sqlStatementPattern = + Pattern.compile(SQL_STATEMENT_REGEX, Pattern.DOTALL); + private static final Pattern alternateSqlStatementPattern = + Pattern.compile(ALTERNATE_SQL_STATEMENT_REGEX, Pattern.DOTALL); + private static final Pattern dbPattern = Pattern.compile(DB_REGEX); + private static final Pattern dbUserPattern = Pattern.compile(USER_REGEX); + private static final Pattern errorPattern = Pattern.compile(ERROR_REGEX); + private static final Pattern fatalErrorPattern = Pattern.compile(FATAL_ERROR_REGEX); + + public Parser(ParserFactory.ParserType parserType) { + super(parserType); + } + + @Override + public String getConfigFileContent() { + return ConfigurationGenerator.getConfig(); + } + + @Override + public Record parseRecord(String payload) { + textPayloadFields = new HashMap<>(); + if (!isValid(payload)) { + logger.debug("Invalid AlloyDB Guardium log record: " + payload); + return null; + } + + String textPayload = getValue(payload, TEXT_PAYLOAD); + parseTextPayload(textPayload); + Record record = extractRecord(payload); + Data data = record.getData(); + if (data != null && Objects.equals(data.getOriginalSqlCommand(), NOT_AVAILABLE)) { + logger.debug("Skipping event " + payload); + throw new RuntimeException( + "Adding payload to skipped events since SQL command is not available " + payload); + } + logger.debug("Successfully parsed AlloyDB Guardium log record: " + payload); + return record; + } + + private void parseTextPayload(String textPayload) { + logger.debug("Parsing text payload: " + textPayload); + textPayloadFields.put(IP, DEFAULT_IP); + textPayloadFields.put(PORT, "-1"); + textPayloadFields.put(Constants.DB_USER, NOT_AVAILABLE); + textPayloadFields.put(SQL_STATEMENT, NOT_AVAILABLE); + textPayloadFields.put(Constants.DB_NAME, NOT_AVAILABLE); + textPayloadFields.put(ERROR, NOT_AVAILABLE); + if (textPayload == null) { + logger.debug( + "textPayload is null, leaving defaults in textPayloadFields={}", textPayloadFields); + return; + } + + Matcher matcher = hostPattern.matcher(textPayload); + if (matcher.find()) { + textPayloadFields.put(IP, matcher.group(1)); + } + + matcher = portPattern.matcher(textPayload); + if (matcher.find()) { + textPayloadFields.put(PORT, matcher.group(1)); + } + + matcher = sqlStatementPattern.matcher(textPayload); + if (matcher.find()) { + textPayloadFields.put(SQL_STATEMENT, matcher.group(1)); + } + + matcher = alternateSqlStatementPattern.matcher(textPayload); + if (matcher.find()) { + textPayloadFields.put(SQL_STATEMENT, matcher.group(1)); + } + + matcher = dbPattern.matcher(textPayload); + if (matcher.find()) { + textPayloadFields.put(Constants.DB_NAME, matcher.group(1)); + } + + matcher = dbUserPattern.matcher(textPayload); + if (matcher.find()) { + textPayloadFields.put(Constants.DB_USER, matcher.group(1)); + } + + matcher = errorPattern.matcher(textPayload); + if (matcher.find()) { + textPayloadFields.put(ERROR, matcher.group(1)); + } + + matcher = fatalErrorPattern.matcher(textPayload); + if (matcher.find()) { + textPayloadFields.put(ERROR, matcher.group(1)); + } + } + + @Override + protected Integer getClientPort(String payload) { + int clientPort; + try { + clientPort = Integer.parseInt(textPayloadFields.get(PORT)); + logger.debug("Parsed client port: {}", clientPort); + return clientPort; + } catch (Exception e) { + return -1; + } + } + + @Override + protected String getClientIp(String payload) { + String clientIp; + if (Objects.equals(textPayloadFields.get(IP), DEFAULT_IP)) { + clientIp = getValue(payload, CLIENT_IP); + } else { + clientIp = textPayloadFields.get(IP); + } + logger.debug("Parsed client ip: {}", clientIp); + + if (Objects.equals(clientIp, "") || clientIp == null) { + return DEFAULT_IP; + } + return clientIp; + } + + @Override + protected String getClientIpv6(String payload) { + String clientIp; + if (Objects.equals(textPayloadFields.get(IP), DEFAULT_IP)) { + clientIp = getValue(payload, CLIENT_IP); + } else { + clientIp = textPayloadFields.get(IP); + } + logger.debug("Parsed client ipv6: {}", clientIp); + + if (Objects.equals(clientIp, "") || clientIp == null) { + return DEFAULT_IPV6; + } + return clientIp; + } + + @Override + protected String getSqlString(String payload) { + String sqlString = textPayloadFields.get(SQL_STATEMENT); + logger.debug("Extracted SQL string from textPayloadFields: {}", sqlString); + if (sqlString != null && !sqlString.equals(NOT_AVAILABLE)) { + return sqlString + .replace("\n", " ") + .replaceAll("\\s{2,}", " ") + .replaceAll("\\(\\s+", "(") + .replaceAll("\\s+\\)", ")") + .trim(); + } else { + logger.debug("SQL string not available in textPayloadFields, returning NOT_AVAILABLE"); + return NOT_AVAILABLE; + } + } + + @Override + protected String getServerHostName(String payload) { + return NOT_AVAILABLE; + } + + @Override + protected String getDbUser(String payload) { + String dbUser = ""; + if (Objects.equals(textPayloadFields.get(Constants.DB_USER), NOT_AVAILABLE)) { + dbUser = getValue(payload, Constants.DB_USER); + } else { + dbUser = textPayloadFields.get(Constants.DB_USER); + } + logger.debug("Parsed dbUser: {}", dbUser); + if (Objects.equals(dbUser, "") || dbUser == null) { + dbUser = NOT_AVAILABLE; + } + return dbUser; + } + + @Override + protected String getDbName(String payload) { + String dbName = ""; + if (Objects.equals(textPayloadFields.get(Constants.DB_NAME), NOT_AVAILABLE)) { + dbName = getValue(payload, Constants.DB_NAME); + } else { + dbName = textPayloadFields.get(Constants.DB_NAME); + } + logger.debug("Parsed dbName: {}", dbName); + if (Objects.equals(dbName, "") || dbName == null) { + dbName = NOT_AVAILABLE; + } + return dbName; + } + + @Override + protected String getServiceName(String payload) { + return getDbName(payload); + } + + @Override + protected ExceptionRecord getException(String payload, String sqlString) { + String severityType = getValue(payload, Constants.EXCEPTION_TYPE_ID); + if (!Objects.equals(severityType, "ERROR") + && !Objects.equals(severityType, "CRITICAL") + && (!Objects.equals(severityType, "ALERT"))) { + + return null; + } else { + ExceptionRecord exceptionRecord = new ExceptionRecord(); + logger.debug("Creating ExceptionRecord with severityType: {}", severityType); + if (Objects.equals(severityType, "ALERT")) { + exceptionRecord.setExceptionTypeId(EXCEPTION_TYPE_AUTHENTICATION_STRING); + } else { + exceptionRecord.setExceptionTypeId(EXCEPTION_TYPE_AUTHORIZATION_STRING); + } + exceptionRecord.setDescription(textPayloadFields.get(ERROR)); + exceptionRecord.setSqlString(getSqlString(payload)); + return exceptionRecord; + } + } + + @Override + protected String getSessionId(String payload) { + return DEFAULT_STRING; + } + + @Override + protected String getLanguage(String payload) { + return getValue(payload, LANGUAGE); + } + + @Override + protected String getDataType(String payload) { + return getValue(payload, DATA_TYPE); + } + + @Override + protected String getServerType(String payload) { + return getValue(payload, Constants.SERVER_TYPE); + } +} diff --git a/filter-plugin/logstash-filter-alloydb-guardium/src/test/java/com/ibm/guardium/alloydb/AlloyDBGuardiumFilterTest.java b/filter-plugin/logstash-filter-alloydb-guardium/src/test/java/com/ibm/guardium/alloydb/AlloyDBGuardiumFilterTest.java new file mode 100644 index 000000000..af17fd5f0 --- /dev/null +++ b/filter-plugin/logstash-filter-alloydb-guardium/src/test/java/com/ibm/guardium/alloydb/AlloyDBGuardiumFilterTest.java @@ -0,0 +1,73 @@ +package com.ibm.guardium.alloydb; + +import co.elastic.logstash.api.Event; +import co.elastic.logstash.api.FilterMatchListener; +import co.elastic.logstash.api.Configuration; +import co.elastic.logstash.api.Context; +import org.logstash.plugins.ConfigurationImpl; +import org.logstash.plugins.ContextImpl; +import org.junit.jupiter.api.Test; + +import java.util.Collection; +import java.util.Collections; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.jupiter.api.Assertions.assertNotNull; + +class AlloyDBGuardiumFilterTest { + + FilterMatchListener matchListener = new TestMatchListener(); + + String id = "1"; + Configuration config = new ConfigurationImpl(Collections.singletonMap("source", "")); + Context context = new ContextImpl(null, null); + AlloyDBGuardiumFilter filter = new AlloyDBGuardiumFilter(id, config, context); + + @Test + void test() { + String payload = + "{\n" + + " \"textPayload\": \"2025-05-05 19:10:44.336 UTC [70374]: [1-1] db=alloydb_postgresql,user=postgres LOG: [postgres.c:1187] statement: SELECT * FROM customers\",\n" + + " \"insertId\": \"s=21bfbbc9ae684df79c81633b63e1fb33;i=18d4f1;b=488acbc40da24f1c85d08b22a1969f2b;m=3cf02fe71;t=6346841bbd621;x=1f898f7d8bd2cd3-0-0@a1\",\n" + + " \"resource\": {\n" + + " \"type\": \"alloydb.googleapis.com/Instance\",\n" + + " \"labels\": {\n" + + " \"location\": \"us-central1\",\n" + + " \"resource_container\": \"projects/485533885456\",\n" + + " \"cluster_id\": \"alloydb-postgresql\",\n" + + " \"instance_id\": \"alloydb-postgresql-instance\"\n" + + " }\n" + + " },\n" + + " \"timestamp\": \"2025-05-05T19:10:44.336161Z\",\n" + + " \"severity\": \"INFO\",\n" + + " \"labels\": {\n" + + " \"DATABASE_VERSION\": \"POSTGRES_15\",\n" + + " \"CONSUMER_PROJECT_NUMBER\": \"485533885456\",\n" + + " \"CONSUMER_PROJECT\": \"project-name\",\n" + + " \"NODE_ID\": \"kvqd\"\n" + + " },\n" + + " \"logName\": \"projects/project-name/logs/alloydb.googleapis.com%2Fpostgres.log\",\n" + + " \"receiveTimestamp\": \"2025-05-05T19:11:05.292741654Z\"\n" + + "}"; + + Event event = new org.logstash.Event(); + event.setField("message", payload); + Collection actualResponse = + filter.filter(Collections.singletonList(event), matchListener); + + assertNotNull(actualResponse.toArray(new Event[0])[0].getField("GuardRecord")); + } + + class TestMatchListener implements FilterMatchListener { + private AtomicInteger matchCount = new AtomicInteger(0); + + public int getMatchCount() { + return matchCount.get(); + } + + @Override + public void filterMatched(co.elastic.logstash.api.Event arg0) { + matchCount.incrementAndGet(); + } + } +} diff --git a/filter-plugin/logstash-filter-alloydb-guardium/src/test/java/com/ibm/guardium/alloydb/ParserTest.java b/filter-plugin/logstash-filter-alloydb-guardium/src/test/java/com/ibm/guardium/alloydb/ParserTest.java new file mode 100644 index 000000000..528d73910 --- /dev/null +++ b/filter-plugin/logstash-filter-alloydb-guardium/src/test/java/com/ibm/guardium/alloydb/ParserTest.java @@ -0,0 +1,301 @@ +package com.ibm.guardium.alloydb; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.ibm.guardium.universalconnector.commons.custom_parsing.ParserFactory; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +class ParserTest { + + Parser parser = new Parser(ParserFactory.ParserType.json); + + @Test + void testSqlExecution() { + String payload = + "{\n" + + " \"textPayload\": \"2025-05-05 19:10:44.336 UTC [70374]: [1-1] db=alloydb_postgresql,user=postgres LOG: [postgres.c:1187] statement: SELECT * FROM customers\",\n" + + " \"insertId\": \"s=21bfbbc9ae684df79c81633b63e1fb33;i=18d4f1;b=488acbc40da24f1c85d08b22a1969f2b;m=3cf02fe71;t=6346841bbd621;x=1f898f7d8bd2cd3-0-0@a1\",\n" + + " \"resource\": {\n" + + " \"type\": \"alloydb.googleapis.com/Instance\",\n" + + " \"labels\": {\n" + + " \"location\": \"us-central1\",\n" + + " \"resource_container\": \"projects/485533885456\",\n" + + " \"cluster_id\": \"alloydb-postgresql\",\n" + + " \"instance_id\": \"alloydb-postgresql-instance\"\n" + + " }\n" + + " },\n" + + " \"timestamp\": \"2025-05-05T19:10:44.336161Z\",\n" + + " \"severity\": \"INFO\",\n" + + " \"labels\": {\n" + + " \"DATABASE_VERSION\": \"POSTGRES_15\",\n" + + " \"CONSUMER_PROJECT_NUMBER\": \"485533885456\",\n" + + " \"CONSUMER_PROJECT\": \"project-name\",\n" + + " \"NODE_ID\": \"kvqd\"\n" + + " },\n" + + " \"logName\": \"projects/project-name/logs/alloydb.googleapis.com%2Fpostgres.log\",\n" + + " \"receiveTimestamp\": \"2025-05-05T19:11:05.292741654Z\"\n" + + "}"; + Record record = parser.parseRecord(payload); + assertNotNull(record); + assertEquals("alloydb_postgresql", record.getDbName()); + assertEquals(-1, record.getSessionLocator().getClientPort()); + assertEquals("0.0.0.0", record.getSessionLocator().getClientIp()); + assertEquals(-1, record.getSessionLocator().getServerPort()); + assertEquals("0.0.0.0", record.getSessionLocator().getServerIp()); + assertEquals("ALLOYDB", record.getAccessor().getDbProtocol()); + assertEquals("postgres", record.getAccessor().getDbUser()); + assertEquals("ALLOYDB", record.getAccessor().getServerType()); + assertEquals("alloydb_postgresql", record.getAccessor().getServiceName()); + assertEquals( + "Time{timstamp=1746472244336, minOffsetFromGMT=0, minDst=0}", record.getTime().toString()); + assertNull(record.getException()); + assertEquals("SELECT * FROM customers", record.getData().getOriginalSqlCommand()); + assertEquals("PGRS", record.getAccessor().getLanguage()); + } + + @Test + void testError() { + + String payload = + "{\n" + + " \"textPayload\": \"2025-05-05 16:38:52.621 UTC [32230]: [1-1] db=alloydb_postgresql,user=postgres ERROR:[parse_relation.c:1395]relation \\\"cars5\\\" does not exist at character 15\",\n" + + " \"insertId\": \"s=21bfbbc9ae684df79c81633b63e1fb33;i=b0a8c;b=488acbc40da24f1c85d08b22a1969f2b;m=1afe942f8;t=6346622a21aa8;x=588441c25ef80d7f-0-0@a1\",\n" + + " \"resource\": {\n" + + " \"type\": \"alloydb.googleapis.com/Instance\",\n" + + " \"labels\": {\n" + + " \"cluster_id\": \"alloydb-postgresql\",\n" + + " \"location\": \"us-central1\",\n" + + " \"instance_id\": \"alloydb-postgresql\",\n" + + " \"resource_container\": \"projects/485533885456\"\n" + + " }\n" + + " },\n" + + " \"timestamp\": \"2025-05-05T16:38:52.621480Z\",\n" + + " \"severity\": \"ERROR\",\n" + + " \"labels\": {\n" + + " \"CONSUMER_PROJECT_NUMBER\": \"485533885456\",\n" + + " \"CONSUMER_PROJECT\": \"project-name\",\n" + + " \"DATABASE_VERSION\": \"POSTGRES_15\",\n" + + " \"NODE_ID\": \"kvqd\"\n" + + " },\n" + + " \"logName\": \"projects/project-name/logs/alloydb.googleapis.com%2Fpostgres.log\",\n" + + " \"receiveTimestamp\": \"2025-05-05T16:38:53.360493534Z\"\n" + + "}"; + Record record = parser.parseRecord(payload); + assertNotNull(record); + assertEquals("alloydb_postgresql", record.getDbName()); + assertEquals(-1, record.getSessionLocator().getClientPort()); + assertEquals("0.0.0.0", record.getSessionLocator().getClientIp()); + assertEquals(-1, record.getSessionLocator().getServerPort()); + assertFalse(record.getSessionLocator().isIpv6()); + assertEquals("0.0.0.0", record.getSessionLocator().getServerIp()); + assertEquals("ALLOYDB", record.getAccessor().getDbProtocol()); + assertEquals("postgres", record.getAccessor().getDbUser()); + assertEquals("ALLOYDB", record.getAccessor().getServerType()); + assertEquals( + "Time{timstamp=1746463132621, minOffsetFromGMT=0, minDst=0}", record.getTime().toString()); + assertNotNull(record.getException()); + assertEquals("SQL_ERROR", record.getException().getExceptionTypeId()); + assertEquals( + "[parse_relation.c:1395]relation \"cars5\" does not exist at character 15", + record.getException().getDescription()); + assertEquals("alloydb_postgresql", record.getAccessor().getServiceName()); + } + + @Test + void testLoginFailed() { + String payload = + "{\n" + + " \"textPayload\": \"2025-06-19 16:40:44.090 UTC [324511]: [1-1] db=postgres,user=postgres FATAL:[auth.c:372] password authentication failed for user \\\"postgres\\\"\",\n" + + " \"insertId\": \"s=81da2c12023949fd8a8f279760f1aa98;i=7a5314;b=f0731cb0ca664583b8609af6ac5059d7;m=136726d1ac;t=637ef6822df2a;x=7b81f3739ae5686b-0-0@a1\",\n" + + " \"resource\": {\n" + + " \"type\": \"alloydb.googleapis.com/Instance\",\n" + + " \"labels\": {\n" + + " \"cluster_id\": \"alloydb-test\",\n" + + " \"instance_id\": \"alloydb-test-primary\",\n" + + " \"location\": \"us-central1\",\n" + + " \"resource_container\": \"projects/485533885456\"\n" + + " }\n" + + " },\n" + + " \"timestamp\": \"2025-06-19T16:40:44.091178Z\",\n" + + " \"severity\": \"ALERT\",\n" + + " \"labels\": {\n" + + " \"CONSUMER_PROJECT\": \"project-name\",\n" + + " \"NODE_ID\": \"m3df\",\n" + + " \"CONSUMER_PROJECT_NUMBER\": \"485533885456\",\n" + + " \"DATABASE_VERSION\": \"POSTGRES_16\"\n" + + " },\n" + + " \"logName\": \"projects/project-name/logs/alloydb.googleapis.com%2Fpostgres.log\",\n" + + " \"receiveTimestamp\": \"2025-06-19T16:40:45.240192087Z\"\n" + + "}"; + + Record record = parser.parseRecord(payload); + assertNotNull(record); + assertNotNull(record.getException()); + assertEquals("LOGIN_FAILED", record.getException().getExceptionTypeId()); + assertEquals( + "[auth.c:372] password authentication failed for user \"postgres\"", + record.getException().getDescription()); + } + + @Test + void testFailedSQL() { + String payload = + "{\n" + + " \"textPayload\": \"2025-09-17 18:32:35.153 UTC [2918167]: [3-1] db=postgres,user=postgres STATEMENT: select * from employs;\",\n" + + " \"insertId\": \"s=e839ffda58874d68b3330617daee3ed8;i=4f01bd8;b=e9292a297c8c447ea7efe8b2bb0b6a03;m=a35f9c87ac;t=63f0375dd5100;x=4ccb2771207ce73f-0-0@a1\",\n" + + " \"resource\": {\n" + + " \"type\": \"alloydb.googleapis.com/Instance\",\n" + + " \"labels\": {\n" + + " \"resource_container\": \"projects/485533885456\",\n" + + " \"instance_id\": \"glenn-alloydb-cluster-primary\",\n" + + " \"cluster_id\": \"glenn-alloydb-cluster\",\n" + + " \"location\": \"us-east4\"\n" + + " }\n" + + " },\n" + + " \"timestamp\": \"2025-09-17T18:32:35.154176Z\",\n" + + " \"severity\": \"ERROR\",\n" + + " \"labels\": {\n" + + " \"CONSUMER_PROJECT_NUMBER\": \"485533885456\",\n" + + " \"CONSUMER_PROJECT\": \"project-name\",\n" + + " \"DATABASE_VERSION\": \"POSTGRES_16\",\n" + + " \"NODE_ID\": \"k351\"\n" + + " },\n" + + " \"logName\": \"projects/project-name/logs/alloydb.googleapis.com%2Fpostgres.log\",\n" + + " \"receiveTimestamp\": \"2025-09-17T18:33:06.249293611Z\"\n" + + "}"; + Record record = parser.parseRecord(payload); + assertNotNull(record); + assertNotNull(record.getException()); + assertEquals("select * from employs;", record.getException().getSqlString()); + } + + @Test + void test2() { + String payload = + "{\n" + + " \"textPayload\": \"2025-10-16 09:39:40.843 UTC [408063]: [1-1] db=postgres,user=postgres LOG: [postgres.c:1214] statement: CREATE TABLE employees (\\n id SERIAL PRIMARY KEY,\\n name VARCHAR(50),\\n department VARCHAR(50),\\n salary NUMERIC(10,2)\\n);\\n\\n\\nINSERT INTO employees (name, department, salary) VALUES\\n('Alice', 'Engineering', 90000),\\n('Bob', 'Marketing', 75000),\\n('Charlie', 'Finance', 82000),\\n('Diana', 'Engineering', 95000);\\n\\n\\nSELECT * FROM employees;\\n\\n\\nUPDATE employees\\nSET salary = 98000\\nWHERE name = 'Diana';\\n\\n\\nSELECT * FROM employees WHERE name = 'Diana';\\n\\n\\nDELETE FROM employees\\nWHERE name = 'Bob';\\n\\n\\nSELECT * FROM employees;\\n\\nDROP TABLE employees;\\n\",\n" + + " \"insertId\": \"s=935ad8a1f4a1460aa2d252313f784084;i=b224ca;b=39deecf7c2a1461893339287af93c848;m=169d8a1ad7;t=641436571f066;x=f13b8397a707becc-0-0@a1\",\n" + + " \"resource\": {\n" + + " \"type\": \"alloydb.googleapis.com/Instance\",\n" + + " \"labels\": {\n" + + " \"location\": \"us-central1\",\n" + + " \"cluster_id\": \"my-cluster\",\n" + + " \"instance_id\": \"my-cluster-primary\",\n" + + " \"resource_container\": \"projects/485533885456\"\n" + + " }\n" + + " },\n" + + " \"timestamp\": \"2025-10-16T09:39:40.844134Z\",\n" + + " \"severity\": \"INFO\",\n" + + " \"labels\": {\n" + + " \"NODE_ID\": \"xq84\",\n" + + " \"CONSUMER_PROJECT\": \"project-name\",\n" + + " \"DATABASE_VERSION\": \"POSTGRES_16\",\n" + + " \"CONSUMER_PROJECT_NUMBER\": \"485533885456\"\n" + + " },\n" + + " \"logName\": \"projects/project-name/logs/alloydb.googleapis.com%2Fpostgres.log\",\n" + + " \"receiveTimestamp\": \"2025-10-16T09:39:41.849130289Z\"\n" + + "}"; + + Record record = parser.parseRecord(payload); + assertNotNull(record); + assertEquals("postgres", record.getDbName()); + + assertEquals("postgres", record.getAccessor().getDbUser()); + assertEquals("ALLOYDB", record.getAccessor().getServerType()); + assertEquals("postgres", record.getAccessor().getServiceName()); + + assertNull(record.getException()); + assertEquals( + "CREATE TABLE employees (id SERIAL PRIMARY KEY, name VARCHAR(50), department VARCHAR(50), salary NUMERIC(10,2)); INSERT INTO employees (name, department, salary) VALUES ('Alice', 'Engineering', 90000), ('Bob', 'Marketing', 75000), ('Charlie', 'Finance', 82000), ('Diana', 'Engineering', 95000); SELECT * FROM employees; UPDATE employees SET salary = 98000 WHERE name = 'Diana'; SELECT * FROM employees WHERE name = 'Diana'; DELETE FROM employees WHERE name = 'Bob'; SELECT * FROM employees; DROP TABLE employees;", + record.getData().getOriginalSqlCommand()); + assertEquals("PGRS", record.getAccessor().getLanguage()); + } + + @Test + void test3() { + String payload = + "{\n" + + " \"textPayload\": \"2025-10-17 10:48:39.823 UTC [780051]: [347-1] db=postgres,user=postgres LOG: [postgres.c:2408] execute : SELECT COUNT(*) AS \\\"RECORDS\\\" FROM PANY\",\n" + + " \"insertId\": \"s=935ad8a1f4a1460aa2d252313f784084;i=15403ce;b=39deecf7c2a1461893339287af93c848;m=2bb2154cbb;t=6415879fd2249;x=8840df928313b684-0-0@a1\",\n" + + " \"resource\": {\n" + + " \"type\": \"alloydb.googleapis.com/Instance\",\n" + + " \"labels\": {\n" + + " \"cluster_id\": \"my-cluster\",\n" + + " \"resource_container\": \"projects/485533885456\",\n" + + " \"instance_id\": \"my-cluster-primary\",\n" + + " \"location\": \"us-central1\"\n" + + " }\n" + + " },\n" + + " \"timestamp\": \"2025-10-17T10:48:39.823945Z\",\n" + + " \"severity\": \"INFO\",\n" + + " \"labels\": {\n" + + " \"NODE_ID\": \"xq84\",\n" + + " \"DATABASE_VERSION\": \"POSTGRES_16\",\n" + + " \"CONSUMER_PROJECT\": \"project-name\",\n" + + " \"CONSUMER_PROJECT_NUMBER\": \"485533885456\"\n" + + " },\n" + + " \"logName\": \"projects/project-name/logs/alloydb.googleapis.com%2Fpostgres.log\",\n" + + " \"receiveTimestamp\": \"2025-10-17T10:48:40.901838976Z\"\n" + + "}"; + + Record record = parser.parseRecord(payload); + assertNotNull(record); + assertEquals("postgres", record.getDbName()); + + assertEquals("postgres", record.getAccessor().getDbUser()); + assertEquals("ALLOYDB", record.getAccessor().getServerType()); + assertEquals("postgres", record.getAccessor().getServiceName()); + + assertNull(record.getException()); + assertEquals( + "SELECT COUNT(*) AS \"RECORDS\" FROM PANY", record.getData().getOriginalSqlCommand()); + } + + @Test + void test4() { + String payload = + "{\n" + + " \"textPayload\": \"2025-10-16 04:22:05.937 UTC [328339]: [1-1] db=guestbook,user=postgres LOG: [postgres.c:1214] statement: CREATE TABLE PANY(ID INT PRIMARY KEY NOT NULL,NAME TEXT NOT NULL,AGE INT NOT NULL,ADDRESS CHAR(50),SALARY REAL);\\nCREATE TABLE DPT(ID INT PRIMARY KEY NOT NULL,DEPT CHAR(50) NOT NULL,EMP_ID INT NOT NULL);\\nINSERT INTO PANY (ID,NAME,AGE,ADDRESS,SALARY) VALUES (1, 'Paul', 32, 'California', 20000.00);\\nINSERT INTO PANY (ID,NAME,AGE,ADDRESS) VALUES (2, 'Allen', 25, 'Texas');\\nINSERT INTO PANY (ID,NAME,AGE,ADDRESS,SALARY) VALUES (3, 'Teddy', 23, 'Norway', 20000.00 );\\nINSERT INTO PANY (ID,NAME,AGE,ADDRESS,SALARY) VALUES (4, 'Mark', 25, 'Rich-Mond ', 65000.00 ), (5, 'David', 27, 'Texas', 85000.00);\\nSELECT ID, NAME, SALARY FROM PANY;\\nSELECT * FROM PANY WHERE SALARY = 10000;\\nSELECT COUNT(*) AS \\\"RECORDS\\\" FROM PANY;\\nSELECT * FROM PANY WHERE AGE >= 25 AND SALARY >= 65000;\\nSELECT * FROM PANY WHERE AGE >= 25 OR SALARY >= 65000;\\nSELECT * FROM PANY WHERE AGE IS NOT NULL;\\nSELECT * FROM PANY WHERE NAME LIKE 'Pa%';\\nSELECT * FROM PANY WHERE AGE IN ( 25, 27 );\\nSELECT * FROM PANY WHERE AGE NOT IN ( 25, 27 );\\nSELECT * FROM PANY WHERE AGE BETWEEN 25 AND 27;\\nUPDATE PANY SET SALARY = 15000 WHERE ID = 3;\\nUPDATE PANY SET ADDRESS = 'Texas', SALARY=20000;\\nDELETE FROM PANY WHERE ID = 2;\\n\\ndrop table PANY;\\ndrop table DPT;\",\n" + + " \"insertId\": \"s=935ad8a1f4a1460aa2d252313f784084;i=8ff679;b=39deecf7c2a1461893339287af93c848;m=122dc74f75;t=6413ef5af2504;x=1cfa30b9194c0d1d-0-0@a1\",\n" + + " \"resource\": {\n" + + " \"type\": \"alloydb.googleapis.com/Instance\",\n" + + " \"labels\": {\n" + + " \"cluster_id\": \"my-cluster\",\n" + + " \"location\": \"us-central1\",\n" + + " \"instance_id\": \"my-cluster-primary\",\n" + + " \"resource_container\": \"projects/485533885456\"\n" + + " }\n" + + " },\n" + + " \"timestamp\": \"2025-10-16T04:22:05.937924Z\",\n" + + " \"severity\": \"INFO\",\n" + + " \"labels\": {\n" + + " \"CONSUMER_PROJECT_NUMBER\": \"485533885456\",\n" + + " \"DATABASE_VERSION\": \"POSTGRES_16\",\n" + + " \"CONSUMER_PROJECT\": \"project-name\",\n" + + " \"NODE_ID\": \"xq84\"\n" + + " },\n" + + " \"logName\": \"projects/project-name/logs/alloydb.googleapis.com%2Fpostgres.log\",\n" + + " \"receiveTimestamp\": \"2025-10-16T04:22:06.439003426Z\"\n" + + "}"; + + Record record = parser.parseRecord(payload); + assertNotNull(record); + assertEquals("guestbook", record.getDbName()); + assertEquals("guestbook", record.getAccessor().getServiceName()); + assertEquals("postgres", record.getAccessor().getDbUser()); + assertEquals("ALLOYDB", record.getAccessor().getServerType()); + + assertNull(record.getException()); + assertEquals( + "CREATE TABLE PANY(ID INT PRIMARY KEY NOT NULL,NAME TEXT NOT NULL,AGE INT NOT NULL,ADDRESS CHAR(50),SALARY REAL); CREATE TABLE DPT(ID INT PRIMARY KEY NOT NULL,DEPT CHAR(50) NOT NULL,EMP_ID INT NOT NULL); INSERT INTO PANY (ID,NAME,AGE,ADDRESS,SALARY) VALUES (1, 'Paul', 32, 'California', 20000.00); INSERT INTO PANY (ID,NAME,AGE,ADDRESS) VALUES (2, 'Allen', 25, 'Texas'); INSERT INTO PANY (ID,NAME,AGE,ADDRESS,SALARY) VALUES (3, 'Teddy', 23, 'Norway', 20000.00); INSERT INTO PANY (ID,NAME,AGE,ADDRESS,SALARY) VALUES (4, 'Mark', 25, 'Rich-Mond ', 65000.00), (5, 'David', 27, 'Texas', 85000.00); SELECT ID, NAME, SALARY FROM PANY; SELECT * FROM PANY WHERE SALARY = 10000; SELECT COUNT(*) AS \"RECORDS\" FROM PANY; SELECT * FROM PANY WHERE AGE >= 25 AND SALARY >= 65000; SELECT * FROM PANY WHERE AGE >= 25 OR SALARY >= 65000; SELECT * FROM PANY WHERE AGE IS NOT NULL; SELECT * FROM PANY WHERE NAME LIKE 'Pa%'; SELECT * FROM PANY WHERE AGE IN (25, 27); SELECT * FROM PANY WHERE AGE NOT IN (25, 27); SELECT * FROM PANY WHERE AGE BETWEEN 25 AND 27; UPDATE PANY SET SALARY = 15000 WHERE ID = 3; UPDATE PANY SET ADDRESS = 'Texas', SALARY=20000; DELETE FROM PANY WHERE ID = 2; drop table PANY; drop table DPT;", + record.getData().getOriginalSqlCommand()); + } +} diff --git a/filter-plugin/logstash-filter-aurora-mysql-guardium/README.md b/filter-plugin/logstash-filter-aurora-mysql-guardium/README.md index 335286e88..17b3caa7d 100644 --- a/filter-plugin/logstash-filter-aurora-mysql-guardium/README.md +++ b/filter-plugin/logstash-filter-aurora-mysql-guardium/README.md @@ -68,6 +68,7 @@ By default, each database instance has an associated log group with a name in th • The aurora-mysql plug-in does not support IPV6. • The aurora-mysql auditing does not audit Procedure, Function, and Show tables operations. • The source program will be seen as blank in the report. + • Syntactically incorrect queries are not captured in audit logs. ## 4. Configuring the aurora-mysql filters in Guardium diff --git a/filter-plugin/logstash-filter-aurora-mysql-guardium/build.gradle b/filter-plugin/logstash-filter-aurora-mysql-guardium/build.gradle index ce96925df..09b88812f 100644 --- a/filter-plugin/logstash-filter-aurora-mysql-guardium/build.gradle +++ b/filter-plugin/logstash-filter-aurora-mysql-guardium/build.gradle @@ -2,6 +2,30 @@ import java.nio.file.Files import static java.nio.file.StandardCopyOption.REPLACE_EXISTING apply plugin: 'java' + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + + dependencies { + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' + classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.1.0" + classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' + } +} + +def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" // =========================================================================== @@ -20,10 +44,10 @@ pluginInfo.pluginClass = "AuroraMysqlGuardiumPluginFilter" pluginInfo.pluginName = "auroramysqlguardiumpluginfilter" // must match the @LogstashPlugin annotation in the main plugin class // =========================================================================== -sourceCompatibility = 1.8 -targetCompatibility = 1.8 +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 -def jacocoVersion = '0.8.4' +def jacocoVersion = '0.8.11' // minimumCoverage can be set by Travis ENV def minimumCoverageStr = System.getenv("MINIMUM_COVERAGE") ?: "50.0%" if (minimumCoverageStr.endsWith("%")) { @@ -31,27 +55,16 @@ if (minimumCoverageStr.endsWith("%")) { } def minimumCoverage = Float.valueOf(minimumCoverageStr) / 100 -buildscript { - repositories { - maven { - url "https://plugins.gradle.org/m2/" - } - mavenCentral() - jcenter() - } - - dependencies { - classpath 'com.github.jengelman.gradle.plugins:shadow:4.0.4' - classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.0.0-RC2" - classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' - } -} - -def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); -def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) repositories { - mavenCentral() + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } } tasks.register("vendor"){ @@ -62,14 +75,13 @@ tasks.register("vendor"){ File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") projectJarFile.mkdirs() Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) - validatePluginJar(projectJarFile, project.group) } } apply plugin: 'com.github.johnrengelman.shadow' shadowJar { - classifier = null + archiveClassifier = null } dependencies { @@ -77,6 +89,7 @@ dependencies { implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: versions.dependencies.log4jCore implementation 'org.apache.commons:commons-lang3:' + versions.dependencies.commonsLang implementation 'com.google.code.gson:gson:' + versions.dependencies.gson + implementation group: 'commons-beanutils', name: 'commons-beanutils', version: versions.dependencies.commonsBeanutils implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "build/libs/logstash-core-*.*.*.jar") implementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "common-*.*.*.jar") @@ -102,6 +115,17 @@ tasks.withType(JavaCompile) { options.encoding = 'UTF-8' } +test { + jvmArgs = [ + '--add-opens=java.base/java.lang=ALL-UNNAMED', + '--add-opens=java.base/java.util=ALL-UNNAMED', + '--add-opens=java.base/java.text=ALL-UNNAMED', + '--add-opens=java.base/java.lang.reflect=ALL-UNNAMED', + '--add-opens=java.base/sun.util.resources=ALL-UNNAMED', + '--add-opens=java.base/sun.util.cldr=ALL-UNNAMED' + ] +} + tasks.register("generateRubySupportFiles") { doLast { generateRubySupportFilesForPlugin(project.description, project.group, version) @@ -144,17 +168,16 @@ apply plugin: "org.barfuin.gradle.jacocolog" // ------------------------------------ jacoco { toolVersion = "${jacocoVersion}" - reportsDir = file("$buildDir/reports/jacoco") } jacocoTestReport { // You will see "Report -> file://...." at the end of a JaCoCo build // If no output, run this first: ./gradlew test reports { - html.enabled true - xml.enabled true - csv.enabled true - html.destination file("${buildDir}/reports/jacoco") - csv.destination file("${buildDir}/reports/jacoco/all.csv") + html.required = true + xml.required = true + csv.required = true + html.outputLocation = file("${buildDir}/reports/jacoco") + csv.outputLocation = file("${buildDir}/reports/jacoco/all.csv") } executionData.from fileTree(dir: "${buildDir}/jacoco/", includes: [ '**/*.exec' diff --git a/filter-plugin/logstash-filter-aurora-mysql-guardium/gradle/wrapper/gradle-wrapper.properties b/filter-plugin/logstash-filter-aurora-mysql-guardium/gradle/wrapper/gradle-wrapper.properties index aa991fcea..81aa1c044 100644 --- a/filter-plugin/logstash-filter-aurora-mysql-guardium/gradle/wrapper/gradle-wrapper.properties +++ b/filter-plugin/logstash-filter-aurora-mysql-guardium/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/filter-plugin/logstash-filter-aurora-mysql-guardium/mainREADME.md b/filter-plugin/logstash-filter-aurora-mysql-guardium/mainREADME.md new file mode 100644 index 000000000..d6306dd05 --- /dev/null +++ b/filter-plugin/logstash-filter-aurora-mysql-guardium/mainREADME.md @@ -0,0 +1,9 @@ +# Aurora MySQL Universal Connector + +## Follow this link to set up and use Aurora MySQL Universal Connector over CloudWatch Logstash Plugin + +[AuroraMySqlOverCloudwatch](./README.md) + +## Follow this link to set up and use Aurora MySQL Universal Connector over CloudWatch Connect + +[AuroraMySqlOverConnectCloudwatch](../../docs/KafkaBasedUCs/AuroraMySqlCloudwatchKafkaConnect.md) diff --git a/filter-plugin/logstash-filter-aurora-mysql-guardium/src/main/java/com/ibm/guardium/auroramysql/AuroraMysqlGuardiumPluginFilter.java b/filter-plugin/logstash-filter-aurora-mysql-guardium/src/main/java/com/ibm/guardium/auroramysql/AuroraMysqlGuardiumPluginFilter.java index 047edd91f..11d574e88 100644 --- a/filter-plugin/logstash-filter-aurora-mysql-guardium/src/main/java/com/ibm/guardium/auroramysql/AuroraMysqlGuardiumPluginFilter.java +++ b/filter-plugin/logstash-filter-aurora-mysql-guardium/src/main/java/com/ibm/guardium/auroramysql/AuroraMysqlGuardiumPluginFilter.java @@ -9,7 +9,15 @@ import co.elastic.logstash.api.PluginConfigSpec; import com.google.gson.*; import com.ibm.guardium.universalconnector.commons.GuardConstants; -import com.ibm.guardium.universalconnector.commons.structures.*; +import com.ibm.guardium.universalconnector.commons.structures.Accessor; +import com.ibm.guardium.universalconnector.commons.structures.Construct; +import com.ibm.guardium.universalconnector.commons.structures.Data; +import com.ibm.guardium.universalconnector.commons.structures.ExceptionRecord; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import com.ibm.guardium.universalconnector.commons.structures.Sentence; +import com.ibm.guardium.universalconnector.commons.structures.SentenceObject; +import com.ibm.guardium.universalconnector.commons.structures.SessionLocator; +import com.ibm.guardium.universalconnector.commons.structures.Time; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/filter-plugin/logstash-filter-aurora-mysql-guardium/src/main/java/com/ibm/guardium/auroramysql/Parser.java b/filter-plugin/logstash-filter-aurora-mysql-guardium/src/main/java/com/ibm/guardium/auroramysql/Parser.java index 827d1f173..a895696cb 100644 --- a/filter-plugin/logstash-filter-aurora-mysql-guardium/src/main/java/com/ibm/guardium/auroramysql/Parser.java +++ b/filter-plugin/logstash-filter-aurora-mysql-guardium/src/main/java/com/ibm/guardium/auroramysql/Parser.java @@ -1,7 +1,15 @@ package com.ibm.guardium.auroramysql; import com.google.gson.JsonObject; -import com.ibm.guardium.universalconnector.commons.structures.*; +import com.ibm.guardium.universalconnector.commons.structures.Accessor; +import com.ibm.guardium.universalconnector.commons.structures.Construct; +import com.ibm.guardium.universalconnector.commons.structures.Data; +import com.ibm.guardium.universalconnector.commons.structures.ExceptionRecord; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import com.ibm.guardium.universalconnector.commons.structures.Sentence; +import com.ibm.guardium.universalconnector.commons.structures.SentenceObject; +import com.ibm.guardium.universalconnector.commons.structures.SessionLocator; +import com.ibm.guardium.universalconnector.commons.structures.Time; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/filter-plugin/logstash-filter-aurora-mysql-guardium/src/test/java/com/ibm/guardium/auroramysql/ParserTest.java b/filter-plugin/logstash-filter-aurora-mysql-guardium/src/test/java/com/ibm/guardium/auroramysql/ParserTest.java index 81f74a152..972a3a409 100644 --- a/filter-plugin/logstash-filter-aurora-mysql-guardium/src/test/java/com/ibm/guardium/auroramysql/ParserTest.java +++ b/filter-plugin/logstash-filter-aurora-mysql-guardium/src/test/java/com/ibm/guardium/auroramysql/ParserTest.java @@ -8,7 +8,15 @@ import com.google.gson.JsonObject; import com.ibm.guardium.auroramysql.Constants; import com.ibm.guardium.auroramysql.Parser; -import com.ibm.guardium.universalconnector.commons.structures.*; +import com.ibm.guardium.universalconnector.commons.structures.Accessor; +import com.ibm.guardium.universalconnector.commons.structures.Construct; +import com.ibm.guardium.universalconnector.commons.structures.Data; +import com.ibm.guardium.universalconnector.commons.structures.ExceptionRecord; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import com.ibm.guardium.universalconnector.commons.structures.Sentence; +import com.ibm.guardium.universalconnector.commons.structures.SentenceObject; +import com.ibm.guardium.universalconnector.commons.structures.SessionLocator; +import com.ibm.guardium.universalconnector.commons.structures.Time; import com.ibm.guardium.universalconnector.commons.structures.Record; import org.junit.Assert; diff --git a/filter-plugin/logstash-filter-azure-apachesolr-guardium/README.md b/filter-plugin/logstash-filter-azure-apachesolr-guardium/README.md index 7ee4691c4..e78bcf882 100644 --- a/filter-plugin/logstash-filter-azure-apachesolr-guardium/README.md +++ b/filter-plugin/logstash-filter-azure-apachesolr-guardium/README.md @@ -48,7 +48,8 @@ In addition, events are configured with the include_lines, exclude_lines, and ad • Locate "filebeat.inputs" in the filebeat.yml file, then add the following parameters. filebeat.inputs: - - type: log + - type: filestream + - id: enabled: true paths: - /var/solr/logs/solr.log diff --git a/filter-plugin/logstash-filter-azure-apachesolr-guardium/build.gradle b/filter-plugin/logstash-filter-azure-apachesolr-guardium/build.gradle index 501ec599f..a5677c6ce 100644 --- a/filter-plugin/logstash-filter-azure-apachesolr-guardium/build.gradle +++ b/filter-plugin/logstash-filter-azure-apachesolr-guardium/build.gradle @@ -2,6 +2,31 @@ import java.nio.file.Files import static java.nio.file.StandardCopyOption.REPLACE_EXISTING apply plugin: 'java' + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + + dependencies { + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' + classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.1.0" + classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' + + } +} + +def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" // =========================================================================== @@ -21,10 +46,10 @@ pluginInfo.pluginName = "apache_solr_azure_connector" // must match the @Lo // =========================================================================== -sourceCompatibility = 1.8 -targetCompatibility = 1.8 +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 -def jacocoVersion = '0.8.4' +def jacocoVersion = '0.8.11' // minimumCoverage can be set by Travis ENV def minimumCoverageStr = System.getenv("MINIMUM_COVERAGE") ?: "50.0%" if (minimumCoverageStr.endsWith("%")) { @@ -32,29 +57,17 @@ if (minimumCoverageStr.endsWith("%")) { } def minimumCoverage = Float.valueOf(minimumCoverageStr) / 100 -buildscript { - repositories { - maven { - url "https://plugins.gradle.org/m2/" - } - mavenCentral() - jcenter() - } - - dependencies { - classpath 'com.github.jengelman.gradle.plugins:shadow:4.0.4' - classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.0.0-RC2" - classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' - - } -} - -def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); -def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) repositories { - mavenCentral() + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } } tasks.register("vendor"){ @@ -65,14 +78,13 @@ tasks.register("vendor"){ File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") projectJarFile.mkdirs() Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) - validatePluginJar(projectJarFile, project.group) } } apply plugin: 'com.github.johnrengelman.shadow' shadowJar { - classifier = null + archiveClassifier = null } dependencies { @@ -80,6 +92,7 @@ dependencies { implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: versions.dependencies.log4jCore implementation 'org.apache.commons:commons-lang3:' + versions.dependencies.commonsLang implementation 'com.google.code.gson:gson:' + versions.dependencies.gson + implementation group: 'commons-beanutils', name: 'commons-beanutils', version: versions.dependencies.commonsBeanutils implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "build/libs/logstash-core-*.*.*.jar") implementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "common-*.*.*.jar") @@ -105,6 +118,17 @@ tasks.withType(JavaCompile) { options.encoding = 'UTF-8' } +test { + jvmArgs = [ + '--add-opens=java.base/java.lang=ALL-UNNAMED', + '--add-opens=java.base/java.util=ALL-UNNAMED', + '--add-opens=java.base/java.text=ALL-UNNAMED', + '--add-opens=java.base/java.lang.reflect=ALL-UNNAMED', + '--add-opens=java.base/sun.util.resources=ALL-UNNAMED', + '--add-opens=java.base/sun.util.cldr=ALL-UNNAMED' + ] +} + tasks.register("generateRubySupportFiles") { @@ -147,17 +171,16 @@ apply plugin: "org.barfuin.gradle.jacocolog" // ------------------------------------ jacoco { toolVersion = "${jacocoVersion}" - reportsDir = file("$buildDir/reports/jacoco") } jacocoTestReport { // You will see "Report -> file://...." at the end of a JaCoCo build // If no output, run this first: ./gradlew test reports { - html.enabled true - xml.enabled true - csv.enabled true - html.destination file("${buildDir}/reports/jacoco") - csv.destination file("${buildDir}/reports/jacoco/all.csv") + html.required = true + xml.required = true + csv.required = true + html.outputLocation = file("${buildDir}/reports/jacoco") + csv.outputLocation = file("${buildDir}/reports/jacoco/all.csv") } executionData.from fileTree(dir: "${buildDir}/jacoco/", includes: [ '**/*.exec' diff --git a/filter-plugin/logstash-filter-azure-apachesolr-guardium/gradle/wrapper/gradle-wrapper.properties b/filter-plugin/logstash-filter-azure-apachesolr-guardium/gradle/wrapper/gradle-wrapper.properties index aa991fcea..81aa1c044 100644 --- a/filter-plugin/logstash-filter-azure-apachesolr-guardium/gradle/wrapper/gradle-wrapper.properties +++ b/filter-plugin/logstash-filter-azure-apachesolr-guardium/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/filter-plugin/logstash-filter-azure-postgresql-guardium/build.gradle b/filter-plugin/logstash-filter-azure-postgresql-guardium/build.gradle index f79d9a899..386bfb47f 100644 --- a/filter-plugin/logstash-filter-azure-postgresql-guardium/build.gradle +++ b/filter-plugin/logstash-filter-azure-postgresql-guardium/build.gradle @@ -2,6 +2,30 @@ import java.nio.file.Files import static java.nio.file.StandardCopyOption.REPLACE_EXISTING apply plugin: 'java' + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + + dependencies { + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' + classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.1.0" + classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' + } +} + +def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" // =========================================================================== @@ -21,10 +45,10 @@ pluginInfo.pluginName = "azure_postgresql_guardium_plugin_filter" // must m // ===========================================================================shPlugin annotation in the main plugin class // =========================================================================== -sourceCompatibility = 1.8 -targetCompatibility = 1.8 +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 -def jacocoVersion = '0.8.4' +def jacocoVersion = '0.8.11' // minimumCoverage can be set by Travis ENV def minimumCoverageStr = System.getenv("MINIMUM_COVERAGE") ?: "50.0%" if (minimumCoverageStr.endsWith("%")) { @@ -32,28 +56,17 @@ if (minimumCoverageStr.endsWith("%")) { } def minimumCoverage = Float.valueOf(minimumCoverageStr) / 100 -buildscript { - repositories { - maven { - url "https://plugins.gradle.org/m2/" - } - mavenCentral() - jcenter() - } - - dependencies { - classpath 'com.github.jengelman.gradle.plugins:shadow:4.0.4' - classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.0.0-RC2" - classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' - } -} - -def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); -def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) repositories { - mavenCentral() + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } } tasks.register("vendor"){ @@ -64,14 +77,13 @@ tasks.register("vendor"){ File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") projectJarFile.mkdirs() Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) - validatePluginJar(projectJarFile, project.group) } } apply plugin: 'com.github.johnrengelman.shadow' shadowJar { - classifier = null + archiveClassifier = null } dependencies { @@ -79,6 +91,7 @@ dependencies { implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: versions.dependencies.log4jCore implementation 'org.apache.commons:commons-lang3:' + versions.dependencies.commonsLang implementation 'com.google.code.gson:gson:' + versions.dependencies.gson + implementation group: 'commons-beanutils', name: 'commons-beanutils', version: versions.dependencies.commonsBeanutils implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "build/libs/logstash-core-*.*.*.jar") implementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "common-*.*.*.jar") @@ -102,6 +115,17 @@ tasks.withType(JavaCompile) { options.encoding = 'UTF-8' } +test { + jvmArgs = [ + '--add-opens=java.base/java.lang=ALL-UNNAMED', + '--add-opens=java.base/java.util=ALL-UNNAMED', + '--add-opens=java.base/java.text=ALL-UNNAMED', + '--add-opens=java.base/java.lang.reflect=ALL-UNNAMED', + '--add-opens=java.base/sun.util.resources=ALL-UNNAMED', + '--add-opens=java.base/sun.util.cldr=ALL-UNNAMED' + ] +} + tasks.register("generateRubySupportFiles") { @@ -144,17 +168,16 @@ apply plugin: "org.barfuin.gradle.jacocolog" // ------------------------------------ jacoco { toolVersion = "${jacocoVersion}" - reportsDir = file("$buildDir/reports/jacoco") } jacocoTestReport { // You will see "Report -> file://...." at the end of a JaCoCo build // If no output, run this first: ./gradlew test reports { - html.enabled true - xml.enabled true - csv.enabled true - html.destination file("${buildDir}/reports/jacoco") - csv.destination file("${buildDir}/reports/jacoco/all.csv") + html.required = true + xml.required = true + csv.required = true + html.outputLocation = file("${buildDir}/reports/jacoco") + csv.outputLocation = file("${buildDir}/reports/jacoco/all.csv") } executionData.from fileTree(dir: "${buildDir}/jacoco/", includes: [ '**/*.exec' diff --git a/filter-plugin/logstash-filter-azure-postgresql-guardium/gradle/wrapper/gradle-wrapper.properties b/filter-plugin/logstash-filter-azure-postgresql-guardium/gradle/wrapper/gradle-wrapper.properties index aa991fcea..81aa1c044 100644 --- a/filter-plugin/logstash-filter-azure-postgresql-guardium/gradle/wrapper/gradle-wrapper.properties +++ b/filter-plugin/logstash-filter-azure-postgresql-guardium/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/filter-plugin/logstash-filter-azure-postgresql-guardium/mainREADME.md b/filter-plugin/logstash-filter-azure-postgresql-guardium/mainREADME.md new file mode 100644 index 000000000..bfbeb6ae9 --- /dev/null +++ b/filter-plugin/logstash-filter-azure-postgresql-guardium/mainREADME.md @@ -0,0 +1,10 @@ +# Azure Postgres Universal Connector + +## Follow this link to set up and use Azure Postgres Universal Connector over Azure Eventhub Logstash Plugin + +[PostgresOverEventHub](./README.md) + +## Follow this link to set up and use Azure Postgres Universal Connector over Azure Eventhub Connect + +[PostgresOverConnectEventHub](../../docs/KafkaBasedUCs/AzurePostgreSQLEventHubKafkaConnect.md) + diff --git a/filter-plugin/logstash-filter-azure-sql-guardium/AzureSQLOverJdbcPackage/AzureSQL/filter.conf b/filter-plugin/logstash-filter-azure-sql-guardium/AzureSQLOverJdbcPackage/AzureSQL/filter.conf index 464997173..fcd04ec7d 100644 --- a/filter-plugin/logstash-filter-azure-sql-guardium/AzureSQLOverJdbcPackage/AzureSQL/filter.conf +++ b/filter-plugin/logstash-filter-azure-sql-guardium/AzureSQLOverJdbcPackage/AzureSQL/filter.conf @@ -1,21 +1,20 @@ -filter -{ -if [type] == "azureSQL" { - +filter { + if [type] == "azureSQL" { + if [additional_information] { - mutate { + mutate { add_field => { "[ADDITIONAL_INFORMATION]" => "%{additional_information}"} } } - + if[database_name] { - mutate { + mutate { add_field => { "[DATABASE_NAME]" => "%{enrollmentId}:%{server_instance_name}:%{database_name}"} } } - - mutate { - add_field => { + + mutate { + add_field => { "[TIMESTAMP]" => "%{updatedeventtime}" "[CLIENT_HOST_NAME]" => "%{host_name}" "[STATEMENT]" => "%{statement}" @@ -23,17 +22,15 @@ if [type] == "azureSQL" { "[SERVER_INSTANCE_NAME]" => "%{server_instance_name}" "[SUCCEEDED]" => "%{succeeded}" "[User_Name]" => "%{server_principal_name}" - "[APPLICATION_NAME]" => "%{application_name}" - "[Session_ID]" => "%{session_id}" + "[APPLICATION_NAME]" => "%{application_name}" + "[Session_ID]" => "%{session_id}" "[Server_Hostname]" => "%{enrollmentId}_%{server_instance_name}" } } mutate { gsub => [ "STATEMENT", "[\n]", "" ] } - - azuresql_guardium_plugin_filter{} - - mutate { remove_field => ["@version","type","@timestamp","additional_information","session_id","updatedeventtime","client_ip","User_Name","server_principal_name","statement","host_name","server_instance_name","succeeded","database_name","application_name","TIMESTAMP","CLIENT_HOST_NAME","STATEMENT","Client_IP","SERVER_INSTANCE_NAME","SUCCEEDED","User_Name","APPLICATION_NAME","Session_ID","Server_Hostname","enrollmentId"] } + + azuresql_guardium_plugin_filter{} + + mutate { remove_field => ["@version","type","@timestamp","additional_information","session_id","updatedeventtime","client_ip","User_Name","server_principal_name","statement","host_name","server_instance_name","succeeded","database_name","application_name","TIMESTAMP","CLIENT_HOST_NAME","STATEMENT","Client_IP","SERVER_INSTANCE_NAME","SUCCEEDED","User_Name","APPLICATION_NAME","Session_ID","Server_Hostname","enrollmentId"] } } - - } \ No newline at end of file diff --git a/filter-plugin/logstash-filter-azure-sql-guardium/AzureSQLOverJdbcPackage/AzureSQL/logstash-filter-azuresql_guardium_filter.zip b/filter-plugin/logstash-filter-azure-sql-guardium/AzureSQLOverJdbcPackage/AzureSQL/logstash-filter-azuresql_guardium_filter.zip new file mode 100644 index 000000000..b9259b61b Binary files /dev/null and b/filter-plugin/logstash-filter-azure-sql-guardium/AzureSQLOverJdbcPackage/AzureSQL/logstash-filter-azuresql_guardium_filter.zip differ diff --git a/filter-plugin/logstash-filter-azure-sql-guardium/CHANGELOG.md b/filter-plugin/logstash-filter-azure-sql-guardium/CHANGELOG.md index c02f622ad..8b6791cf5 100644 --- a/filter-plugin/logstash-filter-azure-sql-guardium/CHANGELOG.md +++ b/filter-plugin/logstash-filter-azure-sql-guardium/CHANGELOG.md @@ -1,4 +1,8 @@ ## 1.0.0 - Added support for Third Party Tools. ## 0.0.1 - - AzureSQL Plugin created with the logstash plugin generator \ No newline at end of file + - AzureSQL Plugin created with the logstash plugin generator +## 0.0.2 +Ticket:https://ibm-datasecurity.atlassian.net/browse/GRD-114698 +Release: Q12026 +Description: Change server type from "MSSQL" to "MS SQL SERVER" \ No newline at end of file diff --git a/filter-plugin/logstash-filter-azure-sql-guardium/README.md b/filter-plugin/logstash-filter-azure-sql-guardium/README.md index 758b40a9d..b6fb4d6a9 100644 --- a/filter-plugin/logstash-filter-azure-sql-guardium/README.md +++ b/filter-plugin/logstash-filter-azure-sql-guardium/README.md @@ -51,6 +51,7 @@ The plug-in is free and open-source (Apache 2.0). It can be used as a starting p g. Then, click on ```Save``` Button. +**Note:** Since auditing configuration differs between Azure SQL Database and Azure SQL Managed Instance, please follow the given link to [enable auditing for Azure SQL Managed Instance](https://learn.microsoft.com/en-us/azure/azure-sql/managed-instance/auditing-configure?view=azuresql) ## Connecting to AzureSQL Database @@ -106,15 +107,15 @@ The Guardium universal connector is the Guardium entry point for native audit lo • AzureSQL-Guardium Logstash filter plug-in is automatically available with Guardium Data Protection versions 12.x, 11.4 with appliance bundle 11.0p490 or later or Guardium Data Protection version 11.5 with appliance bundle 11.0p540 or later releases. -**Note**: For Guardium Data Protection version 11.4 without appliance bundle 11.0p490 or prior or Guardium Data Protection version 11.5 without appliance bundle 11.0p540 or prior, download the [Azure-SQL-Offline-Package.zip](https://github.com/IBM/universal-connectors/releases/download/v1.5.6/logstash-filter-azuresql_guardium_plugin_filter.zip) plug-in. (Do not unzip the offline-package file throughout the procedure). +**Note**: For Guardium Data Protection version 11.4 without appliance bundle 11.0p490 or prior or Guardium Data Protection version 11.5 without appliance bundle 11.0p540 or prior, download the [Azure-SQL-Offline-Package.zip](https://github.com/IBM/universal-connectors/blob/main/filter-plugin/logstash-filter-azure-sql-guardium/AzureSQLOverJdbcPackage/AzureSQL/logstash-filter-azuresql_guardium_filter.zip) plug-in. (Do not unzip the offline-package file throughout the procedure). -• Download the mssql-jdbc-7.4.1.jre8 from [here](https://jar-download.com/artifacts/com.microsoft.sqlserver/mssql-jdbc/7.4.1.jre8) +• Download the mssql-jdbc-7.4.1.jre8 from [here](https://repo1.maven.org/maven2/com/microsoft/sqlserver/mssql-jdbc/7.4.1.jre8/mssql-jdbc-7.4.1.jre8.jar) #### Configuration 1. On the collector, go to ```Setup``` > ```Tools and Views``` > ```Configure Universal Connector```. 2. Enable the universal connector if it is disabled. -3. Click ```Upload File``` and select the offline [Azure-SQL-Offline-Package.zip](https://github.com/IBM/universal-connectors/releases/download/v1.5.6/logstash-filter-azuresql_guardium_plugin_filter.zip) This step is not necessary for Guardium Data Protection v11.0p490 or later, v11.0p540 or later, v12.0 or later. +3. Click ```Upload File``` and select the offline [Azure-SQL-Offline-Package.zip](https://github.com/IBM/universal-connectors/blob/main/filter-plugin/logstash-filter-azure-sql-guardium/AzureSQLOverJdbcPackage/AzureSQL/logstash-filter-azuresql_guardium_filter.zip) This step is not necessary for Guardium Data Protection v11.0p490 or later, v11.0p540 or later, v12.0 or later. a. Again click ```Upload File``` and select the offline mssql-jdbc-7.4.1.jre8 file. After it is uploaded, click ```OK```. 4. Click the Plus sign to open the Connector Configuration dialog box. 5. Type a name in the ```Connector name``` field. diff --git a/filter-plugin/logstash-filter-azure-sql-guardium/azureSQLJDBC.conf b/filter-plugin/logstash-filter-azure-sql-guardium/azureSQLJDBC.conf index 092ec0b4b..94f082049 100644 --- a/filter-plugin/logstash-filter-azure-sql-guardium/azureSQLJDBC.conf +++ b/filter-plugin/logstash-filter-azure-sql-guardium/azureSQLJDBC.conf @@ -23,8 +23,8 @@ input { add_field => {"enrollmentId" => ""} } } -filter { +filter { if [type] == "azureSQL" { if [additional_information] { @@ -51,7 +51,7 @@ filter { "[APPLICATION_NAME]" => "%{application_name}" "[Session_ID]" => "%{session_id}" "[Server_Hostname]" => "%{enrollmentId}_%{server_instance_name}" - } + } } mutate { gsub => [ "STATEMENT", "[\n]", "" ] } @@ -59,5 +59,4 @@ filter { mutate { remove_field => ["@version","type","@timestamp","additional_information","session_id","updatedeventtime","client_ip","User_Name","server_principal_name","statement","host_name","server_instance_name","succeeded","database_name","application_name","TIMESTAMP","CLIENT_HOST_NAME","STATEMENT","Client_IP","SERVER_INSTANCE_NAME","SUCCEEDED","User_Name","APPLICATION_NAME","Session_ID","Server_Hostname","enrollmentId"] } } - -} +} \ No newline at end of file diff --git a/filter-plugin/logstash-filter-azure-sql-guardium/build.gradle b/filter-plugin/logstash-filter-azure-sql-guardium/build.gradle index 84379a9d3..7eed5259a 100644 --- a/filter-plugin/logstash-filter-azure-sql-guardium/build.gradle +++ b/filter-plugin/logstash-filter-azure-sql-guardium/build.gradle @@ -2,6 +2,30 @@ import java.nio.file.Files import static java.nio.file.StandardCopyOption.REPLACE_EXISTING apply plugin: 'java' + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + + dependencies { + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' + classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.1.0" + classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' + } +} + +def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" // =========================================================================== @@ -21,10 +45,10 @@ pluginInfo.pluginName = "azuresql_guardium_plugin_filter" // must match the // =========================================================================== -sourceCompatibility = 1.8 -targetCompatibility = 1.8 +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 -def jacocoVersion = '0.8.4' +def jacocoVersion = '0.8.11' // minimumCoverage can be set by Travis ENV def minimumCoverageStr = System.getenv("MINIMUM_COVERAGE") ?: "50.0%" if (minimumCoverageStr.endsWith("%")) { @@ -32,27 +56,16 @@ if (minimumCoverageStr.endsWith("%")) { } def minimumCoverage = Float.valueOf(minimumCoverageStr) / 100 -buildscript { - repositories { - maven { - url "https://plugins.gradle.org/m2/" - } - mavenCentral() - jcenter() - } - - dependencies { - classpath 'com.github.jengelman.gradle.plugins:shadow:4.0.4' - classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.0.0-RC2" - classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' - } -} - -def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); -def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) repositories { - mavenCentral() + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } } tasks.register("vendor"){ @@ -63,14 +76,13 @@ tasks.register("vendor"){ File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") projectJarFile.mkdirs() Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) - validatePluginJar(projectJarFile, project.group) } } apply plugin: 'com.github.johnrengelman.shadow' shadowJar { - classifier = null + archiveClassifier = null } dependencies { @@ -78,6 +90,7 @@ dependencies { implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: versions.dependencies.log4jCore implementation 'org.apache.commons:commons-lang3:' + versions.dependencies.commonsLang implementation 'com.google.code.gson:gson:' + versions.dependencies.gson + implementation group: 'commons-beanutils', name: 'commons-beanutils', version: versions.dependencies.commonsBeanutils implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "build/libs/logstash-core-*.*.*.jar") implementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "common-*.*.*.jar") @@ -101,6 +114,17 @@ tasks.withType(JavaCompile) { options.encoding = 'UTF-8' } +test { + jvmArgs = [ + '--add-opens=java.base/java.lang=ALL-UNNAMED', + '--add-opens=java.base/java.util=ALL-UNNAMED', + '--add-opens=java.base/java.text=ALL-UNNAMED', + '--add-opens=java.base/java.lang.reflect=ALL-UNNAMED', + '--add-opens=java.base/sun.util.resources=ALL-UNNAMED', + '--add-opens=java.base/sun.util.cldr=ALL-UNNAMED' + ] +} + tasks.register("generateRubySupportFiles") { @@ -143,17 +167,16 @@ apply plugin: "org.barfuin.gradle.jacocolog" // ------------------------------------ jacoco { toolVersion = "${jacocoVersion}" - reportsDir = file("$buildDir/reports/jacoco") } jacocoTestReport { // You will see "Report -> file://...." at the end of a JaCoCo build // If no output, run this first: ./gradlew test reports { - html.enabled true - xml.enabled true - csv.enabled true - html.destination file("${buildDir}/reports/jacoco") - csv.destination file("${buildDir}/reports/jacoco/all.csv") + html.required = true + xml.required = true + csv.required = true + html.outputLocation = file("${buildDir}/reports/jacoco") + csv.outputLocation = file("${buildDir}/reports/jacoco/all.csv") } executionData.from fileTree(dir: "${buildDir}/jacoco/", includes: [ '**/*.exec' diff --git a/filter-plugin/logstash-filter-azure-sql-guardium/gradle/wrapper/gradle-wrapper.properties b/filter-plugin/logstash-filter-azure-sql-guardium/gradle/wrapper/gradle-wrapper.properties index aa991fcea..81aa1c044 100644 --- a/filter-plugin/logstash-filter-azure-sql-guardium/gradle/wrapper/gradle-wrapper.properties +++ b/filter-plugin/logstash-filter-azure-sql-guardium/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/filter-plugin/logstash-filter-azure-sql-guardium/mainREADME.md b/filter-plugin/logstash-filter-azure-sql-guardium/mainREADME.md new file mode 100644 index 000000000..9d3cc4a59 --- /dev/null +++ b/filter-plugin/logstash-filter-azure-sql-guardium/mainREADME.md @@ -0,0 +1,10 @@ +# Azure SQL Universal Connector + +## Follow this link to set up and use Azure SQL Universal Connector over Azure Eventhub Logstash Plugin + +[SqlOverEventHub](./README.md) + +## Follow this link to set up and use Azure SQL Universal Connector over Azure Eventhub Connect + +[SqlOverConnectEventHub](../../docs/KafkaBasedUCs/AzureMsSQLJDBCKafkaConnect.md) + diff --git a/filter-plugin/logstash-filter-azure-sql-guardium/src/main/java/com/ibm/guardium/azureSQL/Constants.java b/filter-plugin/logstash-filter-azure-sql-guardium/src/main/java/com/ibm/guardium/azureSQL/Constants.java index 762698551..aa32c93e9 100644 --- a/filter-plugin/logstash-filter-azure-sql-guardium/src/main/java/com/ibm/guardium/azureSQL/Constants.java +++ b/filter-plugin/logstash-filter-azure-sql-guardium/src/main/java/com/ibm/guardium/azureSQL/Constants.java @@ -19,11 +19,11 @@ public interface Constants { public static final String DEFAULT_IP = "0.0.0.0"; public static final int DEFAULT_PORT = -1; public static final String UNKNOWN_STRING = ""; - public static final String SERVER_TYPE_STRING = "MSSQL"; - public static final String DATA_PROTOCOL_STRING = "AzureSQL native audit"; + public static final String SERVER_TYPE_STRING = "MS SQL SERVER"; + public static final String DATA_PROTOCOL_STRING = "AzureSQL"; public static final String LANGUAGE="MSSQL"; public static final String SQL_ERROR = "SQL_ERROR"; public static final String ZERO = "0"; public static final String ADDITIONAL_INFORMATION = "ADDITIONAL_INFORMATION"; - + public static final String NOT_AVAILABLE = "N.A."; } \ No newline at end of file diff --git a/filter-plugin/logstash-filter-azure-sql-guardium/src/main/java/com/ibm/guardium/azureSQL/Parser.java b/filter-plugin/logstash-filter-azure-sql-guardium/src/main/java/com/ibm/guardium/azureSQL/Parser.java index 01cbec64c..dd5a4c2a0 100644 --- a/filter-plugin/logstash-filter-azure-sql-guardium/src/main/java/com/ibm/guardium/azureSQL/Parser.java +++ b/filter-plugin/logstash-filter-azure-sql-guardium/src/main/java/com/ibm/guardium/azureSQL/Parser.java @@ -10,96 +10,101 @@ import com.ibm.guardium.universalconnector.commons.structures.Record; import com.ibm.guardium.universalconnector.commons.structures.SessionLocator; import com.ibm.guardium.universalconnector.commons.structures.Time; - +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import co.elastic.logstash.api.Event; public class Parser { - - public static Record parseRecord(final Event e) throws ParseException { - - Record record = new Record(); - - record.setSessionId(e.getField(Constants.Session_ID).toString()); - - String databaseName = Constants.UNKNOWN_STRING; - if(e.getField(Constants.DATABASE_NAME) !=null){ - databaseName=e.getField(Constants.DATABASE_NAME).toString(); - } - record.setDbName(databaseName); - - record.setAppUserName(Constants.APP_USER_NAME); - - record.setTime(Parser.parseTimestamp(e)); - - record.setSessionLocator(Parser.parseSessionLocator(e)); - - record.setAccessor(Parser.parseAccessor(e)); - - if(e.getField(Constants.SUCCEEDED).toString().contains("true")){ - Data data=new Data(); - data.setOriginalSqlCommand(e.getField(Constants.STATEMENT).toString()); - record.setData(data); - }else { - ExceptionRecord exceptionRecord = new ExceptionRecord(); - exceptionRecord.setExceptionTypeId(Constants.SQL_ERROR); - exceptionRecord.setDescription(e.getField(Constants.ADDITIONAL_INFORMATION).toString()); - exceptionRecord.setSqlString(e.getField(Constants.STATEMENT).toString()); - record.setException(exceptionRecord); - } - return record; - } - - public static Time parseTimestamp(final Event e) { - - long date=0; - String dateString = e.getField(Constants.TIMESTAMP).toString(); - date=Long.parseLong(dateString); - long mini=date/1000000; - - return new Time(mini,0,0); - } - - public static SessionLocator parseSessionLocator(final Event e) { - - SessionLocator sessionLocator = new SessionLocator(); - - sessionLocator.setClientIp(e.getField(Constants.Client_IP).toString()); - sessionLocator.setClientPort(Constants.DEFAULT_PORT); - sessionLocator.setServerIp(Constants.DEFAULT_IP); - sessionLocator.setServerPort(Constants.DEFAULT_PORT); - sessionLocator.setIpv6(false); - sessionLocator.setClientIpv6(Constants.UNKNOWN_STRING); - sessionLocator.setServerIpv6(Constants.UNKNOWN_STRING); - - return sessionLocator; - } - - public static Accessor parseAccessor(final Event e) { - - Accessor accessor = new Accessor(); - accessor.setDataType(Accessor.DATA_TYPE_GUARDIUM_SHOULD_PARSE_SQL); - accessor.setLanguage(Constants.LANGUAGE); - accessor.setClientHostName(e.getField(Constants.CLIENT_HOST_NAME).toString()); - accessor.setClientOs(Constants.UNKNOWN_STRING); - accessor.setDbUser(e.getField(Constants.User_Name).toString()); - accessor.setServerType(Constants.SERVER_TYPE_STRING); - accessor.setCommProtocol(Constants.UNKNOWN_STRING); - accessor.setDbProtocol(Constants.DATA_PROTOCOL_STRING); - accessor.setDbProtocolVersion(Constants.UNKNOWN_STRING); - accessor.setSourceProgram(e.getField(Constants.APPLICATION_NAME).toString()); - accessor.setClient_mac(Constants.UNKNOWN_STRING); - accessor.setServerDescription(Constants.UNKNOWN_STRING); - - String databaseName = Constants.UNKNOWN_STRING; - if(e.getField(Constants.DATABASE_NAME) !=null){ - databaseName=e.getField(Constants.DATABASE_NAME).toString(); - } - - accessor.setServiceName(databaseName); - accessor.setServerOs(Constants.UNKNOWN_STRING); - accessor.setServerHostName(e.getField(Constants.Server_Hostname).toString()); - accessor.setOsUser(Constants.UNKNOWN_STRING); - return accessor; - } - + + private static String getFieldString(final Event e, final String key, final String defaulString) { + Object v = e.getField(key); + if (v == null) { + return defaulString; + } + return v.toString(); + } + + public static Record parseRecord(final Event e) throws ParseException { + + Record record = new Record(); + record.setSessionId(getFieldString(e, Constants.Session_ID, Constants.UNKNOWN_STRING)); + + String databaseName = getFieldString(e, Constants.DATABASE_NAME, Constants.NOT_AVAILABLE); + record.setDbName(databaseName); + + record.setAppUserName(Constants.APP_USER_NAME); + + record.setTime(Parser.parseTimestamp(e)); + + record.setSessionLocator(Parser.parseSessionLocator(e)); + + record.setAccessor(Parser.parseAccessor(e)); + + if (getFieldString(e, Constants.SUCCEEDED, Constants.UNKNOWN_STRING).contains("true")) { + Data data = new Data(); + data.setOriginalSqlCommand(getFieldString(e, Constants.STATEMENT, Constants.NOT_AVAILABLE)); + record.setData(data); + } else { + ExceptionRecord exceptionRecord = new ExceptionRecord(); + exceptionRecord.setExceptionTypeId(Constants.SQL_ERROR); + exceptionRecord.setDescription( + getFieldString(e, Constants.ADDITIONAL_INFORMATION, Constants.UNKNOWN_STRING)); + exceptionRecord.setSqlString(getFieldString(e, Constants.STATEMENT, Constants.NOT_AVAILABLE)); + record.setException(exceptionRecord); + } + return record; + } + + public static Time parseTimestamp(final Event e) { + long date = 0; + String dateString = getFieldString(e, Constants.TIMESTAMP, ""); + try { + date = Long.parseLong(dateString); + long mini = date / 1000000; + return new Time(mini, 0, 0); + } catch (NumberFormatException nfe) { + return new Time(0, 0, 0); + } + } + + public static SessionLocator parseSessionLocator(final Event e) { + + SessionLocator sessionLocator = new SessionLocator(); + + sessionLocator.setClientIp(getFieldString(e, Constants.Client_IP, Constants.DEFAULT_IP)); + sessionLocator.setClientPort(Constants.DEFAULT_PORT); + sessionLocator.setServerIp(Constants.DEFAULT_IP); + sessionLocator.setServerPort(Constants.DEFAULT_PORT); + sessionLocator.setIpv6(false); + sessionLocator.setClientIpv6(Constants.UNKNOWN_STRING); + sessionLocator.setServerIpv6(Constants.UNKNOWN_STRING); + + return sessionLocator; + } + + public static Accessor parseAccessor(final Event e) { + + Accessor accessor = new Accessor(); + accessor.setDataType(Accessor.DATA_TYPE_GUARDIUM_SHOULD_PARSE_SQL); + accessor.setLanguage(Constants.LANGUAGE); + accessor.setClientHostName( + getFieldString(e, Constants.CLIENT_HOST_NAME, Constants.UNKNOWN_STRING)); + accessor.setClientOs(Constants.UNKNOWN_STRING); + accessor.setDbUser(getFieldString(e, Constants.User_Name, Constants.NOT_AVAILABLE)); + accessor.setServerType(Constants.SERVER_TYPE_STRING); + accessor.setCommProtocol(Constants.UNKNOWN_STRING); + accessor.setDbProtocol(Constants.DATA_PROTOCOL_STRING); + accessor.setDbProtocolVersion(Constants.UNKNOWN_STRING); + accessor.setSourceProgram(getFieldString(e, Constants.APPLICATION_NAME, Constants.NOT_AVAILABLE)); + accessor.setClient_mac(Constants.UNKNOWN_STRING); + accessor.setServerDescription(Constants.UNKNOWN_STRING); + + String databaseName = getFieldString(e, Constants.DATABASE_NAME, Constants.NOT_AVAILABLE); + accessor.setServiceName(databaseName); + accessor.setServerOs(Constants.UNKNOWN_STRING); + accessor.setServerHostName( + getFieldString(e, Constants.Server_Hostname, Constants.UNKNOWN_STRING)); + accessor.setOsUser(Constants.UNKNOWN_STRING); + return accessor; + } } diff --git a/filter-plugin/logstash-filter-capella-guardium/CHANGELOG.md b/filter-plugin/logstash-filter-capella-guardium/CHANGELOG.md index 143dd1602..80d96241c 100644 --- a/filter-plugin/logstash-filter-capella-guardium/CHANGELOG.md +++ b/filter-plugin/logstash-filter-capella-guardium/CHANGELOG.md @@ -2,7 +2,8 @@ # Changelog Notable changes will be documented in this file. - +## [1.0.1] +- Update parser to fix the mapping issue for SQL report. ## [] diff --git a/filter-plugin/logstash-filter-capella-guardium/capellaCouchbaseOverCapellaPackage/capella/capellaCouchbase.conf b/filter-plugin/logstash-filter-capella-guardium/CapellaCouchbaseOverCapellaPackage/capellaCouchbase.conf similarity index 100% rename from filter-plugin/logstash-filter-capella-guardium/capellaCouchbaseOverCapellaPackage/capella/capellaCouchbase.conf rename to filter-plugin/logstash-filter-capella-guardium/CapellaCouchbaseOverCapellaPackage/capellaCouchbase.conf diff --git a/filter-plugin/logstash-filter-capella-guardium/README.md b/filter-plugin/logstash-filter-capella-guardium/README.md index fff78f058..e0ca461e3 100644 --- a/filter-plugin/logstash-filter-capella-guardium/README.md +++ b/filter-plugin/logstash-filter-capella-guardium/README.md @@ -6,7 +6,9 @@ * Environment: Couchbase Capella Cloud * Supported inputs: Capella Input plugin * Supported Guardium versions: - * Guardium Data Protection: 12.0 and above + * Guardium Data Protection 12.0 patch 5005 and above + * Guardium Data Protection 12.1 patch 5005 and above + * Guardium Data Protection 12.2 and above This is a [Logstash](https://github.com/elastic/logstash) filter plug-in for the universal connector that is featured in IBM Security Guardium. It parses Couchbase Capella event logs into @@ -99,10 +101,6 @@ Note: If successful, the request returns an array of all the audit log export jo For each audit log export job, when the export is ready, the download_id field gives a URL that you can use to download the exported audit log. -## Limitations - -* No more than three historical export requests are permitted in a 24-hour period. - ## Configuring the Capella filter in Guardium The Guardium universal connector is the Guardium entry point for native audit/data_access logs. The Guardium universal @@ -115,10 +113,10 @@ enforcements. Configure Guardium to read the native audit/data_access logs by cu * Configure the policies you require. See [policies](/docs/#policies) for more information. * You must have permission for the S-Tap Management role. The admin user includes this role by default * Verify that the Capella input plugin is available on the GDP system. If the plugin is missing, download and install - the [logstash-input-couchbase_capella_input](../../input-plugin/logstash-input-couchbase-capella/logstash-input-couchbase_capella_input.zip) + the [logstash-input-couchbase_capella_input](https://github.com/IBM/universal-connectors/releases/download/v1.7.0/logstash-input-couchbase_capella_input.zip) plug-in. * Download - the [logstash-filter-capella_guardium_filter](capellaCouchbaseOverCapellaPackage/capella/logstash-filter-capella_guardium_filter.zip) + the [logstash-filter-capella_guardium_filter](https://github.com/IBM/universal-connectors/releases/download/v1.7.0/logstash-filter-capella_guardium_filter.zip) plug-in. * Capella-Guardium Logstash filter plug-in is automatically available with Guardium Data Protection versions 12.x, 11.4 with appliance bundle 11.0p490 or later or Guardium Data Protection version 11.5 with appliance bundle 11.0p540 or @@ -129,15 +127,15 @@ enforcements. Configure Guardium to read the native audit/data_access logs by cu 1. On the collector, go to ```Setup``` > ```Tools and Views``` > ```Configure Universal Connector```. 2. Enable the universal connector if it is disabled. 3. Click ```Upload File``` and select the - offline [logstash-filter-capella_guardium_filter](capellaCouchbaseOverCapellaPackage/capella/logstash-filter-capella_guardium_filter.zip) + offline [logstash-filter-capella_guardium_filter](https://github.com/IBM/universal-connectors/releases/download/v1.7.0/logstash-filter-capella_guardium_filter.zip) plug-in. After it is uploaded, click ```OK```. 4. Click the Plus sign to open the Connector Configuration dialog box. 5. Type a name in the Connector name field. 6. Update the input section to add the details from - the [capellaCouchbase.conf](capellaCouchbaseOverCapellaPackage/capella/capellaCouchbase.conf) file's input part, + the [capellaCouchbase.conf](CapellaCouchbaseOverCapellaPackage/capellaCouchbase.conf) file's input part, omitting the keyword "input{" at the beginning and its corresponding "}" at the end. 7. Update the filter section to add the details from - the [capellaCouchbase.conf](capellaCouchbaseOverCapellaPackage/capella/capellaCouchbase.conf) file's filter part, + the [capellaCouchbase.conf](CapellaCouchbaseOverCapellaPackage/capellaCouchbase.conf) file's filter part, omitting the keyword "filter{" at the beginning and its corresponding "}" at the end. 8. The 'type' fields should match in the input and filter configuration sections. This field should be unique for every individual connector added. @@ -145,3 +143,9 @@ enforcements. Configure Guardium to read the native audit/data_access logs by cu 10. After the offline plug-in is installed and the configuration is uploaded and saved in the Guardium machine, restart the Universal Connector using the ```Disable/Enable``` button. +## Limitations +* No more than three historical export requests are permitted over 24-hour period. +* The original Capella audit log contains no values for the following fields: Database Name, Service Name. + +Notes: +* It may take approximately 30 minutes for data to appear in the Full SQL report. diff --git a/filter-plugin/logstash-filter-capella-guardium/VERSION b/filter-plugin/logstash-filter-capella-guardium/VERSION index 3eefcb9dd..7dea76edb 100644 --- a/filter-plugin/logstash-filter-capella-guardium/VERSION +++ b/filter-plugin/logstash-filter-capella-guardium/VERSION @@ -1 +1 @@ -1.0.0 +1.0.1 diff --git a/filter-plugin/logstash-filter-capella-guardium/build.gradle b/filter-plugin/logstash-filter-capella-guardium/build.gradle index 33feea9a5..a674ab50f 100644 --- a/filter-plugin/logstash-filter-capella-guardium/build.gradle +++ b/filter-plugin/logstash-filter-capella-guardium/build.gradle @@ -2,6 +2,30 @@ import java.nio.file.Files import static java.nio.file.StandardCopyOption.REPLACE_EXISTING apply plugin: 'java' + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + + dependencies { + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' + classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.1.0" + classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' + } +} + +def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + apply plugin: 'jacoco' apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" apply plugin: "com.github.johnrengelman.shadow" @@ -26,38 +50,27 @@ pluginInfo.pluginClass = "CapellaGuardiumFilter" pluginInfo.pluginName = "capella_guardium_filter" // must match the @LogstashPlugin annotation in the main plugin class // =========================================================================== -sourceCompatibility = 1.8 -targetCompatibility = 1.8 +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 -def jacocoVersion = '0.8.4' +def jacocoVersion = '0.8.11' // minimumCoverage can be set by Travis ENV def minimumCoverageStr = System.getenv("MINIMUM_COVERAGE") ?: "50.0%" if (minimumCoverageStr.endsWith("%")) { minimumCoverageStr = minimumCoverageStr.substring(0, minimumCoverageStr.length() - 1) } def minimumCoverage = Float.valueOf(minimumCoverageStr) / 100 -def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); -def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) -buildscript { - repositories { - maven { - url "https://plugins.gradle.org/m2/" +repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" } - mavenCentral() - jcenter() } - - dependencies { - classpath 'com.github.jengelman.gradle.plugins:shadow:4.0.4' - classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.0.0-RC2" - classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' - } -} - -repositories { - mavenCentral() } tasks.register("vendor"){ @@ -68,12 +81,11 @@ tasks.register("vendor"){ File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") projectJarFile.mkdirs() Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) - validatePluginJar(projectJarFile, project.group) } } shadowJar { - classifier = null + archiveClassifier = null } @@ -81,6 +93,7 @@ dependencies { implementation 'com.google.code.gson:gson:' + versions.dependencies.gson implementation 'org.apache.commons:commons-lang3:' + versions.dependencies.commonsLang implementation 'commons-validator:commons-validator:' + versions.dependencies.commonsValidator + implementation group: 'commons-beanutils', name: 'commons-beanutils', version: versions.dependencies.commonsBeanutils implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.17.2' testImplementation group: 'org.mockito', name: 'mockito-all', version: versions.dependencies.mockitoAll @@ -150,18 +163,17 @@ tasks.register("copyDependencyLibs", Copy){ // ------------------------------------ jacoco { toolVersion = "${jacocoVersion}" - reportsDir = file("$buildDir/reports/jacoco") } jacocoTestReport { // You will see "Report -> file://...." at the end of a JaCoCo build // If no output, run this first: ./gradlew test reports { - html.enabled true - xml.enabled true - csv.enabled true - html.destination file("${buildDir}/reports/jacoco") - csv.destination file("${buildDir}/reports/jacoco/all.csv") + html.required = true + xml.required = true + csv.required = true + html.outputLocation = file("${buildDir}/reports/jacoco") + csv.outputLocation = file("${buildDir}/reports/jacoco/all.csv") } executionData.from fileTree(dir: "${buildDir}/jacoco/", includes: [ '**/*.exec' diff --git a/filter-plugin/logstash-filter-capella-guardium/gradle/wrapper/gradle-wrapper.properties b/filter-plugin/logstash-filter-capella-guardium/gradle/wrapper/gradle-wrapper.properties index aa991fcea..81aa1c044 100644 --- a/filter-plugin/logstash-filter-capella-guardium/gradle/wrapper/gradle-wrapper.properties +++ b/filter-plugin/logstash-filter-capella-guardium/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/filter-plugin/logstash-filter-capella-guardium/capellaCouchbaseOverCapellaPackage/capella/logstash-filter-capella_guardium_filter.zip b/filter-plugin/logstash-filter-capella-guardium/logstash-filter-capella_guardium_filter.zip similarity index 51% rename from filter-plugin/logstash-filter-capella-guardium/capellaCouchbaseOverCapellaPackage/capella/logstash-filter-capella_guardium_filter.zip rename to filter-plugin/logstash-filter-capella-guardium/logstash-filter-capella_guardium_filter.zip index 3aefbc587..f40fb8d62 100644 Binary files a/filter-plugin/logstash-filter-capella-guardium/capellaCouchbaseOverCapellaPackage/capella/logstash-filter-capella_guardium_filter.zip and b/filter-plugin/logstash-filter-capella-guardium/logstash-filter-capella_guardium_filter.zip differ diff --git a/filter-plugin/logstash-filter-capella-guardium/src/main/java/com/ibm/guardium/capella/ConfigurationGenerator.java b/filter-plugin/logstash-filter-capella-guardium/src/main/java/com/ibm/guardium/capella/ConfigurationGenerator.java index 13b742bcc..d631df60c 100644 --- a/filter-plugin/logstash-filter-capella-guardium/src/main/java/com/ibm/guardium/capella/ConfigurationGenerator.java +++ b/filter-plugin/logstash-filter-capella-guardium/src/main/java/com/ibm/guardium/capella/ConfigurationGenerator.java @@ -12,7 +12,7 @@ static String getConfig() { + " \"index\": \"/\\\"index_name\\\"\",\n" + " \"service_name\": \"/\\\"name\\\"\",\n" + " \"parsing_type\": \"CUSTOM_PARSER\",\n" - + " \"server_port\": \"/\\\"local\\\"/\\\"port\\\"\",\n" + + " \"server_port\": \"{-1}\",\n" + " \"server_ip\": \"/\\\"local\\\"/\\\"ip\\\"\",\n" + " \"server_ipv6\": \"/\\\"local\\\"/\\\"ip\\\"\",\n" + " \"client_ip\": \"/\\\"remote\\\"/\\\"ip\\\"\",\n" @@ -29,7 +29,7 @@ static String getConfig() { + " \"sql_string\": \"/\\\"description\\\"\",\n" + " \"statement\": \"/\\\"statement\\\"\",\n" + " \"status\": \"/\\\"status\\\"\",\n" - + " \"server_hostname\": \"/\\\"serverHostName\\\"\"\n" + + " \"server_hostname\": \"{cloud.couchbase.com}\"\n" + "}"; } } diff --git a/filter-plugin/logstash-filter-capella-guardium/src/main/java/com/ibm/guardium/capella/Parser.java b/filter-plugin/logstash-filter-capella-guardium/src/main/java/com/ibm/guardium/capella/Parser.java index 5b0b2f9e4..4573a0a64 100644 --- a/filter-plugin/logstash-filter-capella-guardium/src/main/java/com/ibm/guardium/capella/Parser.java +++ b/filter-plugin/logstash-filter-capella-guardium/src/main/java/com/ibm/guardium/capella/Parser.java @@ -7,7 +7,15 @@ import com.ibm.guardium.universalconnector.commons.custom_parsing.CustomParser; import com.ibm.guardium.universalconnector.commons.custom_parsing.ParserFactory; -import com.ibm.guardium.universalconnector.commons.structures.*; +import com.ibm.guardium.universalconnector.commons.structures.Accessor; +import com.ibm.guardium.universalconnector.commons.structures.Construct; +import com.ibm.guardium.universalconnector.commons.structures.Data; +import com.ibm.guardium.universalconnector.commons.structures.ExceptionRecord; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import com.ibm.guardium.universalconnector.commons.structures.Sentence; +import com.ibm.guardium.universalconnector.commons.structures.SentenceObject; +import com.ibm.guardium.universalconnector.commons.structures.SessionLocator; +import com.ibm.guardium.universalconnector.commons.structures.Time; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -93,6 +101,7 @@ protected Data getData(String payload, String sqlString) { construct.sentences.add(sentence); construct.setFullSql(sqlString); + construct.setRedactedSensitiveDataSql(sqlString); data.setConstruct(construct); data.setOriginalSqlCommand(sqlString); } @@ -108,8 +117,7 @@ protected String getDataType(String payload) { @Override protected String getServiceName(String payload) { - String value = getValue(payload, SERVICE_NAME); - return value != null ? value : DEFAULT_STRING; + return getDbName(payload); } protected String getStatus(String payload) { @@ -130,7 +138,7 @@ protected ExceptionRecord getException(String payload, String sqlString) { return exceptionRecord; } else if (statement != null && !status.contains(SUCCESS_STATUS)) { exceptionRecord.setDescription(serviceName); - exceptionRecord.setSqlString(sqlString); + exceptionRecord.setSqlString(statement); exceptionRecord.setExceptionTypeId(SQL_ERROR); return exceptionRecord; } diff --git a/filter-plugin/logstash-filter-capella-guardium/src/test/java/com/ibm/guardium/capella/ParserTest.java b/filter-plugin/logstash-filter-capella-guardium/src/test/java/com/ibm/guardium/capella/ParserTest.java index 2976af75c..0ffd60b70 100644 --- a/filter-plugin/logstash-filter-capella-guardium/src/test/java/com/ibm/guardium/capella/ParserTest.java +++ b/filter-plugin/logstash-filter-capella-guardium/src/test/java/com/ibm/guardium/capella/ParserTest.java @@ -20,10 +20,10 @@ void testAuthenticationSuccessPayload() { assertNotNull(record); assertEquals("N.A.", record.getDbName()); - assertEquals("authentication succeeded", record.getAccessor().getServiceName()); + assertEquals("N.A.", record.getAccessor().getServiceName()); assertEquals(51606, record.getSessionLocator().getClientPort()); assertEquals("10.0.0.23", record.getSessionLocator().getClientIp()); - assertEquals(11207, record.getSessionLocator().getServerPort()); + assertEquals(-1, record.getSessionLocator().getServerPort()); assertEquals("10.0.0.11", record.getSessionLocator().getServerIp()); assertEquals("CAPELLA", record.getAccessor().getDbProtocol()); assertEquals("@index@local", record.getAccessor().getDbUser()); @@ -52,7 +52,7 @@ void testBucketSelectionPayload() { assertEquals("ZeeshanTestBucket", record.getDbName()); assertEquals(44518, record.getSessionLocator().getClientPort()); assertEquals("127.0.0.1", record.getSessionLocator().getClientIp()); - assertEquals(11209, record.getSessionLocator().getServerPort()); + assertEquals(-1, record.getSessionLocator().getServerPort()); assertEquals("127.0.0.1", record.getSessionLocator().getServerIp()); assertEquals("CAPELLA", record.getAccessor().getDbProtocol()); assertEquals("@ns_server@local", record.getAccessor().getDbUser()); @@ -101,7 +101,7 @@ void testSuccessfulLoginPayload() { assertEquals("ba2760cee506d0293a8b4a0bf83687b807329667", record.getSessionId()); assertEquals(53322, record.getSessionLocator().getClientPort()); assertEquals("10.144.210.1", record.getSessionLocator().getClientIp()); - assertEquals(8091, record.getSessionLocator().getServerPort()); + assertEquals(-1, record.getSessionLocator().getServerPort()); assertEquals("10.144.210.101", record.getSessionLocator().getServerIp()); assertEquals("CAPELLA", record.getAccessor().getDbProtocol()); assertEquals("testUser@local", record.getAccessor().getDbUser()); @@ -147,7 +147,7 @@ void testFailedLoginPayload() { assertEquals("", record.getSessionId()); assertEquals(53348, record.getSessionLocator().getClientPort()); assertEquals("10.144.210.1", record.getSessionLocator().getClientIp()); - assertEquals(8091, record.getSessionLocator().getServerPort()); + assertEquals(-1, record.getSessionLocator().getServerPort()); assertEquals("10.144.210.101", record.getSessionLocator().getServerIp()); assertEquals("CAPELLA", record.getAccessor().getDbProtocol()); assertEquals("newUser@rejected", record.getAccessor().getDbUser()); @@ -214,7 +214,7 @@ void testCreateBucketPayload() { assertEquals("3f8472056c30014d32f19aca0bb22b10d5cefbee", record.getSessionId()); assertEquals(53837, record.getSessionLocator().getClientPort()); assertEquals("10.144.231.1", record.getSessionLocator().getClientIp()); - assertEquals(8091, record.getSessionLocator().getServerPort()); + assertEquals(-1, record.getSessionLocator().getServerPort()); assertEquals("10.144.231.102", record.getSessionLocator().getServerIp()); assertEquals("CAPELLA", record.getAccessor().getDbProtocol()); assertEquals("Administrator@builtin", record.getAccessor().getDbUser()); @@ -266,7 +266,7 @@ void testCreateUpdateIndexPayload() { assertEquals("", record.getSessionId()); assertEquals(39575, record.getSessionLocator().getClientPort()); assertEquals("127.0.0.1", record.getSessionLocator().getClientIp()); - assertEquals(8094, record.getSessionLocator().getServerPort()); + assertEquals(-1, record.getSessionLocator().getServerPort()); assertEquals("127.0.0.1", record.getSessionLocator().getServerIp()); assertEquals("CAPELLA", record.getAccessor().getDbProtocol()); assertEquals("Administrator@builtin", record.getAccessor().getDbUser()); @@ -330,7 +330,7 @@ void testBucketTtlModificationPayload() { assertEquals("eb1411eaa5eb041ea07fb86ffe93a94a59f8e8e2", record.getSessionId()); assertEquals(53397, record.getSessionLocator().getClientPort()); assertEquals("10.144.210.1", record.getSessionLocator().getClientIp()); - assertEquals(8091, record.getSessionLocator().getServerPort()); + assertEquals(-1, record.getSessionLocator().getServerPort()); assertEquals("10.144.210.101", record.getSessionLocator().getServerIp()); assertEquals("CAPELLA", record.getAccessor().getDbProtocol()); assertEquals("Administrator@builtin", record.getAccessor().getDbUser()); @@ -392,7 +392,7 @@ void testUserCreationPayload() { assertEquals("eb1411eaa5eb041ea07fb86ffe93a94a59f8e8e2", record.getSessionId()); assertEquals(53444, record.getSessionLocator().getClientPort()); assertEquals("10.144.210.1", record.getSessionLocator().getClientIp()); - assertEquals(8091, record.getSessionLocator().getServerPort()); + assertEquals(-1, record.getSessionLocator().getServerPort()); assertEquals("10.144.210.101", record.getSessionLocator().getServerIp()); assertEquals("CAPELLA", record.getAccessor().getDbProtocol()); assertEquals("Administrator@builtin", record.getAccessor().getDbUser()); @@ -416,7 +416,7 @@ void testDeleteBucket() { assertEquals("deleteMeBucket", record.getDbName()); assertEquals(35264, record.getSessionLocator().getClientPort()); assertEquals("127.0.0.1", record.getSessionLocator().getClientIp()); - assertEquals(11209, record.getSessionLocator().getServerPort()); + assertEquals(-1, record.getSessionLocator().getServerPort()); assertEquals("127.0.0.2", record.getSessionLocator().getServerIp()); assertEquals("CAPELLA", record.getAccessor().getDbProtocol()); assertEquals("@ns_server@local", record.getAccessor().getDbUser()); @@ -469,7 +469,7 @@ void testSqlString() { assertEquals("N.A.", record.getDbName()); assertEquals(56928, record.getSessionLocator().getClientPort()); assertEquals("127.0.0.1", record.getSessionLocator().getClientIp()); - assertEquals(18093, record.getSessionLocator().getServerPort()); + assertEquals(-1, record.getSessionLocator().getServerPort()); assertEquals("127.0.0.1", record.getSessionLocator().getServerIp()); assertEquals("CAPELLA", record.getAccessor().getDbProtocol()); assertEquals("2209d939-681d-44ed-a47c-4757d13344a2@local", record.getAccessor().getDbUser()); @@ -514,7 +514,7 @@ void testServiceName() { + "}"; Record record = parser.parseRecord(payload); assertNotNull(record); - assertEquals("change cluster settings", record.getAccessor().getServiceName()); + assertEquals("N.A.", record.getAccessor().getServiceName()); } @Test @@ -547,7 +547,7 @@ void testDeleteUser() { assertEquals("User was deleted", record.getData().getOriginalSqlCommand()); assertEquals("User was deleted", record.getData().getConstruct().fullSql); - assertEquals("delete user", record.getAccessor().getServiceName()); + assertEquals("N.A.", record.getAccessor().getServiceName()); } @Test @@ -591,7 +591,7 @@ void testINFERStatment() { assertEquals("N.A.", record.getDbName()); assertEquals(54908, record.getSessionLocator().getClientPort()); assertEquals("127.0.0.1", record.getSessionLocator().getClientIp()); - assertEquals(18093, record.getSessionLocator().getServerPort()); + assertEquals(-1, record.getSessionLocator().getServerPort()); assertEquals("127.0.0.1", record.getSessionLocator().getServerIp()); assertEquals("CAPELLA", record.getAccessor().getDbProtocol()); assertEquals("e1028d45-5814-44de-b299-af8302f64629@local", record.getAccessor().getDbUser()); @@ -599,7 +599,7 @@ void testINFERStatment() { assertEquals( "__CB POST /#statement=infer `MtBucket`.`new`.`capella_engine_events`", record.getData().getOriginalSqlCommand()); - assertEquals("INFER statement", record.getAccessor().getServiceName()); + assertEquals("N.A.", record.getAccessor().getServiceName()); assertEquals("TEXT", record.getAccessor().getDataType()); assertEquals("COUCHB", record.getAccessor().getLanguage()); } @@ -646,7 +646,7 @@ void testDeleteStatement() { assertEquals("N.A.", record.getDbName()); assertEquals(53221, record.getSessionLocator().getClientPort()); assertEquals("99.233.169.176", record.getSessionLocator().getClientIp()); - assertEquals(18093, record.getSessionLocator().getServerPort()); + assertEquals(-1, record.getSessionLocator().getServerPort()); assertEquals("10.0.0.123", record.getSessionLocator().getServerIp()); assertEquals("CAPELLA", record.getAccessor().getDbProtocol()); assertEquals("admin@local", record.getAccessor().getDbUser()); @@ -654,7 +654,7 @@ void testDeleteStatement() { assertEquals( "__CB POST /#statement=DELETE FROM `MtBucket`.`CreatedByMe`.`TestScope` WHERE id = \"user4\";", record.getData().getOriginalSqlCommand()); - assertEquals("DELETE statement", record.getAccessor().getServiceName()); + assertEquals("N.A.", record.getAccessor().getServiceName()); } @Test @@ -698,7 +698,7 @@ void testComplexSQL() { assertEquals("N.A.", record.getDbName()); assertEquals(43060, record.getSessionLocator().getClientPort()); assertEquals("127.0.0.1", record.getSessionLocator().getClientIp()); - assertEquals(18093, record.getSessionLocator().getServerPort()); + assertEquals(-1, record.getSessionLocator().getServerPort()); assertEquals("127.0.0.1", record.getSessionLocator().getServerIp()); assertEquals("CAPELLA", record.getAccessor().getDbProtocol()); assertEquals("2209d939-681d-44ed-a47c-4757d13344a2@local", record.getAccessor().getDbUser()); @@ -719,10 +719,10 @@ void testException() { "{\"description\":\"REST operation failed due to authentication failure\",\"id\":20485,\"local\":{\"ip\":\"10.0.0.11\",\"port\":11207},\"name\":\"authentication failure\",\"raw_url\": \"/pools\",\"real_userid\":{\"domain\":\"local\",\"user\":\"@index\"},\"remote\":{\"ip\":\"10.0.0.23\",\"port\":51606},\"timestamp\":\"2025-01-30T21:12:48.099382Z\"}"; Record record = parser.parseRecord(payload); assertEquals("N.A.", record.getDbName()); - assertEquals("authentication failure", record.getAccessor().getServiceName()); + assertEquals("N.A.", record.getAccessor().getServiceName()); assertEquals(51606, record.getSessionLocator().getClientPort()); assertEquals("10.0.0.23", record.getSessionLocator().getClientIp()); - assertEquals(11207, record.getSessionLocator().getServerPort()); + assertEquals(-1, record.getSessionLocator().getServerPort()); assertEquals("10.0.0.11", record.getSessionLocator().getServerIp()); assertEquals("CAPELLA", record.getAccessor().getDbProtocol()); assertEquals("@index@local", record.getAccessor().getDbUser()); @@ -740,10 +740,10 @@ void testSQLError() { "{\"description\":\"An unrecognized statement was received by the N1QL query engine\",\"errors\":null,\"id\":28687,\"isAdHoc\":true,\"local\":{\"ip\":\"10.0.0.102\",\"port\":18093},\"metrics\":{\"elapsedTime\":\"0.00037393\",\"errorCount\":1,\"executionTime\":\"0.000311174\",\"resultCount\":0,\"resultSize\":0},\"name\":\"UNRECOGNIZED statement\",\"node\":\"svc-dqi-node-001.vam24ep86horsjja.cloud.couchbase.com:8091\",\"real_userid\":{\"domain\":\"local\",\"user\":\"@index\"},\"remote\":{\"ip\":\"99.233.169.176\",\"port\":58674},\"requestId\":\"d0707282-7b1a-453e-82df-76ea3456e6ca\",\"statement\":\"select * fro test;\",\"status\":\"fatal\",\"timestamp\":\"2025-06-25T15:58:08.215Z\",\"userAgent\":\"Go-http-client/1.1\"}"; Record record = parser.parseRecord(payload); assertEquals("N.A.", record.getDbName()); - assertEquals("UNRECOGNIZED statement", record.getAccessor().getServiceName()); + assertEquals("N.A.", record.getAccessor().getServiceName()); assertEquals(58674, record.getSessionLocator().getClientPort()); assertEquals("99.233.169.176", record.getSessionLocator().getClientIp()); - assertEquals(18093, record.getSessionLocator().getServerPort()); + assertEquals(-1, record.getSessionLocator().getServerPort()); assertEquals("10.0.0.102", record.getSessionLocator().getServerIp()); assertEquals("CAPELLA", record.getAccessor().getDbProtocol()); assertEquals("@index@local", record.getAccessor().getDbUser()); @@ -753,5 +753,6 @@ void testSQLError() { assertEquals("COUCHB", record.getAccessor().getLanguage()); assertEquals("UNRECOGNIZED statement", record.getException().getDescription()); assertEquals("SQL_ERROR", record.getException().getExceptionTypeId()); + assertEquals("select * fro test;", record.getException().getSqlString()); } } diff --git a/filter-plugin/logstash-filter-cassandra-guardium/CHANGELOG.md b/filter-plugin/logstash-filter-cassandra-guardium/CHANGELOG.md index bb49aab48..06fa60de3 100644 --- a/filter-plugin/logstash-filter-cassandra-guardium/CHANGELOG.md +++ b/filter-plugin/logstash-filter-cassandra-guardium/CHANGELOG.md @@ -6,4 +6,7 @@ Notable changes will be documented in this file. ## [1.0.0] - 2021-05-17 ### Added -- Initial release, in parallel to Guardium v11.4. \ No newline at end of file +- Initial release, in parallel to Guardium v11.4. + +## [1.0.1] - 2026-02-25 +- Update the parser for Exception Record to fix ArrayIndexOutOfBoundsException \ No newline at end of file diff --git a/filter-plugin/logstash-filter-cassandra-guardium/CassandraOverFilebeatPackage/Cassandra/logstash-filter-cassandra_guardium_plugin_filter.zip b/filter-plugin/logstash-filter-cassandra-guardium/CassandraOverFilebeatPackage/Cassandra/logstash-filter-cassandra_guardium_plugin_filter.zip new file mode 100644 index 000000000..b5eff7bf8 Binary files /dev/null and b/filter-plugin/logstash-filter-cassandra-guardium/CassandraOverFilebeatPackage/Cassandra/logstash-filter-cassandra_guardium_plugin_filter.zip differ diff --git a/filter-plugin/logstash-filter-cassandra-guardium/README.md b/filter-plugin/logstash-filter-cassandra-guardium/README.md index 027b10a9b..6bd0fd0ba 100644 --- a/filter-plugin/logstash-filter-cassandra-guardium/README.md +++ b/filter-plugin/logstash-filter-cassandra-guardium/README.md @@ -1,6 +1,8 @@ # Cassandra-Guardium Logstash filter plug-in ### Meet Cassandra -* Tested versions: 4.0.1 +* Tested versions: + * Apache Cassandra: 4.0.1 + * DataStax Enterprise Server: 6.8.x * Environment: On-premise * Supported inputs: Filebeat (push) * Supported Guardium versions: @@ -75,7 +77,8 @@ https://www.elastic.co/guide/en/beats/filebeat/current/directory-layout.html • Locate "filebeat.inputs" in the filebeat.yml file, and then add the following parameters. filebeat.inputs: - - type: log + - type: filestream + - id: - enabled: true paths: - diff --git a/filter-plugin/logstash-filter-cassandra-guardium/VERSION b/filter-plugin/logstash-filter-cassandra-guardium/VERSION index 3eefcb9dd..7dea76edb 100644 --- a/filter-plugin/logstash-filter-cassandra-guardium/VERSION +++ b/filter-plugin/logstash-filter-cassandra-guardium/VERSION @@ -1 +1 @@ -1.0.0 +1.0.1 diff --git a/filter-plugin/logstash-filter-cassandra-guardium/build.gradle b/filter-plugin/logstash-filter-cassandra-guardium/build.gradle index 33d40d9d8..b4118fbea 100644 --- a/filter-plugin/logstash-filter-cassandra-guardium/build.gradle +++ b/filter-plugin/logstash-filter-cassandra-guardium/build.gradle @@ -2,6 +2,30 @@ import java.nio.file.Files import static java.nio.file.StandardCopyOption.REPLACE_EXISTING apply plugin: 'java' + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + + dependencies { + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' + classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.1.0" + classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' + } +} + +def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" // =========================================================================== @@ -21,10 +45,10 @@ pluginInfo.pluginName = "cassandra_guardium_plugin_filter" // must match th // =========================================================================== -sourceCompatibility = 1.8 -targetCompatibility = 1.8 +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 -def jacocoVersion = '0.8.4' +def jacocoVersion = '0.8.11' // minimumCoverage can be set by Travis ENV def minimumCoverageStr = System.getenv("MINIMUM_COVERAGE") ?: "50.0%" if (minimumCoverageStr.endsWith("%")) { @@ -32,27 +56,16 @@ if (minimumCoverageStr.endsWith("%")) { } def minimumCoverage = Float.valueOf(minimumCoverageStr) / 100 -buildscript { - repositories { - maven { - url "https://plugins.gradle.org/m2/" - } - mavenCentral() - jcenter() - } - - dependencies { - classpath 'com.github.jengelman.gradle.plugins:shadow:4.0.4' - classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.0.0-RC2" - classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' - } -} - -def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); -def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) repositories { - mavenCentral() + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } } tasks.register("vendor"){ @@ -63,20 +76,20 @@ tasks.register("vendor"){ File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") projectJarFile.mkdirs() Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) - validatePluginJar(projectJarFile, project.group) } } apply plugin: 'com.github.johnrengelman.shadow' shadowJar { - classifier = null + archiveClassifier = null } dependencies { implementation group: 'commons-validator', name: 'commons-validator', version: versions.dependencies.commonsValidator implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: versions.dependencies.log4jCore implementation 'org.apache.commons:commons-lang3:' + versions.dependencies.commonsLang + implementation group: 'commons-beanutils', name: 'commons-beanutils', version: versions.dependencies.commonsBeanutils implementation 'com.google.code.gson:gson:' + versions.dependencies.gson implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "build/libs/logstash-core-*.*.*.jar") implementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "common-*.*.*.jar") @@ -101,6 +114,17 @@ tasks.withType(JavaCompile) { options.encoding = 'UTF-8' } +test { + jvmArgs = [ + '--add-opens=java.base/java.lang=ALL-UNNAMED', + '--add-opens=java.base/java.util=ALL-UNNAMED', + '--add-opens=java.base/java.text=ALL-UNNAMED', + '--add-opens=java.base/java.lang.reflect=ALL-UNNAMED', + '--add-opens=java.base/sun.util.resources=ALL-UNNAMED', + '--add-opens=java.base/sun.util.cldr=ALL-UNNAMED' + ] +} + tasks.register("generateRubySupportFiles") { @@ -143,17 +167,16 @@ apply plugin: "org.barfuin.gradle.jacocolog" // ------------------------------------ jacoco { toolVersion = "${jacocoVersion}" - reportsDir = file("$buildDir/reports/jacoco") } jacocoTestReport { // You will see "Report -> file://...." at the end of a JaCoCo build // If no output, run this first: ./gradlew test reports { - html.enabled true - xml.enabled true - csv.enabled true - html.destination file("${buildDir}/reports/jacoco") - csv.destination file("${buildDir}/reports/jacoco/all.csv") + html.required = true + xml.required = true + csv.required = true + html.outputLocation = file("${buildDir}/reports/jacoco") + csv.outputLocation = file("${buildDir}/reports/jacoco/all.csv") } executionData.from fileTree(dir: "${buildDir}/jacoco/", includes: [ '**/*.exec' diff --git a/filter-plugin/logstash-filter-cassandra-guardium/gradle/wrapper/gradle-wrapper.properties b/filter-plugin/logstash-filter-cassandra-guardium/gradle/wrapper/gradle-wrapper.properties index aa991fcea..81aa1c044 100644 --- a/filter-plugin/logstash-filter-cassandra-guardium/gradle/wrapper/gradle-wrapper.properties +++ b/filter-plugin/logstash-filter-cassandra-guardium/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/filter-plugin/logstash-filter-cassandra-guardium/src/main/java/com/ibm/guardium/cassandra/CassandraGuardiumPluginFilter.java b/filter-plugin/logstash-filter-cassandra-guardium/src/main/java/com/ibm/guardium/cassandra/CassandraGuardiumPluginFilter.java index 4122ea1a2..e78a9511a 100644 --- a/filter-plugin/logstash-filter-cassandra-guardium/src/main/java/com/ibm/guardium/cassandra/CassandraGuardiumPluginFilter.java +++ b/filter-plugin/logstash-filter-cassandra-guardium/src/main/java/com/ibm/guardium/cassandra/CassandraGuardiumPluginFilter.java @@ -45,12 +45,14 @@ public class CassandraGuardiumPluginFilter implements Filter { } private String id; + private Parser parser; public static final PluginConfigSpec SOURCE_CONFIG = PluginConfigSpec.stringSetting("source", "message"); private static Logger log = LogManager.getLogger(CassandraGuardiumPluginFilter.class); public CassandraGuardiumPluginFilter(String id, Configuration config, Context context) { // constructors should validate configuration options this.id = id; + this.parser = new Parser(); } @SuppressWarnings("unchecked") @@ -68,11 +70,11 @@ public Collection filter(Collection events, FilterMatchListener ma String[] secondary_input = intermediate_input[1].split(Constants.INPUT_SPLIT2); for (String input_value : secondary_input) { - + String[] keyValue = input_value.split(Constants.INPUT_SPLIT3,Constants.limit); dataMap.put(keyValue[0], keyValue[1]); } - Record record = Parser.parseRecord(dataMap); + Record record = parser.parseRecord(dataMap); final GsonBuilder builder = new GsonBuilder(); builder.serializeNulls(); final Gson gson = builder.create(); diff --git a/filter-plugin/logstash-filter-cassandra-guardium/src/main/java/com/ibm/guardium/cassandra/Parser.java b/filter-plugin/logstash-filter-cassandra-guardium/src/main/java/com/ibm/guardium/cassandra/Parser.java index 37fed7917..f45fc8feb 100644 --- a/filter-plugin/logstash-filter-cassandra-guardium/src/main/java/com/ibm/guardium/cassandra/Parser.java +++ b/filter-plugin/logstash-filter-cassandra-guardium/src/main/java/com/ibm/guardium/cassandra/Parser.java @@ -15,9 +15,8 @@ import com.ibm.guardium.universalconnector.commons.structures.Time; public class Parser { - static ExceptionRecord exceptionRecord; - public static Record parseRecord(final Map data) throws ParseException { + public Record parseRecord(final Map data) throws ParseException { Record record = new Record(); @@ -30,9 +29,10 @@ public static Record parseRecord(final Map data) throws ParseExc record.setTime(new Time(Long.parseLong(data.get(Constants.TIMESTAMP)), 0, 0)); - record.setSessionLocator(Parser.parseSessionLocator(data)); + record.setSessionLocator(parseSessionLocator(data)); - record.setAccessor(Parser.parseAccessor(data)); + record.setAccessor(parseAccessor(data)); + record.getAccessor().setServiceName(record.getDbName()); setExceptionOrDataPart(record, data); @@ -40,7 +40,7 @@ public static Record parseRecord(final Map data) throws ParseExc } - public static void setDbName(Record record, Map data) { + public void setDbName(Record record, Map data) { if (data.containsKey(Constants.KEYSPACE)) { record.setDbName(data.get(Constants.KEYSPACE)); } else { @@ -49,7 +49,7 @@ public static void setDbName(Record record, Map data) { } // Form Session Locator - public static SessionLocator parseSessionLocator(Map data) { + public SessionLocator parseSessionLocator(Map data) { SessionLocator sessionLocator = new SessionLocator(); int clientPort = Constants.CLIENT_PORT_VALUE; @@ -68,7 +68,6 @@ public static SessionLocator parseSessionLocator(Map data) { if (validator.isValidInet4Address(clientIp)) { sessionLocator.setIpv6(false); clientIpAdd = clientIp; - clientPort = Integer.parseInt(data.get(Constants.CLIENT_PORT)); } else if (validator.isValidInet6Address(clientIp)) { sessionLocator.setIpv6(true); clientIpv6Add = clientIp; @@ -97,7 +96,7 @@ public static SessionLocator parseSessionLocator(Map data) { return sessionLocator; } - public static Accessor parseAccessor(Map data) { + public Accessor parseAccessor(Map data) { Accessor accessor = new Accessor(); accessor.setDataType(Accessor.DATA_TYPE_GUARDIUM_SHOULD_PARSE_SQL); accessor.setDbUser(data.get(Constants.USER)); @@ -113,44 +112,52 @@ public static Accessor parseAccessor(Map data) { accessor.setSourceProgram(Constants.UNKNOWN_STRING); accessor.setClient_mac(Constants.UNKNOWN_STRING); accessor.setServerDescription(Constants.UNKNOWN_STRING); - accessor.setServiceName(Constants.UNKNOWN_STRING); + accessor.setServiceName( + data.containsKey(Constants.KEYSPACE) ? data.get(Constants.KEYSPACE) : Constants.UNKNOWN_STRING); accessor.setLanguage(Constants.CASS_LANGUAGE); return accessor; } - public static void setExceptionOrDataPart(final Record record, Map data) { + public void setExceptionOrDataPart(final Record record, Map data) { String operation = data.get(Constants.OPERATION); if (data.get(Constants.CATEGORY).equals(Constants.AUTH)) { if (data.get(Constants.TYPE).equals(Constants.LOGIN_SUCCESS)) { setData(data, record); } else { - exceptionRecord = new ExceptionRecord(); + ExceptionRecord exceptionRecord = new ExceptionRecord(); String[] error = operation.split(Constants.OPERATION_SPLIT1); exceptionRecord.setExceptionTypeId(Constants.LOGIN_FAILED); - setException(data, record, error); + setException(record, error, exceptionRecord); } } else if (data.get(Constants.CATEGORY).equals(Constants.ERROR)) { - exceptionRecord = new ExceptionRecord(); + ExceptionRecord exceptionRecord = new ExceptionRecord(); String[] error = new String[2]; if(operation.contains(Constants.OPERATION_SPLIT2)) error = operation.split(Constants.OPERATION_SPLIT2); else error = operation.split(Constants.OPERATION_SPLIT1); exceptionRecord.setExceptionTypeId(Constants.SQL_ERROR); - setException(data, record, error); + setException(record, error, exceptionRecord); } else { setData(data, record); } } - static void setException(Map data, Record record, String[] error) { - - exceptionRecord.setDescription(error[1]); - exceptionRecord.setSqlString(error[0]); + void setException(Record record, String[] error, ExceptionRecord exceptionRecord) { + if (error.length >= 2) { + exceptionRecord.setDescription(error[1]); + exceptionRecord.setSqlString(error[0]); + } else if (error.length == 1) { + exceptionRecord.setDescription(Constants.UNKNOWN_STRING); + exceptionRecord.setSqlString(error[0]); + } else { + exceptionRecord.setDescription(Constants.UNKNOWN_STRING); + exceptionRecord.setSqlString(Constants.UNKNOWN_STRING); + } record.setException(exceptionRecord); } - static void setData(Map data, Record record) { + void setData(Map data, Record record) { Data outputData = new Data(); outputData.setOriginalSqlCommand(data.get(Constants.OPERATION)); record.setData(outputData); diff --git a/filter-plugin/logstash-filter-cassandra-guardium/src/test/java/com/ibm/guardium/cassandra/ParserTest.java b/filter-plugin/logstash-filter-cassandra-guardium/src/test/java/com/ibm/guardium/cassandra/ParserTest.java index d6caef44f..8a8130bd2 100644 --- a/filter-plugin/logstash-filter-cassandra-guardium/src/test/java/com/ibm/guardium/cassandra/ParserTest.java +++ b/filter-plugin/logstash-filter-cassandra-guardium/src/test/java/com/ibm/guardium/cassandra/ParserTest.java @@ -36,11 +36,13 @@ Map intializeMap() { @Test public void testParseRecord() throws ParseException { - final Record record = Parser.parseRecord(intializeMap()); + final Record record = parser.parseRecord(intializeMap()); Assert.assertEquals(Constants.TEXT, record.getAccessor().getDataType()); Assert.assertEquals(null, record.getException()); Assert.assertNotNull(record.getData()); + Assert.assertEquals(record.getDbName(),record.getAccessor().getServiceName()); + Assert.assertEquals(Constants.UNKNOWN_STRING, record.getSessionId()); } @SuppressWarnings("unchecked") @@ -54,7 +56,7 @@ public void testParseRecordError() throws ParseException { "Select * from employee;; No keyspace has been specified. USE a keyspace, or explicitly specify keyspace.tablename"); intrimData.put("category", "ERROR"); - final Record record = Parser.parseRecord(intrimData); + final Record record = parser.parseRecord(intrimData); Assert.assertEquals(Constants.TEXT, record.getAccessor().getDataType()); Assert.assertEquals("SQL_ERROR", record.getException().getExceptionTypeId()); @@ -73,7 +75,7 @@ public void testParseRecordAuthSuccess() throws ParseException { intrimData.put("operation", "LOGIN SUCCESSFUL"); intrimData.put("category", "AUTH"); - final Record record = Parser.parseRecord(intrimData); + final Record record = parser.parseRecord(intrimData); Assert.assertEquals(Constants.TEXT, record.getAccessor().getDataType()); Assert.assertNull(record.getException()); @@ -93,7 +95,7 @@ public void testParseRecordAuthFail() throws ParseException { "CREATE USER test WITH PASSWORD *******; User test does not have sufficient privileges to perform the requested operation"); intrimData.put("category", "AUTH"); - final Record record = Parser.parseRecord(intrimData); + final Record record = parser.parseRecord(intrimData); Assert.assertEquals(Constants.TEXT, record.getAccessor().getDataType()); Assert.assertNotNull(record.getException()); diff --git a/filter-plugin/logstash-filter-cockroachdb-guardium/.gitignore b/filter-plugin/logstash-filter-cockroachdb-guardium/.gitignore new file mode 100644 index 000000000..a87f7edb9 --- /dev/null +++ b/filter-plugin/logstash-filter-cockroachdb-guardium/.gitignore @@ -0,0 +1,24 @@ +build +bin +*.idea +*.iml +lib/ +vendor/ +.bundle/ +build/ +out/ +.idea +.gradle +.vscode +.classpath +.project +*.code-workspace +*.DS_Store +*.iml +*.class +*.ipr +*.iws +*.gemspec +Gemfile* +settings.gradle +gradle.properties \ No newline at end of file diff --git a/filter-plugin/logstash-filter-cockroachdb-guardium/CHANGELOG.md b/filter-plugin/logstash-filter-cockroachdb-guardium/CHANGELOG.md new file mode 100644 index 000000000..7593cc3d2 --- /dev/null +++ b/filter-plugin/logstash-filter-cockroachdb-guardium/CHANGELOG.md @@ -0,0 +1,7 @@ +# Changelog +Notable changes will be documented in this file. + +## [1.0.0] + +### Added +- Initial release of the Plugin to collect CockroachDB Audit logs. \ No newline at end of file diff --git a/filter-plugin/logstash-filter-cockroachdb-guardium/CockroachDBOverSyslogPackage/CockroachDBOverSyslog.conf b/filter-plugin/logstash-filter-cockroachdb-guardium/CockroachDBOverSyslogPackage/CockroachDBOverSyslog.conf new file mode 100644 index 000000000..e619939e8 --- /dev/null +++ b/filter-plugin/logstash-filter-cockroachdb-guardium/CockroachDBOverSyslogPackage/CockroachDBOverSyslog.conf @@ -0,0 +1,119 @@ +input { + tcp { + port => + type => "syslog-cockroachdb" + ssl_enabled => true + # ssl_certificate_authorities => ["${SSL_DIR}/ca.pem"] + ssl_certificate => "${SSL_DIR}/tls.crt" + ssl_key => "${SSL_DIR}/tls.key" + ssl_verify => true + } +} + +filter { + if [type] == "syslog-cockroachdb" { + + # Drop system-generated queries FIRST (before any parsing) - optimized for performance + # Check for internal execution markers and system users + if [message] =~ /"intExec":/ or + [message] =~ /"User":"node"/ or + [message] =~ /"User":"root"/ or + [message] =~ /"ApplicationName":"\$ internal/ or + [message] =~ /"ExecMode":"exec-internal"/ or + [message] =~ /system\.(jobs|lease|sql_instances|job_info|statement_statistics|transaction_statistics|job_progress_history|reports_meta|statement_diagnostics_requests|scheduled_jobs|settings|zones|tenants|migrations|eventlog|privileges|protected_ts_meta)/ { + drop { } + } + + # Drop CockroachDB configuration/informational messages that don't have JSON payloads + if [message] =~ /\[config\]/ and [message] !~ /\{.*\}$/ { + drop { } + } + + # Extract JSON from CockroachDB structured logging format + grok { + match => { + "message" => ".*\s(?\{.*\})$" + } + } + + # Remove grok failure tags if parsing succeeded + if [cockroach_wrapper_json] { + mutate { + remove_tag => ["_grokparsefailure", "_grokparsefailure_sysloginput"] + } + } + + # Parse the outer wrapper JSON (contains channel, tags, and nested event) + if [cockroach_wrapper_json] { + json { + source => "cockroach_wrapper_json" + target => "cockroach_wrapper" + } + + # Extract the nested "event" object which contains the actual audit data + if [cockroach_wrapper][event] { + # Copy the event object to cockroachdb field + mutate { + copy => { "[cockroach_wrapper][event]" => "cockroachdb" } + } + + # Extract client IP and port from tags if available + # Structured logging format has tags like: "tags":{"client":"ip:port"} + if [cockroach_wrapper][tags][client] { + grok { + match => { + "[cockroach_wrapper][tags][client]" => "%{IP:client_ip}:%{NUMBER:client_port}" + } + } + } + + # Add client IP and port to cockroachdb object + if [client_ip] { + mutate { + add_field => { + "[cockroachdb][ClientIP]" => "%{client_ip}" + "[cockroachdb][ClientPort]" => "%{client_port}" + } + } + } + + # Add server host (IP or hostname) from syslog - will be validated in Java code + if [host] { + mutate { + add_field => { "[cockroachdb][ServerHost]" => "%{host}" } + } + } + + # Convert nanosecond timestamp to readable format + if [cockroachdb][Timestamp] { + ruby { + code => " + timestamp_ns = event.get('[cockroachdb][Timestamp]') + timestamp_sec = timestamp_ns / 1_000_000_000.0 + event.set('[cockroachdb][TimestampReadable]', Time.at(timestamp_sec).utc.strftime('%Y-%m-%d %H:%M:%S.%6N UTC')) + " + } + } + + # Apply the Guardium filter (system queries already dropped early) + cockroachdb_guardium_filter {} + } else { + # No event field found - tag as invalid + mutate { + add_tag => ["_no_event_field"] + } + } + } + + # Clean up temporary fields + mutate { + remove_field => [ + "cockroach_wrapper_json", + "cockroach_wrapper", + "client_ip", + "client_port", + "message" + ] + } + } +} \ No newline at end of file diff --git a/filter-plugin/logstash-filter-cockroachdb-guardium/CockroachDBOverSyslogPackage/logstash-filter-singlestoredb_guardium_filter.zip b/filter-plugin/logstash-filter-cockroachdb-guardium/CockroachDBOverSyslogPackage/logstash-filter-singlestoredb_guardium_filter.zip new file mode 100644 index 000000000..981d7a8c2 Binary files /dev/null and b/filter-plugin/logstash-filter-cockroachdb-guardium/CockroachDBOverSyslogPackage/logstash-filter-singlestoredb_guardium_filter.zip differ diff --git a/filter-plugin/logstash-filter-cockroachdb-guardium/LICENSE b/filter-plugin/logstash-filter-cockroachdb-guardium/LICENSE new file mode 100644 index 000000000..a66908146 --- /dev/null +++ b/filter-plugin/logstash-filter-cockroachdb-guardium/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright IBM Corp. 2021, 2024 + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/filter-plugin/logstash-filter-cockroachdb-guardium/README.md b/filter-plugin/logstash-filter-cockroachdb-guardium/README.md new file mode 100644 index 000000000..5ae2ce8de --- /dev/null +++ b/filter-plugin/logstash-filter-cockroachdb-guardium/README.md @@ -0,0 +1,286 @@ +# CockroachDB-Guardium Logstash filter plug-in + +### Meet CockroachDB +* Tested versions: 23.2.x, 25.4.1 +* Environment: On-premise, IaaS +* Supported inputs: Syslog (push) +* Supported Guardium versions: + * Guardium Data Protection: 12.0 and above + +This is a [Logstash](https://github.com/elastic/logstash) filter plug-in for the universal connector that is featured in IBM Security Guardium. It parses events and messages from the CockroachDB audit log into a [Guardium record](https://github.com/IBM/universal-connectors/blob/main/common/src/main/java/com/ibm/guardium/universalconnector/commons/structures/Record.java) instance (which is a standard structure made out of several parts). The information is then sent over to Guardium. Guardium records include the accessor (the person who tried to access the data), the session, data, and exceptions. If there are no errors, the data contains details about the query "construct". The construct details the main action (verb) and collections (objects) involved. + +The plug-in is free and open-source (Apache 2.0). It can be used as a starting point to develop additional filter plug-ins for Guardium universal connector. + +## 1. Enabling the audit logs + +### Procedure + +1. Connect to your CockroachDB cluster using the SQL client. + +2. Enable audit logging by running the following commands: + ```sql + SET CLUSTER SETTING server.auth_log.sql_connections.enabled = true; + SET CLUSTER SETTING server.auth_log.sql_sessions.enabled = true; + SET CLUSTER SETTING sql.log.all_statements.enabled = true; + SET CLUSTER SETTING sql.log.admin_audit.enabled = true; + ``` + +3. Verify the configuration: + ```sql + SHOW CLUSTER SETTING server.auth_log.sql_connections.enabled; + SHOW CLUSTER SETTING server.auth_log.sql_sessions.enabled; + SHOW CLUSTER SETTING sql.log.all_statements.enabled; + SHOW CLUSTER SETTING sql.log.admin_audit.enabled; + ``` + +4. Create a logging configuration file on your CockroachDB server: + ```bash + mkdir -p /path/to/logs/directory/scripts + vi /path/to/logs/directory/scripts/log-config.yaml + ``` + +5. Add the following YAML configuration to enable structured JSON logging for audit events: + + ```bash + vi /path/to/logs/directory/scripts/log-config.yaml + ``` + + ```yaml + file-defaults: + format: json + redact: false + redactable: true + + sinks: + file-groups: + # In v23: cockroach-sensitive-access.log + # In v25: cockroach-sql-audit.log + sensitive-access: + channels: [SENSITIVE_ACCESS] + dir: /path/to/logs/directory/ + + sql-exec: + channels: [SQL_EXEC] + dir: /path/to/logs/directory/ + + sql-schema: + channels: [SQL_SCHEMA] + dir: /path/to/logs/directory/ + + # In v23: cockroach-sessions.log + # In v25: cockroach-sql-auth.log + sessions: + channels: [SESSIONS] + dir: /path/to/logs/directory/ + ``` + +6. Update your CockroachDB systemd service file to use the logging configuration: + ```bash + sudo vi /etc/systemd/system/cockroachdb.service + ``` + +7. Add the `--log-config-file` flag to the ExecStart line and **remove** any `--log-dir` flag if present (these flags are incompatible): + ``` + ExecStart=/usr/local/bin/cockroach start \ + ... + --log-config-file=/path/to/logs/directory/log-config.yaml + ``` + +8. Reload systemd and restart CockroachDB: + ```bash + systemctl daemon-reload + systemctl restart cockroachdb + ``` + +9. Verify the service started successfully: + ```bash + systemctl status cockroachdb + ``` + +10. Verify the structured JSON logs are being created: + ```bash + ls -la /path/to/logs/directory/ + ``` + +### Important Notes: +- The `file-defaults` section must be at the top level of the YAML file, not nested inside sinks +- You cannot use both `--log-dir` and `--log-config-file` flags together +- The log directory is specified in the YAML configuration under each sink's `dir` field +- Use the [CockroachDBOverSyslog.conf](CockroachDBOverSyslogPackage/CockroachDBOverSyslog.conf) configuration file with structured JSON logging + +## 2. Viewing the audit logs + +The audit logs are stored in the directory specified in your YAML configuration (`/path/to/logs/directory/` by default). Each file-group creates a separate log file: + +- **cockroach-sensitive-access.log** (v23) or **cockroach-sql-audit.log** (v25) - Administrative and privileged operations + - EventType: `admin_query` + - Examples: User management, permission changes, privilege grants/revokes + +- **cockroach-sql-exec.log** - Regular SQL query execution + - EventType: `query_execute` + - Examples: SELECT, INSERT, UPDATE, DELETE statements + +- **cockroach-sessions.log** (v23) or **cockroach-sql-auth.log** (v25) - Authentication and session events + - EventTypes: `client_authentication_failed` + - Examples: Login attempts (failure) + +- **cockroach-sql-schema.log** - Schema change operations (DDL statements) + - EventType: DDL operations + - Examples: CREATE TABLE, DROP TABLE, ALTER TABLE, CREATE INDEX, DROP INDEX + +**Note:** All log files use structured JSON format when configured with the YAML file. Each log entry includes a `"channel"` field indicating its source (e.g., `"channel":"SQL_EXEC"`, `"channel":"SQL_SCHEMA"`), and an `"event"` object containing the audit data. For more information about logging channels and files depending on your cockroachDB version, see [CockroachDB Logging Channels](https://www.cockroachlabs.com/docs/v25.4/logging-overview#logging-channels). + +## 3. Configuring Syslog to push logs to Guardium + +### Procedure +To make Logstash able to process the data collected by syslog, configure available +syslog utility. The example is based on rsyslog utility available in many +versions of the Linux distributions. + +#### Rsyslog installation guide: +* [Ubuntu](https://www.rsyslog.com/ubuntu-repository) +* [RHEL](https://www.rsyslog.com/rhelcentos-rpms) + +1. Install Rsyslog on the CockroachDB server if not already installed: + ```bash + # For Ubuntu/Debian + sudo apt-get install rsyslog + + # For RHEL/CentOS + sudo yum install rsyslog + ``` + +2. To check the service is active and running, execute the below command: + ```bash + systemctl status rsyslog + ``` + +3. Generate Certificate Authority (CA): + * **Guardium Data Protection**
+ To obtain the Certificate Authority content on the Collector, run the following API command: + ```text + grdapi generate_ssl_key_universal_connector + ``` + This API command will display the content of the public Certificate Authority. Copy this certificate authority content to your database source and save it as a file named 'ca.pem' . + +4. Create the Rsyslog configuration file `cockroachdb.conf` for CockroachDB in the following directory: + ```bash + vi /etc/rsyslog.d/cockroachdb.conf + ``` + +5. This configuration reads the logs from the CockroachDB log directory path and sends + the syslog messages to the provided host `TARGET_HOST` at the provided port `TARGET_PORT`. + + **Note:** You can set any port number except 5000 when using Guardium Data Protection version 12.0 or 12.1. + + Add the following configuration: + + **For TLS connection:** + ``` + global( + DefaultNetstreamDriverCAFile="/path/to/certs/ca.pem" + ) + + module(load="imfile" PollingInterval="10") + $MaxMessageSize 64k + + ruleset(name="imfile_to_gdp") { + action(type="omfwd" + protocol="tcp" + StreamDriver="gtls" + StreamDriverMode="1" + StreamDriverAuthMode="x509/certvalid" + template="RSYSLOG_SyslogProtocol23Format" + target="" + port="") + } + + # Monitor CockroachDB SQL query execution logs + input(type="imfile" + File="/path/to/logs/directory/cockroach-sql-exec.log" + Tag="cockroachdb-sql-exec" + Ruleset="imfile_to_gdp") + + # Monitor CockroachDB admin/sensitive access logs + # For v23: cockroach-sensitive-access.log + # For v25: cockroach-sql-audit.log + input(type="imfile" + File="/path/to/logs/directory/cockroach-sensitive-access.log" + Tag="cockroachdb-sensitive-access" + Ruleset="imfile_to_gdp") + + # Monitor CockroachDB schema change logs + input(type="imfile" + File="/path/to/logs/directory/cockroach-sql-schema.log" + Tag="cockroachdb-sql-schema" + Ruleset="imfile_to_gdp") + + # Monitor CockroachDB authentication/session logs + # For v23: use cockroach-sessions.log + # For v25: use cockroach-sql-auth.log + input(type="imfile" + File="/path/to/logs/directory/cockroach-sessions.log" + Tag="cockroachdb-sessions" + Ruleset="imfile_to_gdp") + ``` + +5. Restart Rsyslog service: + ```bash + systemctl restart rsyslog + ``` + +6. Verify Rsyslog is running: + ```bash + systemctl status rsyslog + ``` + +## 4. Limitations +- CockroachDB wraps query values in Unicode characters `‹` (U+2039) and `›` (U+203A) in audit logs (e.g., `UPDATE table SET id = ‹2› WHERE id > ‹1›`). The plugin automatically removes these characters to restore the original query format. +- CockroachDB automatically logs `SHOW database` queries (along with query executions) and are sent to Guardium. +- The plugin automatically filters out the following system-generated queries and are not sent to Guardium: + - Internal execution queries (`intExec=`) + - Automatic job queries (`job=AUTO`) + - User: "node" (internal CockroachDB operations) + - Application Name starting with "$ internal" + - ExecMode: "exec-internal" + - Queries against system schemas: + - `crdb_internal.*` + - `pg_catalog.*` + - `information_schema.*` + - System tables: `system.jobs`, `system.lease`, `system.sql_instances`, `system.job_info`, `system.statement_statistics`, `system.transaction_statistics`, `system.job_progress_history`, `system.reports_meta` +- The following fields are not found in CockroachDB audit logs (applies to queries and failed logins): + - Database Name + - Service Name + - Client Host Name + - Server Port and Server IP - it is set to default value `0.0.0.0` + - Source Program (might be missing in some audit logs) +- For failed login attempts, CockroachDB returns different errors depending on the failure type (has duplicate events): + - When the username does not exist: `USER_NOT_FOUND` error + - When the username exists but the password is incorrect: `PRE_HOOK_ERROR` error +- When using `cockroach sql` CLI with password authentication, a `client_authentication_failed` event is logged before successful login. The failed event is reported to Guardium as a LOGIN_FAILED exception. +- The audit logs captures sql errors for syntactically correct queries (appears in both Full SQL report and Exception report) and does not capture syntactically incorrect queries. +- Some operations may appear in multiple log files. For example, DDL operations (CREATE, ALTER, DROP, etc.) may appear in `cockroach-sql-exec.log`, `cockroach-sensitive-access.log`, and/or `cockroach-sql-schema.log`. + +## 5. Configuring the CockroachDB filter in Guardium + +The Guardium universal connector is the Guardium entry point for native audit logs. The universal connector identifies and parses received events, and then converts them to a standard Guardium format. The output of the universal connector is forwarded to the Guardium sniffer on the collector, for policy and auditing enforcements. Configure Guardium to read the native audit logs by customizing the CockroachDB template. + +### Before you begin + +• Configure the policies to match the CockroachDB events. + +• You must have permission for the S-Tap Management role. The admin user includes this role, by default. + +• Download the [logstash-filter-cockroachdb_guardium_filter.zip](https://github.com/IBM/universal-connectors/releases/download/v1.7.2/logstash-filter-cockroachdb_guardium_filter.zip) plug-in. + +### Procedure + +1. On the collector, go to **Setup** > **Tools and Views** > **Configure Universal Connector**. +2. First enable the Universal Guardium connector, if it is disabled. +3. Click **Upload File** and select the offline [logstash-filter-cockroachdb_guardium_filter.zip](https://github.com/IBM/universal-connectors/releases/download/v1.7.2/logstash-filter-cockroachdb_guardium_filter.zip) plug-in. After it is uploaded, click **OK**. +4. Click the **Plus sign** to open the Connector Configuration dialog box. +5. Type a name in the **Connector name** field. +6. Update the input section to add the details from the [CockroachDBOverSyslog.conf](CockroachDBOverSyslogPackage/CockroachDBOverSyslog.conf) file input section, omitting the keyword "input{" at the beginning and its corresponding "}" at the end. +7. Update the filter section to add the details from the [CockroachDBOverSyslog.conf](CockroachDBOverSyslogPackage/CockroachDBOverSyslog.conf) file filter section, omitting the keyword "filter{" at the beginning and its corresponding "}" at the end. +8. The "type" fields should match in the input and the filter configuration section. This field should be unique for every individual connector added. +9. Click **Save**. Guardium validates the new connector and displays it in the Configure Universal Connector page. \ No newline at end of file diff --git a/filter-plugin/logstash-filter-cockroachdb-guardium/VERSION b/filter-plugin/logstash-filter-cockroachdb-guardium/VERSION new file mode 100644 index 000000000..afaf360d3 --- /dev/null +++ b/filter-plugin/logstash-filter-cockroachdb-guardium/VERSION @@ -0,0 +1 @@ +1.0.0 \ No newline at end of file diff --git a/filter-plugin/logstash-filter-cockroachdb-guardium/build.gradle b/filter-plugin/logstash-filter-cockroachdb-guardium/build.gradle new file mode 100644 index 000000000..b28338440 --- /dev/null +++ b/filter-plugin/logstash-filter-cockroachdb-guardium/build.gradle @@ -0,0 +1,215 @@ +import java.nio.file.Files +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING + +apply plugin: 'java' +def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + + +apply plugin: 'jacoco' + + +apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" +apply plugin: "com.github.johnrengelman.shadow" +apply plugin: "eclipse" + + +// =========================================================================== +// plugin info +// =========================================================================== +group "com.ibm.guardium.cockroachdb" // must match the package of the main plugin class +version "${file("VERSION").text.trim()}" // read from required VERSION file +description = "CockroachDB-Guardium filter plugin" +pluginInfo.licenses = ['Apache-2.0'] // list of SPDX license IDs +pluginInfo.longDescription = "This gem is a Logstash CockroachDB filter plugin required to be installed as part of IBM Security Guardium, Guardium Universal connector configuration. This gem is not a stand-alone program." +pluginInfo.authors = ['IBM', '', ''] +pluginInfo.email = [''] +pluginInfo.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html" +pluginInfo.pluginType = "filter" +pluginInfo.pluginClass = "CockroachdbGuardiumFilter" +pluginInfo.pluginName = "cockroachdb_guardium_filter" // must match the @LogstashPlugin annotation in the main plugin class +// =========================================================================== + +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 + +def jacocoVersion = '0.8.9' +// minimumCoverage can be set by Travis ENV +def minimumCoverageStr = System.getenv("MINIMUM_COVERAGE") ?: "50.0%" +if (minimumCoverageStr.endsWith("%")) { + minimumCoverageStr = minimumCoverageStr.substring(0, minimumCoverageStr.length() - 1) +} +def minimumCoverage = Float.valueOf(minimumCoverageStr) / 100 + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + + dependencies { + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' + classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:4.0.1" + classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' + } +} + + + +repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } +} + +tasks.register("vendor") { + dependsOn shadowJar + doLast { + String vendorPathPrefix = "vendor/jar-dependencies" + String projectGroupPath = project.group.replaceAll('\\.', '/') + File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") + projectJarFile.mkdirs() + Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) + } +} + +apply plugin: 'com.github.johnrengelman.shadow' + +shadowJar { + archiveClassifier = null +} + + +dependencies { + implementation 'com.google.code.gson:gson:' + versions.dependencies.gson + implementation 'org.apache.commons:commons-lang3:' + versions.dependencies.commonsLang + implementation 'commons-validator:commons-validator:' + versions.dependencies.commonsValidator + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.17.2' + implementation group: 'com.github.jsqlparser', name: 'jsqlparser', version: '5.2' + implementation group: 'commons-beanutils', name: 'commons-beanutils', version: versions.dependencies.commonsBeanutils + + testImplementation group: 'org.mockito', name: 'mockito-all', version: versions.dependencies.mockitoAll + testImplementation 'org.junit.jupiter:junit-jupiter:' + versions.dependencies.junitJupiter + testImplementation 'org.jruby:jruby-complete:' + versions.dependencies.jrubyComplete + implementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "common-*.*.*.jar") + + implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "build/libs/logstash-core-*.*.*.jar") + implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: versions.dependencies.log4jCore + implementation group: 'org.json', name: 'json', version: versions.dependencies.json + implementation group: 'org.parboiled', name: 'parboiled-java', version: versions.dependencies.parboiledJava + implementation group: 'org.glassfish', name: 'javax.json', version: versions.dependencies.javaxJson + testImplementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "common-*.*.*.jar") +} + +clean { + delete "${projectDir}/Gemfile" + delete "${projectDir}/" + pluginInfo.pluginFullName() + ".gemspec" + delete "${projectDir}/lib/" + delete "${projectDir}/vendor/" + new FileNameFinder().getFileNames(projectDir.toString(), pluginInfo.pluginFullName() + "-*.*.*.gem").each { filename -> + delete filename + } +} + +tasks.withType(JavaCompile) { + options.encoding = 'UTF-8' +} +test { + useJUnitPlatform() +} +tasks.register("generateRubySupportFiles") { + doLast { + generateRubySupportFilesForPlugin(project.description, project.group, version) + } +} + +tasks.register("removeObsoleteJars") { + doLast { + new FileNameFinder().getFileNames( + projectDir.toString(), + "vendor/**/" + pluginInfo.pluginFullName() + "*.jar", + "vendor/**/" + pluginInfo.pluginFullName() + "-" + version + ".jar").each { f -> + delete f + } + } +} + +tasks.register("gem") { + dependsOn = [downloadAndInstallJRuby, removeObsoleteJars, vendor, generateRubySupportFiles] + doLast { + buildGem(projectDir, buildDir, pluginInfo.pluginFullName() + ".gemspec") + } +} + +tasks.register("copyDependencyLibs", Copy) { + into "dependenciesLib" + from configurations.compileClasspath + from configurations.runtimeClasspath + from configurations.testCompileClasspath + from configurations.testRuntimeClasspath +} + +apply plugin: 'jacoco' +//apply plugin: 'org.barfuin.gradle.jacocolog' version '2.0.0' +apply plugin: "org.barfuin.gradle.jacocolog" +// ------------------------------------ +// JaCoCo is a code coverage tool +// ------------------------------------ +jacoco { + toolVersion = "${jacocoVersion}" +} +jacocoTestReport { + // You will see "Report -> file://...." at the end of a JaCoCo build + // If no output, run this first: ./gradlew test + reports { + html.required = true + xml.required = true + csv.required = true + html.destination file("${buildDir}/reports/jacoco") + csv.destination file("${buildDir}/reports/jacoco/all.csv") + } + executionData.from fileTree(dir: "${buildDir}/jacoco/", includes: [ + '**/*.exec' + ]) + afterEvaluate { + // objective is to test TicketingService class + classDirectories.setFrom(files(classDirectories.files.collect { + fileTree(dir: it, exclude: []) + })) + } + doLast { + println "Report -> file://${buildDir}/reports/jacoco/index.html" + } +} +test.finalizedBy jacocoTestReport +jacocoTestCoverageVerification { + violationRules { + rule { + limit { + minimum = minimumCoverage + } + } + } + executionData.from fileTree(dir: "${buildDir}/jacoco/", includes: [ + '**/*.exec' + ]) + afterEvaluate { + // objective is to test TicketingService class + classDirectories.setFrom(files(classDirectories.files.collect { + fileTree(dir: it, exclude: []) + })) + } +} +project.tasks.check.dependsOn(jacocoTestCoverageVerification, jacocoTestReport) diff --git a/filter-plugin/logstash-filter-cockroachdb-guardium/gradle/wrapper/gradle-wrapper.jar b/filter-plugin/logstash-filter-cockroachdb-guardium/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 000000000..7454180f2 Binary files /dev/null and b/filter-plugin/logstash-filter-cockroachdb-guardium/gradle/wrapper/gradle-wrapper.jar differ diff --git a/filter-plugin/logstash-filter-cockroachdb-guardium/gradle/wrapper/gradle-wrapper.properties b/filter-plugin/logstash-filter-cockroachdb-guardium/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 000000000..81aa1c044 --- /dev/null +++ b/filter-plugin/logstash-filter-cockroachdb-guardium/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,5 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/filter-plugin/logstash-filter-cockroachdb-guardium/gradlew b/filter-plugin/logstash-filter-cockroachdb-guardium/gradlew new file mode 100755 index 000000000..744e882ed --- /dev/null +++ b/filter-plugin/logstash-filter-cockroachdb-guardium/gradlew @@ -0,0 +1,185 @@ +#!/usr/bin/env sh + +# +# Copyright 2015 the original author or authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MSYS* | MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin or MSYS, switch paths to Windows format before running java +if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=`expr $i + 1` + done + case $i in + 0) set -- ;; + 1) set -- "$args0" ;; + 2) set -- "$args0" "$args1" ;; + 3) set -- "$args0" "$args1" "$args2" ;; + 4) set -- "$args0" "$args1" "$args2" "$args3" ;; + 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=`save "$@"` + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +exec "$JAVACMD" "$@" diff --git a/filter-plugin/logstash-filter-cockroachdb-guardium/gradlew.bat b/filter-plugin/logstash-filter-cockroachdb-guardium/gradlew.bat new file mode 100644 index 000000000..107acd32c --- /dev/null +++ b/filter-plugin/logstash-filter-cockroachdb-guardium/gradlew.bat @@ -0,0 +1,89 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto execute + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/filter-plugin/logstash-filter-cockroachdb-guardium/src/main/java/com/ibm/guardium/cockroachdb/CockroachdbGuardiumFilter.java b/filter-plugin/logstash-filter-cockroachdb-guardium/src/main/java/com/ibm/guardium/cockroachdb/CockroachdbGuardiumFilter.java new file mode 100644 index 000000000..26e0689b0 --- /dev/null +++ b/filter-plugin/logstash-filter-cockroachdb-guardium/src/main/java/com/ibm/guardium/cockroachdb/CockroachdbGuardiumFilter.java @@ -0,0 +1,136 @@ +/* +Copyright IBM Corp. 2021, 2024 All rights reserved. +SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.guardium.cockroachdb; + +import co.elastic.logstash.api.*; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonObject; +import com.ibm.guardium.universalconnector.commons.GuardConstants; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.core.LoggerContext; + +import java.io.File; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; + +import static com.ibm.guardium.cockroachdb.Constants.*; + +/** + * CockroachDB Guardium Filter Plugin + *

+ * This filter processes CockroachDB audit logs received via syslog and converts them + * into Guardium universal connector format. + */ +@LogstashPlugin(name = "cockroachdb_guardium_filter") +public class CockroachdbGuardiumFilter implements Filter { + + public static final String LOG42_CONF = "log4j2uc.properties"; + + static { + try { + String uc_etc = System.getenv("UC_ETC"); + LoggerContext context = (LoggerContext) LogManager.getContext(false); + File file = new File(uc_etc + File.separator + LOG42_CONF); + context.setConfigLocation(file.toURI()); + } catch (Exception e) { + System.err.println("Failed to load log4j configuration " + e.getMessage()); + e.printStackTrace(); + } + } + + private static Logger log = LogManager.getLogger(CockroachdbGuardiumFilter.class); + + private String id; + private Parser parser; + + public static final PluginConfigSpec SOURCE_CONFIG = + PluginConfigSpec.stringSetting("source", COCKROACHDB); + + public CockroachdbGuardiumFilter(String id, Configuration config, Context context) { + this.id = id; + this.parser = new Parser(); + } + + @Override + public Collection> configSchema() { + return Collections.singletonList(SOURCE_CONFIG); + } + + /** + * Returns the id + * + * @return id + */ + @Override + public String getId() { + return this.id; + } + + /** + * Filters the received events by skipping the invalid ones and normalizing them by parsing the + * provided payloads into Guardium Generic Records. + * + * @param events A list of received events + * @param filterMatchListener The listener for this plugin + * @return A list of normalized events + */ + @Override + public Collection filter(Collection events, FilterMatchListener filterMatchListener) { + ArrayList skippedEvents = new ArrayList<>(); + + for (Event e : events) { + if (log.isDebugEnabled()) { + log.debug("Event Now: {}", e.getData()); + } + try { + // Check if the event has the cockroachdb field + Object cockroachdbField = e.getField(COCKROACHDB); + + if (cockroachdbField == null) { + log.error("Event does not contain cockroachdb field, tagging as invalid"); + e.tag(INVALID_MSG); + skippedEvents.add(e); + continue; + } + + // Convert the cockroachdb field to JsonObject + JsonObject inputJSON = new Gson().toJsonTree(cockroachdbField).getAsJsonObject(); + + // Add server host from Event's host field (IP or hostname from TCP input) + if (e.getField("host") != null) { + String hostValue = e.getField("host").toString(); + if (hostValue != null && !hostValue.isEmpty()) { + inputJSON.addProperty("ServerHost", hostValue); + } + } + + // Parse the record (filtering is handled by Logstash Grok filter) + Record record = this.parser.parseRecord(inputJSON); + + // Convert to JSON and add to event + final GsonBuilder builder = new GsonBuilder(); + builder.serializeNulls(); + final Gson gson = builder.create(); + e.setField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME, gson.toJson(record)); + + filterMatchListener.filterMatched(e); + log.debug("==========>Final JSON to be send to Guardium: {}", gson.toJson(record)); + + } catch (Exception exception) { + log.error("CockroachDB Filter: Error parsing CockroachDB event: " + exception.getMessage(), exception); + e.tag(INVALID_MSG); + skippedEvents.add(e); + } + } + + events.removeAll(skippedEvents); + return events; + } +} \ No newline at end of file diff --git a/filter-plugin/logstash-filter-cockroachdb-guardium/src/main/java/com/ibm/guardium/cockroachdb/Constants.java b/filter-plugin/logstash-filter-cockroachdb-guardium/src/main/java/com/ibm/guardium/cockroachdb/Constants.java new file mode 100644 index 000000000..b226428d6 --- /dev/null +++ b/filter-plugin/logstash-filter-cockroachdb-guardium/src/main/java/com/ibm/guardium/cockroachdb/Constants.java @@ -0,0 +1,40 @@ +/* +Copyright IBM Corp. 2021, 2024 All rights reserved. +SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.guardium.cockroachdb; + +public class Constants { + static final String COCKROACHDB = "cockroachdb"; + static final String INVALID_MSG = "EVENT_IS_INVALID"; + static final String SERVER_TYPE = "COCKROACHDB"; + static final String DB_PROTOCOL = "COCKROACHDB"; + static final String UNKNOWN_STRING = ""; + static final String NOT_AVAILABLE = "N.A."; + static final String DEFAULT_IP = "0.0.0.0"; + static final String DEFAULT_IPV6 = "0000:0000:0000:0000:0000:FFFF:0000:0000"; + static final int DEFAULT_PORT = -1; + static final String LANGUAGE_COCKROACHDB = "COCKROACH"; + + // Event types + static final String CLIENT_AUTH_FAILED = "client_authentication_failed"; + static final String LOGIN_FAILED = "LoginFailed"; + static final String ERROR_TEXT = "ErrorText"; + static final String SQLSTATE = "SQLSTATE"; + static final String EXCEPTION_TYPE_SQL_ERROR_STRING = "SQL_ERROR"; + static final String EXCEPTION_TYPE_LOGIN_FAILED_STRING = "LOGIN_FAILED"; + + // CockroachDB JSON field names + static final String TIMESTAMP = "Timestamp"; + static final String USER = "User"; + static final String STATEMENT = "Statement"; + static final String APPLICATION_NAME = "ApplicationName"; + static final String DATABASE_NAME = "DatabaseName"; + static final String CLIENT_IP = "ClientIP"; + static final String CLIENT_PORT = "ClientPort"; + static final String SERVER_HOST = "ServerHost"; + static final String EVENT_TYPE = "EventType"; + static final String REASON = "Reason"; + static final String TABLE_NAME = "TableName"; +} diff --git a/filter-plugin/logstash-filter-cockroachdb-guardium/src/main/java/com/ibm/guardium/cockroachdb/Parser.java b/filter-plugin/logstash-filter-cockroachdb-guardium/src/main/java/com/ibm/guardium/cockroachdb/Parser.java new file mode 100644 index 000000000..fb0cd7d8f --- /dev/null +++ b/filter-plugin/logstash-filter-cockroachdb-guardium/src/main/java/com/ibm/guardium/cockroachdb/Parser.java @@ -0,0 +1,305 @@ +/* +Copyright IBM Corp. 2021, 2024 All rights reserved. +SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.guardium.cockroachdb; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.ibm.guardium.universalconnector.commons.structures.Accessor; +import com.ibm.guardium.universalconnector.commons.structures.Construct; +import com.ibm.guardium.universalconnector.commons.structures.Data; +import com.ibm.guardium.universalconnector.commons.structures.ExceptionRecord; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import com.ibm.guardium.universalconnector.commons.structures.Sentence; +import com.ibm.guardium.universalconnector.commons.structures.SentenceObject; +import com.ibm.guardium.universalconnector.commons.structures.SessionLocator; +import com.ibm.guardium.universalconnector.commons.structures.Time; +import org.apache.commons.validator.routines.InetAddressValidator; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; + +import static com.ibm.guardium.cockroachdb.Constants.*; +import static com.ibm.guardium.universalconnector.commons.structures.Accessor.LANGUAGE_FREE_TEXT_STRING; + +/** + * Parser Class will perform operation on parsing events and messages from the CockroachDB audit logs into + * a Guardium record instance. Guardium records include the accessor, the sessionLocator, data, and + * exceptions. If there are no errors, the data contains details about the query "construct" + * + * @className Parser + */ +public class Parser { + + private static Logger logger = LogManager.getLogger(Parser.class); + private static final InetAddressValidator inetAddressValidator = InetAddressValidator.getInstance(); + + public Parser() { + } + + /** + * Parse the CockroachDB event into a Guardium Record + * + * @param data JsonObject containing the CockroachDB audit log data + * @return Record object containing parsed data + */ + public Record parseRecord(JsonObject data) { + logger.debug("=== Starting parseRecord ==="); + logger.debug("Input data: {}", data.toString()); + + Record record = new Record(); + + record.setSessionId(UNKNOWN_STRING); + String dbName = data.has(DATABASE_NAME) && !data.get(DATABASE_NAME).isJsonNull() + ? data.get(DATABASE_NAME).getAsString() + : extractDatabaseNameFromTableName(data); + record.setDbName(dbName); + + // Extract and clean username + String rawUser = data.has(USER) && !data.get(USER).isJsonNull() + ? data.get(USER).getAsString() + : NOT_AVAILABLE; + record.setAppUserName(cleanExtraChars(rawUser)); + + String sqlString = data.has(STATEMENT) && !data.get(STATEMENT).isJsonNull() + ? data.get(STATEMENT).getAsString() + : UNKNOWN_STRING; + + record.setAccessor(getAccessor(data, dbName)); + record.setSessionLocator(getSessionLocator(data)); + record.setTime(getTimestamp(data)); + + // Check for authentication failures by EventType + if (data.has(EVENT_TYPE) && !data.get(EVENT_TYPE).isJsonNull() + && CLIENT_AUTH_FAILED.equals(data.get(EVENT_TYPE).getAsString())) { + // Failed authentication - create Exception record only + record.setData(null); + record.setException(getLoginFailedException(data)); + } + // Check for SQL errors + else if (data.has(ERROR_TEXT) && !data.get(ERROR_TEXT).isJsonNull()) { + // SQL ERROR - Query with error, create Exception record only + record.setData(null); + record.setException(getException(data, sqlString)); + } // Regular query execution + else { + record.setData(getData(sqlString)); + } + + return record; + } + + /** + * Get Accessor object with database user and service information + */ + protected Accessor getAccessor(JsonObject data, String dbName) { + Accessor accessor = new Accessor(); + + accessor.setServiceName(dbName); + + // Extract and clean username + String rawUser = data.has(USER) && !data.get(USER).isJsonNull() + ? data.get(USER).getAsString() + : NOT_AVAILABLE; + accessor.setDbUser(cleanExtraChars(rawUser)); + accessor.setDbProtocolVersion(UNKNOWN_STRING); + + // Set DB protocol and server type + accessor.setDbProtocol(DB_PROTOCOL); + accessor.setServerType(SERVER_TYPE); + + accessor.setServerOs(UNKNOWN_STRING); + accessor.setServerDescription(UNKNOWN_STRING); + + // Set server hostname - only if ServerHost is NOT an IP address + // If it's an IP, set to N.A. since we don't have the actual hostname + String serverHostName = NOT_AVAILABLE; + if (data.has(SERVER_HOST) && !data.get(SERVER_HOST).isJsonNull()) { + String serverHost = data.get(SERVER_HOST).getAsString(); + // Only set as hostname if it's NOT a valid IP address + if (!inetAddressValidator.isValid(serverHost)) { + serverHostName = serverHost; + } + // else: it's an IP, keep serverHostName as NOT_AVAILABLE + } + + accessor.setServerHostName(serverHostName); + accessor.setClientHostName(UNKNOWN_STRING); + accessor.setClient_mac(UNKNOWN_STRING); + accessor.setClientOs(UNKNOWN_STRING); + accessor.setCommProtocol(UNKNOWN_STRING); + accessor.setOsUser(UNKNOWN_STRING); + accessor.setSourceProgram(data.has(APPLICATION_NAME) && !data.get(APPLICATION_NAME).isJsonNull() + ? data.get(APPLICATION_NAME).getAsString() + : UNKNOWN_STRING); + accessor.setLanguage(LANGUAGE_COCKROACHDB); + accessor.setDataType(Accessor.DATA_TYPE_GUARDIUM_SHOULD_PARSE_SQL); + + return accessor; + } + + /** + * Get SessionLocator with client and server IP/port information + */ + protected SessionLocator getSessionLocator(JsonObject data) { + SessionLocator sessionLocator = new SessionLocator(); + + sessionLocator.setIpv6(false); + sessionLocator.setClientIp(data.has(CLIENT_IP) && !data.get(CLIENT_IP).isJsonNull() + ? data.get(CLIENT_IP).getAsString() + : DEFAULT_IP); + sessionLocator.setClientPort(data.has(CLIENT_PORT) && !data.get(CLIENT_PORT).isJsonNull() + ? Integer.parseInt(data.get(CLIENT_PORT).getAsString()) + : DEFAULT_PORT); + + // Set server IP only if ServerHost field contains a valid IP address + String serverIp = DEFAULT_IP; + if (data.has(SERVER_HOST) && !data.get(SERVER_HOST).isJsonNull()) { + String serverHost = data.get(SERVER_HOST).getAsString(); + // Only set as IP if it's a valid IP address + if (inetAddressValidator.isValid(serverHost)) { + serverIp = serverHost; + } + } + sessionLocator.setServerIp(serverIp); + + sessionLocator.setServerPort(DEFAULT_PORT); + sessionLocator.setClientIpv6(DEFAULT_IPV6); + sessionLocator.setServerIpv6(DEFAULT_IPV6); + + return sessionLocator; + } + + /** + * Get timestamp from the CockroachDB event + */ + protected Time getTimestamp(JsonObject data) { + if (data.has(TIMESTAMP) && !data.get(TIMESTAMP).isJsonNull()) { + try { + long timestampNs = data.get(TIMESTAMP).getAsLong(); + long timestampMs = timestampNs / 1_000_000; + return new Time(timestampMs, 0, 0); + } catch (Exception e) { + logger.error("Error parsing timestamp: {}", e.getMessage()); + } + } + // Fallback to current time if timestamp is missing or invalid + return new Time(System.currentTimeMillis(), 0, 0); + } + + /** + * Get Data object containing SQL statement + */ + protected Data getData(String sqlString) { + Data data = new Data(); + data.setOriginalSqlCommand(cleanExtraChars(sqlString)); + return data; + } + + /** + * Get ExceptionRecord for error events + */ + protected ExceptionRecord getException(JsonObject data, String sqlString) { + ExceptionRecord exception = new ExceptionRecord(); + + String sqlState = data.has(SQLSTATE) && !data.get(SQLSTATE).isJsonNull() + ? data.get(SQLSTATE).getAsString() + : UNKNOWN_STRING; + + String errorText = data.has(ERROR_TEXT) && !data.get(ERROR_TEXT).isJsonNull() + ? data.get(ERROR_TEXT).getAsString() + : UNKNOWN_STRING; + + // Build description with SQLSTATE if available + String description = errorText; + if (sqlState != null && !sqlState.isEmpty()) { + description = String.format("[SQLSTATE: %s] %s", sqlState, errorText); + } + + exception.setExceptionTypeId(EXCEPTION_TYPE_SQL_ERROR_STRING); + exception.setDescription(cleanExtraChars(description)); + exception.setSqlString(UNKNOWN_STRING.equals(sqlString) ? NOT_AVAILABLE : cleanExtraChars(sqlString)); + + return exception; + } + + /** + * Get ExceptionRecord for failed login attempts + */ + protected ExceptionRecord getLoginFailedException(JsonObject data) { + ExceptionRecord exception = new ExceptionRecord(); + + // Extract and clean username + String rawUser = data.has(USER) && !data.get(USER).isJsonNull() + ? data.get(USER).getAsString() + : NOT_AVAILABLE; + String user = cleanExtraChars(rawUser); + + String reason = data.has(REASON) && !data.get(REASON).isJsonNull() + ? data.get(REASON).getAsString() + : UNKNOWN_STRING; + + String description = String.format("Login failed for user '%s': %s", user, reason); + + exception.setExceptionTypeId(EXCEPTION_TYPE_LOGIN_FAILED_STRING); + exception.setDescription(cleanExtraChars(description)); + exception.setSqlString(UNKNOWN_STRING); + + return exception; + } + + /** + * Extract database name from TableName field. + * For DDL operations (create_table, alter_table, drop_table, create_index, etc.), + * the database name is embedded in the TableName field in format: database.schema.table + * CockroachDB uses format: database.schema.table or database.table + * + * @param data The JsonObject containing the event data + * @return Database name if found, otherwise NOT_AVAILABLE + */ + private String extractDatabaseNameFromTableName(JsonObject data) { + // Check if TableName exists at the top level + if (data.has(TABLE_NAME) && !data.get(TABLE_NAME).isJsonNull()) { + String tableName = data.get(TABLE_NAME).getAsString(); + + if (tableName == null || tableName.isEmpty()) { + return NOT_AVAILABLE; + } + + // Clean any special characters first + tableName = cleanExtraChars(tableName); + + // Split by dot and take the first part as database name + String[] parts = tableName.split("\\."); + if (parts.length > 0 && !parts[0].isEmpty()) { + return parts[0]; + } + } + + return NOT_AVAILABLE; + } + + /** + * Remove CockroachDB's special Unicode quote characters (‹ and ›) + * that appear as pairs wrapping values. This handles extra characters + * like ‹value› by removing only the quote characters while preserving + * the content between them. + * + * @param value The string to clean + * @return Cleaned string without special quotes, or original value if null/empty + */ + private String cleanExtraChars(String value) { + if (value == null || value.isEmpty()) { + return value; + } + // Remove paired Unicode quotes (‹content›) by replacing them with just the content + // This regex matches the opening quote, captures any content, and the closing quote + return value.replaceAll("‹([^›]*)›", "$1"); + } +} \ No newline at end of file diff --git a/filter-plugin/logstash-filter-cockroachdb-guardium/src/test/java/com/ibm/guardium/cockroachdb/ParserTest.java b/filter-plugin/logstash-filter-cockroachdb-guardium/src/test/java/com/ibm/guardium/cockroachdb/ParserTest.java new file mode 100644 index 000000000..8e88c3157 --- /dev/null +++ b/filter-plugin/logstash-filter-cockroachdb-guardium/src/test/java/com/ibm/guardium/cockroachdb/ParserTest.java @@ -0,0 +1,675 @@ +/* +Copyright IBM Corp. 2021, 2024 All rights reserved. +SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.guardium.cockroachdb; + +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import com.ibm.guardium.universalconnector.commons.structures.Accessor; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import com.ibm.guardium.universalconnector.commons.structures.SessionLocator; +import org.junit.jupiter.api.Test; + +import static com.ibm.guardium.cockroachdb.Constants.*; +import static org.junit.jupiter.api.Assertions.*; + +class ParserTest { + + private final Parser parser = new Parser(); + + @Test + void testParseRegularQuery() { + String payload = "{" + + "\"Timestamp\":1768332558964138753," + + "\"EventType\":\"query_execute\"," + + "\"Statement\":\"SELECT * FROM customers WHERE city = 'New York'\"," + + "\"Tag\":\"SELECT\"," + + "\"User\":\"guardium_qa\"," + + "\"ApplicationName\":\"$ cockroach sql\"," + + "\"DatabaseName\":\"testdb\"," + + "\"ClientIP\":\"127.0.0.1\"," + + "\"ClientPort\":\"49696\"," + + "\"ServerHostname\":\"cockroach-node1\"" + + "}"; + + final JsonObject data = new Gson().fromJson(payload, JsonObject.class); + Record record = parser.parseRecord(data); + + assertNotNull(record); + assertEquals("guardium_qa", record.getAppUserName()); + assertEquals("testdb", record.getDbName()); + assertEquals("127.0.0.1", record.getSessionLocator().getClientIp()); + assertEquals(49696, record.getSessionLocator().getClientPort()); + assertEquals(DB_PROTOCOL, record.getAccessor().getDbProtocol()); + assertEquals("guardium_qa", record.getAccessor().getDbUser()); + assertEquals(SERVER_TYPE, record.getAccessor().getServerType()); + assertEquals("SELECT * FROM customers WHERE city = 'New York'", + record.getData().getOriginalSqlCommand()); + assertEquals(1768332558964L, record.getTime().getTimstamp()); + assertEquals("testdb", record.getAccessor().getServiceName()); + assertEquals(record.getDbName(), record.getAccessor().getServiceName()); + assertNull(record.getException()); + } + + @Test + void testParseSQLError() { + String payload = "{" + + "\"Timestamp\":1768332558964138753," + + "\"EventType\":\"query_execute\"," + + "\"Statement\":\"CREATE TABLE customers (id INT8 PRIMARY KEY, name STRING)\"," + + "\"Tag\":\"CREATE TABLE\"," + + "\"User\":\"guardium_qa\"," + + "\"ApplicationName\":\"$ cockroach sql\"," + + "\"SQLSTATE\":\"42P07\"," + + "\"ErrorText\":\"relation \\\"customers\\\" already exists\"," + + "\"ClientIP\":\"127.0.0.1\"," + + "\"ClientPort\":\"49696\"" + + "}"; + + final JsonObject data = new Gson().fromJson(payload, JsonObject.class); + Record record = parser.parseRecord(data); + + assertNotNull(record); + assertEquals("guardium_qa", record.getAppUserName()); + assertNull(record.getData()); + assertNotNull(record.getException()); + assertEquals(EXCEPTION_TYPE_SQL_ERROR_STRING, record.getException().getExceptionTypeId()); + assertTrue(record.getException().getDescription().contains("already exists")); + assertTrue(record.getException().getDescription().contains("[SQLSTATE: 42P07]")); + assertEquals("CREATE TABLE customers (id INT8 PRIMARY KEY, name STRING)", + record.getException().getSqlString()); + } + + @Test + void testParseAuthenticationFailure() { + String payload = "{" + + "\"Timestamp\":1768332558964138753," + + "\"EventType\":\"client_authentication_failed\"," + + "\"User\":\"invalid_user\"," + + "\"Reason\":\"USER_NOT_FOUND\"," + + "\"ClientIP\":\"192.168.1.100\"," + + "\"ClientPort\":\"54321\"" + + "}"; + + final JsonObject data = new Gson().fromJson(payload, JsonObject.class); + Record record = parser.parseRecord(data); + + assertNotNull(record); + assertEquals("invalid_user", record.getAppUserName()); + assertNull(record.getData()); + assertNotNull(record.getException()); + assertEquals(EXCEPTION_TYPE_LOGIN_FAILED_STRING, record.getException().getExceptionTypeId()); + assertTrue(record.getException().getDescription().contains("invalid_user")); + assertTrue(record.getException().getDescription().contains("USER_NOT_FOUND")); + } + + @Test + void testCleanSpecialCharacters() { + String payload = "{" + + "\"Timestamp\":1768332558964138753," + + "\"EventType\":\"query_execute\"," + + "\"Statement\":\"SELECT 1\"," + + "\"User\":\"‹guardium_qa›\"," + + "\"ApplicationName\":\"$ cockroach sql\"," + + "\"ClientIP\":\"127.0.0.1\"," + + "\"ClientPort\":\"49696\"" + + "}"; + + final JsonObject data = new Gson().fromJson(payload, JsonObject.class); + Record record = parser.parseRecord(data); + + assertNotNull(record); + assertEquals("guardium_qa", record.getAppUserName()); + assertFalse(record.getAppUserName().contains("‹")); + assertFalse(record.getAppUserName().contains("›")); + assertEquals("guardium_qa", record.getAccessor().getDbUser()); + } + + @Test + void testSessionLocatorEmpty() { + String payload = "{" + + "\"Timestamp\":1768332558964138753," + + "\"EventType\":\"query_execute\"," + + "\"Statement\":\"SELECT 1\"," + + "\"User\":\"test_user\"" + + "}"; + + final JsonObject data = new Gson().fromJson(payload, JsonObject.class); + SessionLocator sessionLocator = parser.getSessionLocator(data); + + assertEquals(DEFAULT_IP, sessionLocator.getClientIp()); + assertEquals(DEFAULT_IP, sessionLocator.getServerIp()); + assertEquals(DEFAULT_PORT, sessionLocator.getClientPort()); + assertEquals(DEFAULT_PORT, sessionLocator.getServerPort()); + } + + @Test + void testAccessorWithMissingUser() { + String payload = "{" + + "\"Timestamp\":1768332558964138753," + + "\"EventType\":\"query_execute\"," + + "\"Statement\":\"SELECT 1\"" + + "}"; + + final JsonObject data = new Gson().fromJson(payload, JsonObject.class); + Accessor accessor = parser.getAccessor(data, NOT_AVAILABLE); + + assertEquals(NOT_AVAILABLE, accessor.getDbUser()); + assertEquals(SERVER_TYPE, accessor.getServerType()); + assertEquals(DB_PROTOCOL, accessor.getDbProtocol()); + assertEquals(LANGUAGE_COCKROACHDB, accessor.getLanguage()); + } + + @Test + void testMultipleErrors() { + String payload = "{" + + "\"Timestamp\":1768332558964138753," + + "\"EventType\":\"query_execute\"," + + "\"Statement\":\"INSERT INTO test VALUES (1, 'test')\"," + + "\"User\":\"test_user\"," + + "\"ErrorText\":\"duplicate key value violates unique constraint\"," + + "\"SQLSTATE\":\"23505\"," + + "\"ClientIP\":\"10.0.0.5\"," + + "\"ClientPort\":\"12345\"" + + "}"; + + final JsonObject data = new Gson().fromJson(payload, JsonObject.class); + Record record = parser.parseRecord(data); + + assertNotNull(record); + assertNull(record.getData()); + assertNotNull(record.getException()); + assertEquals(EXCEPTION_TYPE_SQL_ERROR_STRING, record.getException().getExceptionTypeId()); + assertTrue(record.getException().getDescription().contains("duplicate key")); + assertTrue(record.getException().getDescription().contains("[SQLSTATE: 23505]")); + assertEquals("INSERT INTO test VALUES (1, 'test')", record.getException().getSqlString()); + } + + @Test + void testSQLErrorWithoutSQLSTATE() { + // Test error without SQLSTATE field + String payload = "{" + + "\"Timestamp\":1768332558964138753," + + "\"EventType\":\"query_execute\"," + + "\"Statement\":\"SELECT * FROM products WHERE category = 'Electronics'\"," + + "\"User\":\"test_user\"," + + "\"ErrorText\":\"column \\\"category\\\" does not exist\"," + + "\"ClientIP\":\"10.0.0.5\"," + + "\"ClientPort\":\"12345\"" + + "}"; + + final JsonObject data = new Gson().fromJson(payload, JsonObject.class); + Record record = parser.parseRecord(data); + + assertNotNull(record); + assertNull(record.getData()); + assertNotNull(record.getException()); + assertEquals(EXCEPTION_TYPE_SQL_ERROR_STRING, record.getException().getExceptionTypeId()); + assertEquals("column \"category\" does not exist", record.getException().getDescription()); + assertFalse(record.getException().getDescription().contains("SQLSTATE")); + assertEquals("SELECT * FROM products WHERE category = 'Electronics'", record.getException().getSqlString()); + } + + @Test + void testTimestampConversion() { + String payload = "{" + + "\"Timestamp\":1768332558964138753," + + "\"EventType\":\"query_execute\"," + + "\"Statement\":\"SELECT NOW()\"," + + "\"User\":\"test_user\"" + + "}"; + + final JsonObject data = new Gson().fromJson(payload, JsonObject.class); + Record record = parser.parseRecord(data); + + assertNotNull(record); + assertNotNull(record.getTime()); + assertEquals(1768332558964L, record.getTime().getTimstamp()); + assertEquals(0, record.getTime().getMinOffsetFromGMT()); + assertEquals(0, record.getTime().getMinDst()); + } + + @Test + void testComplexJoinQuery() { + String payload = "{" + + "\"Timestamp\":1768332558964138753," + + "\"EventType\":\"query_execute\"," + + "\"Statement\":\"SELECT u.name, o.order_date, o.total FROM users u INNER JOIN orders o ON u.id = o.user_id WHERE o.total > 500\"," + + "\"Tag\":\"SELECT\"," + + "\"User\":\"guardium_qa\"," + + "\"ApplicationName\":\"$ cockroach sql\"," + + "\"DatabaseName\":\"testdb\"," + + "\"ClientIP\":\"127.0.0.1\"," + + "\"ClientPort\":\"49696\"" + + "}"; + + final JsonObject data = new Gson().fromJson(payload, JsonObject.class); + Record record = parser.parseRecord(data); + + assertNotNull(record); + assertEquals("guardium_qa", record.getAppUserName()); + assertEquals("testdb", record.getDbName()); + assertNotNull(record.getData()); + assertTrue(record.getData().getOriginalSqlCommand().contains("INNER JOIN")); + assertTrue(record.getData().getOriginalSqlCommand().contains("WHERE")); + assertNull(record.getException()); + } + + @Test + void testMultiLineQuery() { + String multiLineQuery = "SELECT u.id, u.name, u.email, c.name as customer_name, p.product_name, o.total, o.status, o.order_date\\n" + + "FROM users u\\n" + + "LEFT JOIN orders o ON u.id = o.user_id\\n" + + "LEFT JOIN customers c ON o.customer_id = c.id\\n" + + "WHERE u.active = true AND o.status IN ('pending', 'completed')\\n" + + "ORDER BY o.order_date DESC"; + + String payload = "{" + + "\"Timestamp\":1768332558964138753," + + "\"EventType\":\"query_execute\"," + + "\"Statement\":\"" + multiLineQuery + "\"," + + "\"Tag\":\"SELECT\"," + + "\"User\":\"guardium_qa\"," + + "\"ApplicationName\":\"$ cockroach sql\"," + + "\"DatabaseName\":\"testdb\"," + + "\"ClientIP\":\"127.0.0.1\"," + + "\"ClientPort\":\"49696\"" + + "}"; + + final JsonObject data = new Gson().fromJson(payload, JsonObject.class); + Record record = parser.parseRecord(data); + + assertNotNull(record); + assertNotNull(record.getData()); + assertTrue(record.getData().getOriginalSqlCommand().contains("LEFT JOIN")); + assertTrue(record.getData().getOriginalSqlCommand().contains("ORDER BY")); + assertNull(record.getException()); + } + + @Test + void testQueryWithSubquery() { + String payload = "{" + + "\"Timestamp\":1768332558964138753," + + "\"EventType\":\"query_execute\"," + + "\"Statement\":\"SELECT * FROM users WHERE id IN (SELECT user_id FROM orders WHERE total > 500)\"," + + "\"Tag\":\"SELECT\"," + + "\"User\":\"guardium_qa\"," + + "\"ApplicationName\":\"$ cockroach sql\"," + + "\"DatabaseName\":\"testdb\"," + + "\"ClientIP\":\"127.0.0.1\"," + + "\"ClientPort\":\"49696\"" + + "}"; + + final JsonObject data = new Gson().fromJson(payload, JsonObject.class); + Record record = parser.parseRecord(data); + + assertNotNull(record); + assertNotNull(record.getData()); + assertTrue(record.getData().getOriginalSqlCommand().contains("SELECT user_id FROM orders")); + assertNull(record.getException()); + } + + @Test + void testQueryWithCaseStatement() { + String payload = "{" + + "\"Timestamp\":1768332558964138753," + + "\"EventType\":\"query_execute\"," + + "\"Statement\":\"SELECT name, CASE WHEN salary < 50000 THEN 'Junior' WHEN salary < 100000 THEN 'Mid-level' ELSE 'Senior' END as level FROM employees\"," + + "\"Tag\":\"SELECT\"," + + "\"User\":\"guardium_qa\"," + + "\"ApplicationName\":\"$ cockroach sql\"," + + "\"DatabaseName\":\"testdb\"," + + "\"ClientIP\":\"127.0.0.1\"," + + "\"ClientPort\":\"49696\"" + + "}"; + + final JsonObject data = new Gson().fromJson(payload, JsonObject.class); + Record record = parser.parseRecord(data); + + assertNotNull(record); + assertNotNull(record.getData()); + assertTrue(record.getData().getOriginalSqlCommand().contains("CASE WHEN")); + assertTrue(record.getData().getOriginalSqlCommand().contains("ELSE")); + assertNull(record.getException()); + } + + @Test + void testVeryLongQuery() { + String longQuery = "SELECT u.id, u.name, u.email, u.active, u.last_login, u.created_at, " + + "c.name as customer_name, c.email as customer_email, c.phone, " + + "p.product_name, p.price, p.category, p.stock, " + + "o.total, o.status, o.order_date, oi.quantity, oi.price as item_price " + + "FROM users u " + + "LEFT JOIN orders o ON u.id = o.user_id " + + "LEFT JOIN customers c ON o.customer_id = c.id " + + "LEFT JOIN order_items oi ON o.id = oi.order_id " + + "LEFT JOIN products p ON oi.product_id = p.id " + + "WHERE u.active = true AND o.status IN ('pending', 'completed', 'shipped') " + + "AND p.category IN ('Electronics', 'Furniture') " + + "ORDER BY o.order_date DESC, u.name ASC LIMIT 100"; + + String payload = "{" + + "\"Timestamp\":1768332558964138753," + + "\"EventType\":\"query_execute\"," + + "\"Statement\":\"" + longQuery + "\"," + + "\"Tag\":\"SELECT\"," + + "\"User\":\"guardium_qa\"," + + "\"ApplicationName\":\"$ cockroach sql\"," + + "\"DatabaseName\":\"testdb\"," + + "\"ClientIP\":\"127.0.0.1\"," + + "\"ClientPort\":\"49696\"" + + "}"; + + final JsonObject data = new Gson().fromJson(payload, JsonObject.class); + Record record = parser.parseRecord(data); + + assertNotNull(record); + assertNotNull(record.getData()); + assertEquals(longQuery, record.getData().getOriginalSqlCommand()); + assertTrue(record.getData().getOriginalSqlCommand().length() > 400); + assertNull(record.getException()); + } + + @Test + void testQueryWithSpecialCharactersInString() { + String payload = "{" + + "\"Timestamp\":1768332558964138753," + + "\"EventType\":\"query_execute\"," + + "\"Statement\":\"INSERT INTO users (name, email) VALUES ('O''Brien', 'test@example.com')\"," + + "\"Tag\":\"INSERT\"," + + "\"User\":\"guardium_qa\"," + + "\"ApplicationName\":\"$ cockroach sql\"," + + "\"DatabaseName\":\"testdb\"," + + "\"ClientIP\":\"127.0.0.1\"," + + "\"ClientPort\":\"49696\"" + + "}"; + + final JsonObject data = new Gson().fromJson(payload, JsonObject.class); + Record record = parser.parseRecord(data); + + assertNotNull(record); + assertNotNull(record.getData()); + assertTrue(record.getData().getOriginalSqlCommand().contains("O''Brien")); + assertNull(record.getException()); + } + + @Test + void testComplexErrorWithMultiLineQuery() { + String multiLineQuery = "SELECT u.id, u.name\\nFROM users u\\nINNER JOIN orders o\\nWHERE invalid_column = 1"; + String payload = "{" + + "\"Timestamp\":1768332558964138753," + + "\"EventType\":\"query_execute\"," + + "\"Statement\":\"" + multiLineQuery + "\"," + + "\"Tag\":\"SELECT\"," + + "\"User\":\"guardium_qa\"," + + "\"ErrorText\":\"column \\\"invalid_column\\\" does not exist\"," + + "\"SQLSTATE\":\"42703\"," + + "\"ClientIP\":\"127.0.0.1\"," + + "\"ClientPort\":\"49696\"" + + "}"; + + final JsonObject data = new Gson().fromJson(payload, JsonObject.class); + Record record = parser.parseRecord(data); + + assertNotNull(record); + assertNull(record.getData()); + assertNotNull(record.getException()); + assertEquals(EXCEPTION_TYPE_SQL_ERROR_STRING, record.getException().getExceptionTypeId()); + assertTrue(record.getException().getDescription().contains("does not exist")); + assertTrue(record.getException().getSqlString().contains("INNER JOIN")); + } + + @Test + void testQueryWithNullValues() { + String payload = "{" + + "\"Timestamp\":1768332558964138753," + + "\"EventType\":\"query_execute\"," + + "\"Statement\":\"SELECT * FROM users WHERE email IS NULL OR last_login IS NOT NULL\"," + + "\"Tag\":\"SELECT\"," + + "\"User\":\"guardium_qa\"," + + "\"ApplicationName\":\"$ cockroach sql\"," + + "\"DatabaseName\":\"testdb\"," + + "\"ClientIP\":\"127.0.0.1\"," + + "\"ClientPort\":\"49696\"" + + "}"; + + final JsonObject data = new Gson().fromJson(payload, JsonObject.class); + Record record = parser.parseRecord(data); + + assertNotNull(record); + assertNotNull(record.getData()); + assertTrue(record.getData().getOriginalSqlCommand().contains("IS NULL")); + assertTrue(record.getData().getOriginalSqlCommand().contains("IS NOT NULL")); + assertNull(record.getException()); + } + + // ============================================================================ + // Tests for cleanExtraChars method + // ============================================================================ + + @Test + void testCleanRedactionMarkersFromUsername() { + // Test that CockroachDB extra characters are removed from username + String payload = "{" + + "\"Timestamp\":1768332558964138753," + + "\"EventType\":\"query_execute\"," + + "\"Statement\":\"SELECT 1\"," + + "\"User\":\"‹guardium_qa›\"," + + "\"ApplicationName\":\"$ cockroach sql\"," + + "\"ClientIP\":\"127.0.0.1\"," + + "\"ClientPort\":\"49696\"" + + "}"; + + final JsonObject data = new Gson().fromJson(payload, JsonObject.class); + Record record = parser.parseRecord(data); + + assertNotNull(record); + assertEquals("guardium_qa", record.getAppUserName()); + assertEquals("guardium_qa", record.getAccessor().getDbUser()); + assertFalse(record.getAppUserName().contains("‹")); + assertFalse(record.getAppUserName().contains("›")); + } + + @Test + void testPreserveLessThanGreaterThanOperators() { + // Test that SQL comparison operators < and > are NOT removed + String payload = "{" + + "\"Timestamp\":1768332558964138753," + + "\"EventType\":\"query_execute\"," + + "\"Statement\":\"SELECT * FROM users WHERE age > 18 AND salary < 100000\"," + + "\"Tag\":\"SELECT\"," + + "\"User\":\"guardium_qa\"," + + "\"ApplicationName\":\"$ cockroach sql\"," + + "\"DatabaseName\":\"testdb\"," + + "\"ClientIP\":\"127.0.0.1\"," + + "\"ClientPort\":\"49696\"" + + "}"; + + final JsonObject data = new Gson().fromJson(payload, JsonObject.class); + Record record = parser.parseRecord(data); + + assertNotNull(record); + assertNotNull(record.getData()); + String sql = record.getData().getOriginalSqlCommand(); + + // Verify < and > operators are preserved + assertTrue(sql.contains(">"), "Greater than operator should be preserved"); + assertTrue(sql.contains("<"), "Less than operator should be preserved"); + assertEquals("SELECT * FROM users WHERE age > 18 AND salary < 100000", sql); + } + + @Test + void testPreserveLessThanEqualGreaterThanEqual() { + // Test that <= and >= operators are preserved + String payload = "{" + + "\"Timestamp\":1768332558964138753," + + "\"EventType\":\"query_execute\"," + + "\"Statement\":\"SELECT * FROM products WHERE price >= 10.00 AND stock <= 100\"," + + "\"Tag\":\"SELECT\"," + + "\"User\":\"guardium_qa\"," + + "\"ApplicationName\":\"$ cockroach sql\"," + + "\"DatabaseName\":\"testdb\"," + + "\"ClientIP\":\"127.0.0.1\"," + + "\"ClientPort\":\"49696\"" + + "}"; + + final JsonObject data = new Gson().fromJson(payload, JsonObject.class); + Record record = parser.parseRecord(data); + + assertNotNull(record); + assertNotNull(record.getData()); + String sql = record.getData().getOriginalSqlCommand(); + + assertTrue(sql.contains(">="), "Greater than or equal operator should be preserved"); + assertTrue(sql.contains("<="), "Less than or equal operator should be preserved"); + assertEquals("SELECT * FROM products WHERE price >= 10.00 AND stock <= 100", sql); + } + + @Test + void testPreserveNotEqualOperator() { + // Test that <> (not equal) operator is preserved + String payload = "{" + + "\"Timestamp\":1768332558964138753," + + "\"EventType\":\"query_execute\"," + + "\"Statement\":\"SELECT * FROM users WHERE status <> 'deleted'\"," + + "\"Tag\":\"SELECT\"," + + "\"User\":\"guardium_qa\"," + + "\"ApplicationName\":\"$ cockroach sql\"," + + "\"DatabaseName\":\"testdb\"," + + "\"ClientIP\":\"127.0.0.1\"," + + "\"ClientPort\":\"49696\"" + + "}"; + + final JsonObject data = new Gson().fromJson(payload, JsonObject.class); + Record record = parser.parseRecord(data); + + assertNotNull(record); + assertNotNull(record.getData()); + String sql = record.getData().getOriginalSqlCommand(); + + assertTrue(sql.contains("<>"), "Not equal operator <> should be preserved"); + assertEquals("SELECT * FROM users WHERE status <> 'deleted'", sql); + } + + @Test + void testMixedRedactionMarkersAndSQLOperators() { + // Test query with both extra characters AND SQL operators + String payload = "{" + + "\"Timestamp\":1768332558964138753," + + "\"EventType\":\"query_execute\"," + + "\"Statement\":\"SELECT * FROM users WHERE age > ‹18› AND salary < ‹100000› AND status <> ‹'inactive'›\"," + + "\"Tag\":\"SELECT\"," + + "\"User\":\"‹guardium_qa›\"," + + "\"ApplicationName\":\"$ cockroach sql\"," + + "\"DatabaseName\":\"testdb\"," + + "\"ClientIP\":\"127.0.0.1\"," + + "\"ClientPort\":\"49696\"" + + "}"; + + final JsonObject data = new Gson().fromJson(payload, JsonObject.class); + Record record = parser.parseRecord(data); + + assertNotNull(record); + assertNotNull(record.getData()); + String sql = record.getData().getOriginalSqlCommand(); + + // Verify extra characters are removed + assertFalse(sql.contains("‹"), "Extra character ‹ should be removed"); + assertFalse(sql.contains("›"), "Extra character › should be removed"); + + // Verify SQL operators are preserved + assertTrue(sql.contains(">"), "Greater than operator should be preserved"); + assertTrue(sql.contains("<"), "Less than operator should be preserved"); + assertTrue(sql.contains("<>"), "Not equal operator should be preserved"); + + // Verify final result + assertEquals("SELECT * FROM users WHERE age > 18 AND salary < 100000 AND status <> 'inactive'", sql); + + // Verify username is also cleaned + assertEquals("guardium_qa", record.getAppUserName()); + } + + @Test + void testMultipleRedactionMarkersInValues() { + // Test multiple values with extra characters + String payload = "{" + + "\"Timestamp\":1768332558964138753," + + "\"EventType\":\"query_execute\"," + + "\"Statement\":\"INSERT INTO products (id, name, price) VALUES (‹1›, ‹'Product A'›, ‹19.99›), (‹2›, ‹'Product B'›, ‹29.99›)\"," + + "\"Tag\":\"INSERT\"," + + "\"User\":\"guardium_qa\"," + + "\"ApplicationName\":\"$ cockroach sql\"," + + "\"DatabaseName\":\"testdb\"," + + "\"ClientIP\":\"127.0.0.1\"," + + "\"ClientPort\":\"49696\"" + + "}"; + + final JsonObject data = new Gson().fromJson(payload, JsonObject.class); + Record record = parser.parseRecord(data); + + assertNotNull(record); + assertNotNull(record.getData()); + String sql = record.getData().getOriginalSqlCommand(); + + assertFalse(sql.contains("‹")); + assertFalse(sql.contains("›")); + assertEquals("INSERT INTO products (id, name, price) VALUES (1, 'Product A', 19.99), (2, 'Product B', 29.99)", sql); + } + + @Test + void testComplexQueryWithBothMarkerTypes() { + // Test complex query with BETWEEN, IN, and extra characters + String payload = "{" + + "\"Timestamp\":1768332558964138753," + + "\"EventType\":\"query_execute\"," + + "\"Statement\":\"SELECT * FROM orders WHERE total BETWEEN ‹100› AND ‹500› AND status IN (‹'pending'›, ‹'completed'›) AND created_at > ‹'2024-01-01'›\"," + + "\"Tag\":\"SELECT\"," + + "\"User\":\"guardium_qa\"," + + "\"ApplicationName\":\"$ cockroach sql\"," + + "\"DatabaseName\":\"testdb\"," + + "\"ClientIP\":\"127.0.0.1\"," + + "\"ClientPort\":\"49696\"" + + "}"; + + final JsonObject data = new Gson().fromJson(payload, JsonObject.class); + Record record = parser.parseRecord(data); + + assertNotNull(record); + assertNotNull(record.getData()); + String sql = record.getData().getOriginalSqlCommand(); + + // Verify extra characters removed + assertFalse(sql.contains("‹")); + assertFalse(sql.contains("›")); + + // Verify > operator preserved + assertTrue(sql.contains(">")); + + // Verify final result + assertEquals("SELECT * FROM orders WHERE total BETWEEN 100 AND 500 AND status IN ('pending', 'completed') AND created_at > '2024-01-01'", sql); + } + + @Test + void testOnlyRedactionMarkers() { + // Test string with only extra characters + String payload = "{" + + "\"Timestamp\":1768332558964138753," + + "\"EventType\":\"query_execute\"," + + "\"Statement\":\"‹›‹›‹›\"," + + "\"Tag\":\"SELECT\"," + + "\"User\":\"‹›\"," + + "\"ClientIP\":\"127.0.0.1\"," + + "\"ClientPort\":\"49696\"" + + "}"; + + final JsonObject data = new Gson().fromJson(payload, JsonObject.class); + Record record = parser.parseRecord(data); + + assertNotNull(record); + assertNotNull(record.getData()); + assertEquals("", record.getData().getOriginalSqlCommand()); + assertEquals("", record.getAppUserName()); + } +} \ No newline at end of file diff --git a/filter-plugin/logstash-filter-cosmos-azure-guardium/build.gradle b/filter-plugin/logstash-filter-cosmos-azure-guardium/build.gradle index 4416fc462..584b55c9b 100644 --- a/filter-plugin/logstash-filter-cosmos-azure-guardium/build.gradle +++ b/filter-plugin/logstash-filter-cosmos-azure-guardium/build.gradle @@ -2,6 +2,29 @@ import java.nio.file.Files import static java.nio.file.StandardCopyOption.REPLACE_EXISTING apply plugin: 'java' + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + + dependencies { + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' + classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' + } +} + +def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + apply plugin: 'jacoco' apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" @@ -21,29 +44,19 @@ pluginInfo.pluginClass = "AzureCosmosGuardiumFilter" pluginInfo.pluginName = "azure_cosmos_guardium_filter" // must match the @LogstashPlugin annotation in the main plugin class // =========================================================================== -sourceCompatibility = 1.8 -targetCompatibility = 1.8 - -buildscript { - repositories { - maven { - url "https://plugins.gradle.org/m2/" - } - mavenCentral() - jcenter() - } - - dependencies { - classpath 'com.github.jengelman.gradle.plugins:shadow:4.0.4' - classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' - } -} +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 -def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); -def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) repositories { - mavenCentral() + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } } task copyDependencyLibs(type: Copy) { @@ -58,7 +71,7 @@ task copyDependencyLibs(type: Copy) { apply plugin: 'com.github.johnrengelman.shadow' shadowJar { - classifier = null + archiveClassifier = null transform(com.github.jengelman.gradle.plugins.shadow.transformers.Log4j2PluginsCacheFileTransformer) } @@ -67,7 +80,8 @@ dependencies { implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: versions.dependencies.log4jCore implementation 'org.apache.commons:commons-lang3:' + versions.dependencies.commonsLang implementation 'com.google.code.gson:gson:' + versions.dependencies.gson - implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "build/libs/logstash-core-*.*.*.jar") + implementation group: 'commons-beanutils', name: 'commons-beanutils', version: versions.dependencies.commonsBeanutils + implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "build/libs/logstash-core.jar") implementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "common-*.*.*.jar") implementation group: 'org.parboiled', name: 'parboiled-java', version: versions.dependencies.parboiledJava @@ -94,6 +108,17 @@ tasks.withType(JavaCompile) { options.encoding = 'UTF-8' } +test { + jvmArgs = [ + '--add-opens=java.base/java.lang=ALL-UNNAMED', + '--add-opens=java.base/java.util=ALL-UNNAMED', + '--add-opens=java.base/java.text=ALL-UNNAMED', + '--add-opens=java.base/java.lang.reflect=ALL-UNNAMED', + '--add-opens=java.base/sun.util.resources=ALL-UNNAMED', + '--add-opens=java.base/sun.util.cldr=ALL-UNNAMED' + ] +} + tasks.register("vendor"){ dependsOn shadowJar doLast { @@ -102,7 +127,6 @@ tasks.register("vendor"){ File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") projectJarFile.mkdirs() Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) - validatePluginJar(projectJarFile, project.group) } } @@ -131,8 +155,8 @@ tasks.register("gem"){ } jacocoTestReport { reports { - xml.enabled true - html.enabled true + xml.required = true + html.required = true } afterEvaluate { // (optional) : to exclude classes / packages from coverage diff --git a/filter-plugin/logstash-filter-cosmos-azure-guardium/gradle/wrapper/gradle-wrapper.properties b/filter-plugin/logstash-filter-cosmos-azure-guardium/gradle/wrapper/gradle-wrapper.properties index aa991fcea..81aa1c044 100644 --- a/filter-plugin/logstash-filter-cosmos-azure-guardium/gradle/wrapper/gradle-wrapper.properties +++ b/filter-plugin/logstash-filter-cosmos-azure-guardium/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/filter-plugin/logstash-filter-couchbasedb-guardium/CHANGELOG.md b/filter-plugin/logstash-filter-couchbasedb-guardium/CHANGELOG.md index 21a4ce94e..4f2652278 100644 --- a/filter-plugin/logstash-filter-couchbasedb-guardium/CHANGELOG.md +++ b/filter-plugin/logstash-filter-couchbasedb-guardium/CHANGELOG.md @@ -3,6 +3,9 @@ Notable changes will be documented in this file. ## [Unreleased] +## [1.0.3] - 2026-05-06 +- GRD-124860: Fix two S-TAP entry in the report by setting server port to default server port `-1` + ## [1.0.2] - 2025-05-30 - Support different operations from UI diff --git a/filter-plugin/logstash-filter-couchbasedb-guardium/CouchbasedbOverFilebeatPackage/CouchbaseDB/logstash-filter-couchbasedb_guardium_plugin_filter.zip b/filter-plugin/logstash-filter-couchbasedb-guardium/CouchbasedbOverFilebeatPackage/CouchbaseDB/logstash-filter-couchbasedb_guardium_plugin_filter.zip index c64b10d68..83589f059 100644 Binary files a/filter-plugin/logstash-filter-couchbasedb-guardium/CouchbasedbOverFilebeatPackage/CouchbaseDB/logstash-filter-couchbasedb_guardium_plugin_filter.zip and b/filter-plugin/logstash-filter-couchbasedb-guardium/CouchbasedbOverFilebeatPackage/CouchbaseDB/logstash-filter-couchbasedb_guardium_plugin_filter.zip differ diff --git a/filter-plugin/logstash-filter-couchbasedb-guardium/README.md b/filter-plugin/logstash-filter-couchbasedb-guardium/README.md index bb52dbdc6..cc5a98aba 100644 --- a/filter-plugin/logstash-filter-couchbasedb-guardium/README.md +++ b/filter-plugin/logstash-filter-couchbasedb-guardium/README.md @@ -73,8 +73,9 @@ https://www.elastic.co/guide/en/beats/filebeat/current/directory-layout.html • Locate "filebeat.inputs" in the filebeat.yml file, then add the following parameters. filebeat.inputs: - - type: log - - enabled: true + - type: filestream + - id: + - enabled: true paths: - diff --git a/filter-plugin/logstash-filter-couchbasedb-guardium/VERSION b/filter-plugin/logstash-filter-couchbasedb-guardium/VERSION index 6d7de6e6a..21e8796a0 100644 --- a/filter-plugin/logstash-filter-couchbasedb-guardium/VERSION +++ b/filter-plugin/logstash-filter-couchbasedb-guardium/VERSION @@ -1 +1 @@ -1.0.2 +1.0.3 diff --git a/filter-plugin/logstash-filter-couchbasedb-guardium/build.gradle b/filter-plugin/logstash-filter-couchbasedb-guardium/build.gradle index d024fa980..9bd2ab520 100644 --- a/filter-plugin/logstash-filter-couchbasedb-guardium/build.gradle +++ b/filter-plugin/logstash-filter-couchbasedb-guardium/build.gradle @@ -2,6 +2,30 @@ import java.nio.file.Files import static java.nio.file.StandardCopyOption.REPLACE_EXISTING apply plugin: 'java' + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + + dependencies { + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' + classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.1.0" + classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' + } +} + +def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" // =========================================================================== @@ -20,10 +44,10 @@ pluginInfo.pluginClass = "CouchbasedbGuardiumPluginFilter" pluginInfo.pluginName = "couchbasedb_guardium_plugin_filter" // must match the @LogstashPlugin annotation in the main plugin class // =========================================================================== -sourceCompatibility = 1.8 -targetCompatibility = 1.8 +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 -def jacocoVersion = '0.8.4' +def jacocoVersion = '0.8.11' // minimumCoverage can be set by Travis ENV def minimumCoverageStr = System.getenv("MINIMUM_COVERAGE") ?: "50.0%" if (minimumCoverageStr.endsWith("%")) { @@ -31,28 +55,17 @@ if (minimumCoverageStr.endsWith("%")) { } def minimumCoverage = Float.valueOf(minimumCoverageStr) / 100 -buildscript { - repositories { - maven { - url "https://plugins.gradle.org/m2/" - } - mavenCentral() - jcenter() - } - - dependencies { - classpath 'com.github.jengelman.gradle.plugins:shadow:4.0.4' - classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.0.0-RC2" - classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' - } -} - -def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); -def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) repositories { - mavenCentral() + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } } tasks.register("vendor"){ @@ -63,14 +76,13 @@ tasks.register("vendor"){ File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") projectJarFile.mkdirs() Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) - validatePluginJar(projectJarFile, project.group) } } apply plugin: 'com.github.johnrengelman.shadow' shadowJar { - classifier = null + archiveClassifier = null } dependencies { @@ -78,6 +90,7 @@ dependencies { implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: versions.dependencies.log4jCore implementation 'org.apache.commons:commons-lang3:' + versions.dependencies.commonsLang implementation 'com.google.code.gson:gson:' + versions.dependencies.gson + implementation group: 'commons-beanutils', name: 'commons-beanutils', version: versions.dependencies.commonsBeanutils implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "build/libs/logstash-core-*.*.*.jar") implementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "common-*.*.*.jar") @@ -101,6 +114,17 @@ tasks.withType(JavaCompile) { options.encoding = 'UTF-8' } +test { + jvmArgs = [ + '--add-opens=java.base/java.lang=ALL-UNNAMED', + '--add-opens=java.base/java.util=ALL-UNNAMED', + '--add-opens=java.base/java.text=ALL-UNNAMED', + '--add-opens=java.base/java.lang.reflect=ALL-UNNAMED', + '--add-opens=java.base/sun.util.resources=ALL-UNNAMED', + '--add-opens=java.base/sun.util.cldr=ALL-UNNAMED' + ] +} + tasks.register("generateRubySupportFiles") { @@ -143,17 +167,16 @@ apply plugin: "org.barfuin.gradle.jacocolog" // ------------------------------------ jacoco { toolVersion = "${jacocoVersion}" - reportsDir = file("$buildDir/reports/jacoco") } jacocoTestReport { // You will see "Report -> file://...." at the end of a JaCoCo build // If no output, run this first: ./gradlew test reports { - html.enabled true - xml.enabled true - csv.enabled true - html.destination file("${buildDir}/reports/jacoco") - csv.destination file("${buildDir}/reports/jacoco/all.csv") + html.required = true + xml.required = true + csv.required = true + html.outputLocation = file("${buildDir}/reports/jacoco") + csv.outputLocation = file("${buildDir}/reports/jacoco/all.csv") } executionData.from fileTree(dir: "${buildDir}/jacoco/", includes: [ '**/*.exec' diff --git a/filter-plugin/logstash-filter-couchbasedb-guardium/gradle/wrapper/gradle-wrapper.properties b/filter-plugin/logstash-filter-couchbasedb-guardium/gradle/wrapper/gradle-wrapper.properties index aa991fcea..81aa1c044 100644 --- a/filter-plugin/logstash-filter-couchbasedb-guardium/gradle/wrapper/gradle-wrapper.properties +++ b/filter-plugin/logstash-filter-couchbasedb-guardium/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/filter-plugin/logstash-filter-couchbasedb-guardium/logstash-filter-couchbasedb_guardium_plugin_filter.zip b/filter-plugin/logstash-filter-couchbasedb-guardium/logstash-filter-couchbasedb_guardium_plugin_filter.zip index c64b10d68..83589f059 100644 Binary files a/filter-plugin/logstash-filter-couchbasedb-guardium/logstash-filter-couchbasedb_guardium_plugin_filter.zip and b/filter-plugin/logstash-filter-couchbasedb-guardium/logstash-filter-couchbasedb_guardium_plugin_filter.zip differ diff --git a/filter-plugin/logstash-filter-couchbasedb-guardium/src/main/java/com/ibm/guardium/couchbasedb/Parser.java b/filter-plugin/logstash-filter-couchbasedb-guardium/src/main/java/com/ibm/guardium/couchbasedb/Parser.java index 42593d6f5..f3f1565fc 100644 --- a/filter-plugin/logstash-filter-couchbasedb-guardium/src/main/java/com/ibm/guardium/couchbasedb/Parser.java +++ b/filter-plugin/logstash-filter-couchbasedb-guardium/src/main/java/com/ibm/guardium/couchbasedb/Parser.java @@ -149,7 +149,6 @@ public static void setServerIPAndPort(final SessionLocator sessionLocator, final if (!validator.isValid(serverIp)|| serverIp.equals(Constants.LOOPBACK_ADDRESS)) { serverIp = data.get(Constants.SERVER_IP).getAsString(); } - serverPort = Integer.parseInt(nodeData[1]); }else { serverIp = data.get(Constants.SERVER_IP).getAsString(); diff --git a/filter-plugin/logstash-filter-couchbasedb-guardium/src/test/java/com/ibm/guardium/couchbasedb/ParserTest.java b/filter-plugin/logstash-filter-couchbasedb-guardium/src/test/java/com/ibm/guardium/couchbasedb/ParserTest.java index 87d2e0f78..e89dc728e 100644 --- a/filter-plugin/logstash-filter-couchbasedb-guardium/src/test/java/com/ibm/guardium/couchbasedb/ParserTest.java +++ b/filter-plugin/logstash-filter-couchbasedb-guardium/src/test/java/com/ibm/guardium/couchbasedb/ParserTest.java @@ -6,7 +6,15 @@ import java.text.ParseException; -import com.ibm.guardium.universalconnector.commons.structures.*; +import com.ibm.guardium.universalconnector.commons.structures.Accessor; +import com.ibm.guardium.universalconnector.commons.structures.Construct; +import com.ibm.guardium.universalconnector.commons.structures.Data; +import com.ibm.guardium.universalconnector.commons.structures.ExceptionRecord; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import com.ibm.guardium.universalconnector.commons.structures.Sentence; +import com.ibm.guardium.universalconnector.commons.structures.SentenceObject; +import com.ibm.guardium.universalconnector.commons.structures.SessionLocator; +import com.ibm.guardium.universalconnector.commons.structures.Time; import org.junit.Assert; import org.junit.Test; @@ -77,7 +85,7 @@ public void testForSessionLocator() throws ParseException { SessionLocator sessionLocator = Parser.parseSessionLocator(couchbaseJson); Assert.assertEquals(40389, sessionLocator.getClientPort()); - Assert.assertEquals(8091, sessionLocator.getServerPort()); + Assert.assertEquals(Constants.DEFAULT_SERVER_PORT, sessionLocator.getServerPort()); Assert.assertEquals(false, sessionLocator.isIpv6()); } diff --git a/filter-plugin/logstash-filter-couchdb-guardium/README.md b/filter-plugin/logstash-filter-couchdb-guardium/README.md index 0b6d63de1..dac7500ee 100644 --- a/filter-plugin/logstash-filter-couchdb-guardium/README.md +++ b/filter-plugin/logstash-filter-couchdb-guardium/README.md @@ -66,7 +66,8 @@ Filebeat must be configured to send the output to the chosen Logstash host and p • Locate "filebeat.inputs" in the filebeat.yml file and then use the "paths" attribute to set the location of the couchdb logs: ``` filebeat.inputs: - - type: log + - type: filestream + - id: enabled: true paths: - /var/log/couchdb/*.log diff --git a/filter-plugin/logstash-filter-couchdb-guardium/build.gradle b/filter-plugin/logstash-filter-couchdb-guardium/build.gradle index 8adb88feb..4b9b95f8a 100644 --- a/filter-plugin/logstash-filter-couchdb-guardium/build.gradle +++ b/filter-plugin/logstash-filter-couchdb-guardium/build.gradle @@ -2,6 +2,30 @@ import java.nio.file.Files import static java.nio.file.StandardCopyOption.REPLACE_EXISTING apply plugin: 'java' + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + + dependencies { + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' + classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.1.0" + classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' + } +} + +def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" // =========================================================================== @@ -20,10 +44,10 @@ pluginInfo.pluginClass = "CouchdbGuardiumFilter" pluginInfo.pluginName = "couchdb_guardium_filter" // must match the @LogstashPlugin annotation in the main plugin class // =========================================================================== -sourceCompatibility = 1.8 -targetCompatibility = 1.8 +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 -def jacocoVersion = '0.8.4' +def jacocoVersion = '0.8.11' // minimumCoverage can be set by Travis ENV def minimumCoverageStr = System.getenv("MINIMUM_COVERAGE") ?: "50.0%" if (minimumCoverageStr.endsWith("%")) { @@ -31,27 +55,16 @@ if (minimumCoverageStr.endsWith("%")) { } def minimumCoverage = Float.valueOf(minimumCoverageStr) / 100 -buildscript { - repositories { - maven { - url "https://plugins.gradle.org/m2/" - } - mavenCentral() - jcenter() - } - - dependencies { - classpath 'com.github.jengelman.gradle.plugins:shadow:4.0.4' - classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.0.0-RC2" - classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' - } -} - -def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); -def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) repositories { - mavenCentral() + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } } tasks.register("vendor"){ @@ -62,14 +75,13 @@ tasks.register("vendor"){ File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") projectJarFile.mkdirs() Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) - validatePluginJar(projectJarFile, project.group) } } apply plugin: 'com.github.johnrengelman.shadow' shadowJar { - classifier = null + archiveClassifier = null } dependencies { @@ -77,6 +89,7 @@ dependencies { implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: versions.dependencies.log4jCore implementation 'org.apache.commons:commons-lang3:' + versions.dependencies.commonsLang implementation 'com.google.code.gson:gson:' + versions.dependencies.gson + implementation group: 'commons-beanutils', name: 'commons-beanutils', version: versions.dependencies.commonsBeanutils // Use JUnit Jupiter for testing. testImplementation group: 'org.mockito', name: 'mockito-all', version: versions.dependencies.mockitoAll @@ -154,17 +167,16 @@ apply plugin: "org.barfuin.gradle.jacocolog" // ------------------------------------ jacoco { toolVersion = "${jacocoVersion}" - reportsDir = file("$buildDir/reports/jacoco") } jacocoTestReport { // You will see "Report -> file://...." at the end of a JaCoCo build // If no output, run this first: ./gradlew test reports { - html.enabled true - xml.enabled true - csv.enabled true - html.destination file("${buildDir}/reports/jacoco") - csv.destination file("${buildDir}/reports/jacoco/all.csv") + html.required = true + xml.required = true + csv.required = true + html.outputLocation = file("${buildDir}/reports/jacoco") + csv.outputLocation = file("${buildDir}/reports/jacoco/all.csv") } executionData.from fileTree(dir: "${buildDir}/jacoco/", includes: [ '**/*.exec' diff --git a/filter-plugin/logstash-filter-couchdb-guardium/gradle/wrapper/gradle-wrapper.properties b/filter-plugin/logstash-filter-couchdb-guardium/gradle/wrapper/gradle-wrapper.properties index aa991fcea..81aa1c044 100644 --- a/filter-plugin/logstash-filter-couchdb-guardium/gradle/wrapper/gradle-wrapper.properties +++ b/filter-plugin/logstash-filter-couchdb-guardium/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/filter-plugin/logstash-filter-couchdb-guardium/src/main/java/com/ibm/guardium/couchdb/Parser.java b/filter-plugin/logstash-filter-couchdb-guardium/src/main/java/com/ibm/guardium/couchdb/Parser.java index 0c850f66a..a8ffcad7d 100644 --- a/filter-plugin/logstash-filter-couchdb-guardium/src/main/java/com/ibm/guardium/couchdb/Parser.java +++ b/filter-plugin/logstash-filter-couchdb-guardium/src/main/java/com/ibm/guardium/couchdb/Parser.java @@ -17,7 +17,15 @@ import org.apache.logging.log4j.Logger; import com.ibm.guardium.couchdb.Parser; -import com.ibm.guardium.universalconnector.commons.structures.*; +import com.ibm.guardium.universalconnector.commons.structures.Accessor; +import com.ibm.guardium.universalconnector.commons.structures.Construct; +import com.ibm.guardium.universalconnector.commons.structures.Data; +import com.ibm.guardium.universalconnector.commons.structures.ExceptionRecord; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import com.ibm.guardium.universalconnector.commons.structures.Sentence; +import com.ibm.guardium.universalconnector.commons.structures.SentenceObject; +import com.ibm.guardium.universalconnector.commons.structures.SessionLocator; +import com.ibm.guardium.universalconnector.commons.structures.Time; import com.ibm.guardium.universalconnector.commons.structures.Record; import co.elastic.logstash.api.Event; diff --git a/filter-plugin/logstash-filter-databricks-guardium/AzureDatabricksOverAzureEventHub/databricks.conf b/filter-plugin/logstash-filter-databricks-guardium/AzureDatabricksOverAzureEventHub/databricks.conf index 0f278ff42..8c0c7dd18 100644 --- a/filter-plugin/logstash-filter-databricks-guardium/AzureDatabricksOverAzureEventHub/databricks.conf +++ b/filter-plugin/logstash-filter-databricks-guardium/AzureDatabricksOverAzureEventHub/databricks.conf @@ -1,9 +1,19 @@ input { azure_event_hubs { - event_hub_connections => [""] + config_mode => "basic" + #Insert primary connection string from shared access policies in event hub from azure portal + event_hub_connections => [""] # event_hub_connections => ["", ""] initial_position => "end" - threads => 8 + threads => 16 + # Consumer Group is recommanded when the requirement is to read from multiple EventHubs. + consumer_group => "gconsumer_group" + decorate_events => false type => "databricks" + #Insert Storage Connection String retrived while creating the storage account. + #Recommanded when the requirement is to read from multiple EventHubs. Else, data loss may happen. + storage_connection => "" + #Insert your enrollmentId of azure account + add_field => {"enrollmentId" => } } } diff --git a/filter-plugin/logstash-filter-databricks-guardium/logstash-filter-databricks_guardium_filter.zip b/filter-plugin/logstash-filter-databricks-guardium/AzureDatabricksOverAzureEventHub/logstash-filter-databricks_guardium_filter-1.0.1.zip similarity index 55% rename from filter-plugin/logstash-filter-databricks-guardium/logstash-filter-databricks_guardium_filter.zip rename to filter-plugin/logstash-filter-databricks-guardium/AzureDatabricksOverAzureEventHub/logstash-filter-databricks_guardium_filter-1.0.1.zip index d1ae5cd49..f2dbef64b 100644 Binary files a/filter-plugin/logstash-filter-databricks-guardium/logstash-filter-databricks_guardium_filter.zip and b/filter-plugin/logstash-filter-databricks-guardium/AzureDatabricksOverAzureEventHub/logstash-filter-databricks_guardium_filter-1.0.1.zip differ diff --git a/filter-plugin/logstash-filter-databricks-guardium/CHANGELOG.md b/filter-plugin/logstash-filter-databricks-guardium/CHANGELOG.md index 143dd1602..ac3233700 100644 --- a/filter-plugin/logstash-filter-databricks-guardium/CHANGELOG.md +++ b/filter-plugin/logstash-filter-databricks-guardium/CHANGELOG.md @@ -4,7 +4,14 @@ Notable changes will be documented in this file. -## [] +## [1.0.0] - 2025-07-11 ### Added - Initial release, in parallel to Guardium . +## [1.0.1] - 2025-08-13 +### Added +- Set service name and database name to be the same . + +## [1.0.1] - 2025-09-25 +### Added +- Fixed serverhostname, added default value to source program. \ No newline at end of file diff --git a/filter-plugin/logstash-filter-databricks-guardium/README.md b/filter-plugin/logstash-filter-databricks-guardium/README.md index 4d5cd731f..317f05fb0 100644 --- a/filter-plugin/logstash-filter-databricks-guardium/README.md +++ b/filter-plugin/logstash-filter-databricks-guardium/README.md @@ -4,7 +4,7 @@ * Environment: Azure * Supported inputs: Azure Event Hub (pull) * Supported Guardium versions: -* Guardium Data Protection: 11.4 and above + * Guardium Data Protection: 11.5 and later This is a [Logstash](https://github.com/elastic/logstash) filter plug-in for the universal connector that is featured in IBM Security Guardium. It parses events and messages from the Azure-Databricks audit log into a [Guardium record](https://github.com/IBM/universal-connectors/blob/main/common/src/main/java/com/ibm/guardium/universalconnector/commons/structures/Record.java) instance (which is a standard structure made out of several parts). The information is then sent over to Guardium. Guardium records include the accessor (the person who tried to access the data), the session, data, and exceptions. If there are no errors, the data contains details about the query "construct". The construct details the main action (verb) and collections (objects) involved. @@ -27,69 +27,70 @@ The plug-in is free and open-source (Apache 2.0). It can be used as a starting p ### Azure Event Hub Connection: - 1. Search "event hub" in the search bar. - 2. Select **create event hubs namespace**. - - 3. To create a namespace: - - Select the subscription in which you want to create the namespace. - - Select the resource group you created in the previous step. - - Enter a unique name for the namespace. - - Select a location for the namespace. - - Choose the appropriate pricing tier. (In this example, we selected basic). - - Leave the throughput units (or processing units for standard and premium tier) settings as is. - - Select **Review + Create**. Review the settings and select **Create**. - - Your recently created namespace appears in **resource group**. +1. Search "event hub" in the search bar. +2. Select **create event hubs namespace**. + +3. To create a namespace: + - Select the subscription in which you want to create the namespace. + - Select the resource group you created in the previous step. + - Enter a unique name for the namespace. + - Select a location for the namespace. + - Choose the appropriate pricing tier. (In this example, we selected basic). + - Leave the throughput units (or processing units for standard and premium tier) settings as is. + - Select **Review + Create**. Review the settings and select **Create**. + - Your recently created namespace appears in **resource group**. - 4. To create an event hub: - - Go to the Event Hubs Namespace page. - - Click **+ Event Hub**. - - Enter a unique name for the event hub. - - Choose at least the maximum number of partitions that you expect to require during peak usage for this event hub. - For example, if you want to generate traffic from 2 DB instances, then choose at least 2 partitions if not more. - - Click **Review+create**. - - Review the settings and click **Create**. +4. To create an event hub: + - Go to the Event Hubs Namespace page. + - Click **+ Event Hub**. + - Enter a unique name for the event hub. + - Choose at least the maximum number of partitions that you expect to require during peak usage for this event hub. + For example, if you want to generate traffic from 2 DB instances, then choose at least 2 partitions if not more. + - Click **Review+create**. + - Review the settings and click **Create**. - 5. Connection string for an event hub: - - In the list of event hubs, select your event hub. - - On the Event Hubs instance page, go to **Settings** > **Shared access policies** > **Add**. - - Name the policy, click **manage** to provide permissions, and create the policy. - - Select Connection string–primary key string from policy (it would be required in input plugin). - - 6. Azure Storage Accounts Creation: - - Login to https://portal.azure.com. - - Search Storage accounts in search bar. - - Click **Create**. - - Basic Tab: - - Select the subscription in which you want to create the storage account. - - Select an existing resource group or create a new one. - - Enter a unique name for the storage account. - - Select the same region for the storage account that you selected for the server. - - Choose any performance type. - - Select **Geo-redundant(GRS) Redundancy configuration**. - - Select **Make read access to data**. - - Click **Next:Advance**. - - Advanced tab: - - **Require secure transfer** should already be selected. - - **Allow enabling public access** should already be selected. - - **Enable storage account key access** should already be selected. - - Select the latest TLS version. - - Permitted scope should display the default value (from any storage account). - - The remaining parameters (Hierarchical Namespace, Access protocols, Blob storage, and Azure Files) should display the default values provided by Azure. - - Click **Next:Networking**. - - Networking tab: - - Enable public access from all networks for **Network access**. - - Select **Microsoft network routing** for **Routing preference**. - - Click **Next:Data protection**. - - Data protection tab: - - Keep the default values provided by Azure. - - Click **Next:Encryption**. - - Encryption tab: - - **Encryption type** should already be set to **Microsoft-managed key(MMK)**. - - **Enable support for customer-managed keys** should be set to the default value (**blobs and files**). - - By default, **Infrastructure encryption** should not be enabled. - - Click **Next:Tags**. - - For the Tags tab, make no changes and click **Next:Review**. - - Click **Create** after you review all the parameters. +5. Connection string for an event hub: + - In the list of event hubs, select your event hub. + - On the Event Hubs instance page, go to **Settings** > **Shared access policies** > **Add**. + - Name the policy, click **manage** to provide permissions, and create the policy. + - Select Connection string–primary key string from policy (it would be required in input plugin). + +6. Azure Storage Accounts Creation: + - Login to https://portal.azure.com. + - Search Storage accounts in search bar. + - Click **Create**. + - Basic Tab: + - Select the subscription in which you want to create the storage account. + - Select an existing resource group or create a new one. + - Enter a unique name for the storage account. + - Select the same region for the storage account that you selected for the server. + - Choose any performance type. + - Select **Geo-redundant(GRS) Redundancy configuration**. + - Select **Make read access to data**. + - Click **Next:Advance**. + - Advanced tab: + - **Require secure transfer** should already be selected. + - **Allow enabling public access** should already be selected. + - **Enable storage account key access** should already be selected. + - Select the latest TLS version. + - Permitted scope should display the default value (from any storage account). + - The remaining parameters (Hierarchical Namespace, Access protocols, Blob storage, and Azure Files) should display the default values provided by Azure. + - Click **Next:Networking**. + - Networking tab: + - Enable public access from all networks for **Network access**. + - Select **Microsoft network routing** for **Routing preference**. + - Click **Next:Data protection**. + - Data protection tab: + - Keep the default values provided by Azure. + - Click **Next:Encryption**. + - Encryption tab: + - **Encryption type** should already be set to **Microsoft-managed key(MMK)**. + - **Enable support for customer-managed keys** should be set to the default value (**blobs and files**). + - By default, **Infrastructure encryption** should not be enabled. + - Click **Next:Tags**. + - For the Tags tab, make no changes and click **Next:Review**. + - Click **Create** after you review all the parameters. + ### Link event hub to Databricks @@ -98,24 +99,28 @@ The plug-in is free and open-source (Apache 2.0). It can be used as a starting p 2. Navigate to your Azure Databricks. Open the Diagnostic settings pane under the Monitoring section. 3. After the page opens, you will need to create a new diagnostic setting. 4. In the Diagnostic settings pane, fill in the form with your preferred categories. -5. Select your categories details, and then send your logs to your preferred destination, in this case, we check **Stream to an event hub**, and put prefered event hub information in. +5. Select your categories details, and then send your logs to your preferred destination, in this case, we check **Stream to an event hub** and **Archive to a storage account**, and put prefered event hub and Storage account information in. 6. Launch your Databricks Workspace and go to profile at top right corner. 7. click ```Settings```, go to ```Advanced```, search for ```Verbose Audit Logs``` and turn it on. -## 4. Connecting to the Azure Databricks +## 2. Connecting to the Azure Databricks ### Insert/Update data through Data Explorer 1. Login to https://portal.azure.com. 2. Navigate to your Azure Databricks. Launch Workspace. 3. Under **SQL Editor**, you can run sql command by creating new quert scripts. -## 5. Limitations +## 3. Limitations 1. The following important fields couldn't be mapped with Databricks audit logs: - The following field are not found in original audit log from Azure Databricks: Database name, ProtocolVersion, AppUserName, Client mac, Common Protocol, Os User, ClientOs, ServerOs. -2. The log with sql excution will not have client ip, but it will come with another log with action name of "commandFinish". + The following fields are not found in original audit log from Azure Databricks: Database name, ProtocolVersion, AppUserName, Client mac, Common Protocol, Os User, ClientOs, ServerOs. +2. The log with sql execution will not have client ip, but it will come with another log with action name of "commandFinish". +3. The eventhub takes 10~30 minutes to receive raw logs from Databricks, the same delay time for Guardium is expected. +4. If queries are submitted as part of a notebook cell, job, or script, Databricks may log the entire execution context (e.g., the notebook run or job task) rather than each individual SQL query. In this case, Guardium will not be able to form separate records and only parse the first statement. +5. The Databricks auditing does not audit authentication failure(Login Failed) operations. + -## 6. Configuring the Azure-Databricks filter in Guardium +## 4. Configuring the Azure-Databricks filter in Guardium The Guardium universal connector is the Guardium entry point for native audit logs. The Guardium universal connector identifies and parses the received events, and converts them to a standard Guardium format. The output of the Guardium universal connector is forwarded to the Guardium sniffer on the collector, for policy and auditing enforcements. Configure Guardium to read the native audit logs by customizing the Azure-Databricks template. ### Before you begin @@ -129,7 +134,7 @@ Azure-Databricks-Guardium Logstash filter plug-in is automatically available wit ### Configuration 1. On the collector, go to ```Setup``` > ```Tools and Views``` > ```Configure Universal Connector```. 2. Enable the universal connector if it is disabled. -3. Click ```Upload File``` and select the offline [logstash-filter-databricks_guardium_filter.zip](../../filter-plugin/logstash-filter-databricks-guardium/logstash-filter-databricks_guardium_filter.zip) +3. Click ```Upload File``` and select the offline [logstash-filter-databricks_guardium_filter-1.0.1.zip](https://github.com/IBM/universal-connectors/releases/download/v1.7.0/logstash-filter-databricks_guardium_filter.zip) plug-in. After it is uploaded, click ```OK```. 4. Click the Plus sign to open the Connector Configuration dialog box. 5. Type a name in the ```Connector name``` field. diff --git a/filter-plugin/logstash-filter-databricks-guardium/VERSION b/filter-plugin/logstash-filter-databricks-guardium/VERSION index afaf360d3..7f207341d 100644 --- a/filter-plugin/logstash-filter-databricks-guardium/VERSION +++ b/filter-plugin/logstash-filter-databricks-guardium/VERSION @@ -1 +1 @@ -1.0.0 \ No newline at end of file +1.0.1 \ No newline at end of file diff --git a/filter-plugin/logstash-filter-databricks-guardium/build.gradle b/filter-plugin/logstash-filter-databricks-guardium/build.gradle index a446c391f..b6c1ab186 100644 --- a/filter-plugin/logstash-filter-databricks-guardium/build.gradle +++ b/filter-plugin/logstash-filter-databricks-guardium/build.gradle @@ -2,6 +2,30 @@ import java.nio.file.Files import static java.nio.file.StandardCopyOption.REPLACE_EXISTING apply plugin: 'java' + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + + dependencies { + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' + classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.1.0" + classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' + } +} + +def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + apply plugin: 'jacoco' apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" apply plugin: "eclipse" @@ -23,10 +47,10 @@ pluginInfo.pluginClass = "DatabricksGuardiumFilter" pluginInfo.pluginName = "databricks_guardium_filter" // must match the @LogstashPlugin annotation in the main plugin class // =========================================================================== -sourceCompatibility = 1.8 -targetCompatibility = 1.8 +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 -def jacocoVersion = '0.8.4' +def jacocoVersion = '0.8.11' // minimumCoverage can be set by Travis ENV def minimumCoverageStr = System.getenv("MINIMUM_COVERAGE") ?: "50.0%" if (minimumCoverageStr.endsWith("%")) { @@ -34,27 +58,16 @@ if (minimumCoverageStr.endsWith("%")) { } def minimumCoverage = Float.valueOf(minimumCoverageStr) / 100 -buildscript { - repositories { - maven { - url "https://plugins.gradle.org/m2/" - } - mavenCentral() - jcenter() - } - - dependencies { - classpath 'com.github.jengelman.gradle.plugins:shadow:4.0.4' - classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.0.0-RC2" - classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' - } -} - -def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); -def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) repositories { - mavenCentral() + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } } tasks.register("vendor"){ @@ -65,14 +78,13 @@ tasks.register("vendor"){ File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") projectJarFile.mkdirs() Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) - validatePluginJar(projectJarFile, project.group) } } apply plugin: 'com.github.johnrengelman.shadow' shadowJar { - classifier = null + archiveClassifier = null } @@ -80,6 +92,7 @@ dependencies { implementation 'com.google.code.gson:gson:' + versions.dependencies.gson implementation 'org.apache.commons:commons-lang3:' + versions.dependencies.commonsLang implementation 'commons-validator:commons-validator:' + versions.dependencies.commonsValidator + implementation group: 'commons-beanutils', name: 'commons-beanutils', version: versions.dependencies.commonsBeanutils implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.17.2' implementation group: 'com.github.jsqlparser', name: 'jsqlparser', version: '5.2' testImplementation group: 'org.mockito', name: 'mockito-all', version: versions.dependencies.mockitoAll @@ -150,17 +163,16 @@ apply plugin: "org.barfuin.gradle.jacocolog" // ------------------------------------ jacoco { toolVersion = "${jacocoVersion}" - reportsDir = file("$buildDir/reports/jacoco") } jacocoTestReport { // You will see "Report -> file://...." at the end of a JaCoCo build // If no output, run this first: ./gradlew test reports { - html.enabled true - xml.enabled true - csv.enabled true - html.destination file("${buildDir}/reports/jacoco") - csv.destination file("${buildDir}/reports/jacoco/all.csv") + html.required = true + xml.required = true + csv.required = true + html.outputLocation = file("${buildDir}/reports/jacoco") + csv.outputLocation = file("${buildDir}/reports/jacoco/all.csv") } executionData.from fileTree(dir: "${buildDir}/jacoco/", includes: [ '**/*.exec' diff --git a/filter-plugin/logstash-filter-databricks-guardium/gradle/wrapper/gradle-wrapper.properties b/filter-plugin/logstash-filter-databricks-guardium/gradle/wrapper/gradle-wrapper.properties index aa991fcea..81aa1c044 100644 --- a/filter-plugin/logstash-filter-databricks-guardium/gradle/wrapper/gradle-wrapper.properties +++ b/filter-plugin/logstash-filter-databricks-guardium/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/filter-plugin/logstash-filter-databricks-guardium/mainREADME.md b/filter-plugin/logstash-filter-databricks-guardium/mainREADME.md new file mode 100644 index 000000000..b6cc815e3 --- /dev/null +++ b/filter-plugin/logstash-filter-databricks-guardium/mainREADME.md @@ -0,0 +1,10 @@ +# Databricks Universal Connector + +## Follow this link to set up and use Databricks Universal Connector over Azure Eventhub Logstash Plugin + +[DatabricksOverEventHub](./README.md) + +## Follow this link to set up and use Databricks Universal Connector over Azure Eventhub Connect + +[DatabricksOverConnectEventHub](../../docs/KafkaBasedUCs/AzureDatabricksEventHubKafkaConnect.md) + diff --git a/filter-plugin/logstash-filter-databricks-guardium/src/main/java/com/ibm/guardium/databricks/Parser.java b/filter-plugin/logstash-filter-databricks-guardium/src/main/java/com/ibm/guardium/databricks/Parser.java index bc53bf45c..f4976c17a 100644 --- a/filter-plugin/logstash-filter-databricks-guardium/src/main/java/com/ibm/guardium/databricks/Parser.java +++ b/filter-plugin/logstash-filter-databricks-guardium/src/main/java/com/ibm/guardium/databricks/Parser.java @@ -64,11 +64,12 @@ public static Record parseRecord(final JsonObject records) { String accountId = getAccountId(records);// second part of resourceId String sessionId = getSessionId(requestParams); String serviceName = getServiceName(properties); + String dbName = subId+":"+serviceName; record.setSessionId(sessionId); - record.setDbName(Constants.UNKNOWN_STRING); + record.setDbName(dbName); record.setAppUserName(Constants.UNKNOWN_STRING); record.setAccessor(parseAccessor(subId, accountId, properties, records)); - record.getAccessor().setServiceName(serviceName); + record.getAccessor().setServiceName(dbName); record.setSessionLocator(parserSessionLocator(properties)); String response = properties.get(Constants.RESPONSE).toString(); @@ -90,7 +91,7 @@ public static Record parseRecord(final JsonObject records) { /** * Method to get queryStatement from JsonObject - * + * * @param properties * @return */ @@ -165,6 +166,24 @@ private static String getSubscriptionId(JsonObject records) { return subId; } + /** + * Method to get InstanceName from the JsonObject + * + * @param records + * @return + */ + private static String getInstanceName(JsonObject records) { + String instanceName = Constants.UNKNOWN_STRING; + if (records.has(Constants.RESOURCEID)) { + instanceName = records.get(Constants.RESOURCEID).getAsString(); + if (instanceName.contains("/")) { + String[] stringArr =instanceName.split("/"); + instanceName = stringArr[stringArr.length-1]; + } + } + return instanceName; + } + /** * Method to get accountId from the JsonObject * @@ -260,7 +279,7 @@ private static Construct parseConstruct(String fullsqlString) { /** * Method to get queryStatement from JsonObject - * + * * @param commandText * @return */ @@ -346,11 +365,10 @@ static Accessor parseAccessor(String subId, String accountId, JsonObject propert clientHostName = properties.get(Constants.SOURCE_IP).getAsString(); } accessor.setClientHostName(clientHostName); - accessor.setServerHostName( !subId.isEmpty() && !accountId.isEmpty() - ? subId.concat("-").concat(accountId).concat("azuredatabricks.net") - : "databricks.net"); + ? subId+":"+getInstanceName(record) + : Constants.UNKNOWN_STRING); // Set database user @@ -365,13 +383,10 @@ static Accessor parseAccessor(String subId, String accountId, JsonObject propert accessor.setServerType(Constants.SERVER_TYPE); accessor.setDbProtocol(Constants.DATA_PROTOCOL); accessor.setDbProtocolVersion(Constants.UNKNOWN_STRING); - accessor.setServiceName(properties.has(Constants.SERVICE_NAME) - ? properties.get(Constants.SERVICE_NAME).toString() - : Constants.UNKNOWN_STRING); // Set source program (user agent) accessor.setSourceProgram(properties.has(Constants.USER_AGENT) - ? properties.get(Constants.USER_AGENT).toString() + ? (properties.get(Constants.USER_AGENT).toString().isEmpty()||properties.get(Constants.USER_AGENT).toString().contains("\"")?Constants.UNKNOWN_STRING:properties.get(Constants.USER_AGENT).toString()) : Constants.UNKNOWN_STRING); // Set server description diff --git a/filter-plugin/logstash-filter-databricks-guardium/src/test/java/com/ibm/guardium/databricks/ParserTest.java b/filter-plugin/logstash-filter-databricks-guardium/src/test/java/com/ibm/guardium/databricks/ParserTest.java index 4d4234249..a8d00b75a 100644 --- a/filter-plugin/logstash-filter-databricks-guardium/src/test/java/com/ibm/guardium/databricks/ParserTest.java +++ b/filter-plugin/logstash-filter-databricks-guardium/src/test/java/com/ibm/guardium/databricks/ParserTest.java @@ -23,7 +23,7 @@ void testParseServiceName() { final String DatabricksString = "{ \"resourceId\": \"/SUBSCRIPTIONS/5C0C81D4-656F-415D-8599-DCD86F2F665E/RESOURCEGROUPS/DATABRICKSTEST/PROVIDERS/MICROSOFT.DATABRICKS/WORKSPACES/DATABRICK-TEST\", \"operationVersion\": \"1.0.0\", \"identity\": \"{\\\"email\\\":\\\"abc.h@abc.com\\\",\\\"subjectName\\\":null}\", \"operationName\": \"Microsoft.Databricks/accounts/tokenLogin\", \"time\": \"2025-05-01T13:43:28Z\", \"category\": \"accounts\", \"properties\": {\"sourceIPAddress\":\"20.193.136.102\",\"logId\":\"87e1a69e-444a-434e-a0be-648b1797e6d9\",\"serviceName\":\"accounts\",\"userAgent\":\"Apache-HttpClient/4.5.14 (Java/17.0.13) Databricks-Service/driver DBHttpClient/v2RawClient\",\"response\":\"{\\\"statusCode\\\":200}\",\"sessionId\":null,\"actionName\":\"tokenLogin\",\"requestId\":\"0018d88c-2f52-4cf0-86a4-8d1dc416ab10\",\"requestParams\":\"{\\\"user\\\":\\\"abc@abc.com\\\",\\\"tokenId\\\":\\\"kfhjgjfhdgjkdh39284783297423943hejhfkdsfh39\\\",\\\"authenticationMethod\\\":\\\"API_INT_PAT_TOKEN\\\"}\"}, \"Host\": \"1234-123456-ab1c2d3e-12-123-12-1\"}"; final JsonObject DatabricksJson = JsonParser.parseString(DatabricksString).getAsJsonObject(); Record record = Parser.parseRecord(DatabricksJson); - assertEquals("accounts", record.getAccessor().getServiceName()); + assertEquals("5C0C81D4-656F-415D-8599-DCD86F2F665E:accounts", record.getAccessor().getServiceName()); } diff --git a/filter-plugin/logstash-filter-documentdb-aws-guardium/CHANGELOG.md b/filter-plugin/logstash-filter-documentdb-aws-guardium/CHANGELOG.md index 2d6744e09..a485d7286 100644 --- a/filter-plugin/logstash-filter-documentdb-aws-guardium/CHANGELOG.md +++ b/filter-plugin/logstash-filter-documentdb-aws-guardium/CHANGELOG.md @@ -1,6 +1,14 @@ # Changelog Notable changes will be documented in this file. +## [0.1.24] +- GRD-126808: Add a support for MongoStyle audit logs + +## [0.1.23] +- Code improvement +- Remove session id +- Client port to -1 + ## [0.1.22] - SQL query will not appear in Object and Verb column. - Service name and Database name should be identical. diff --git a/filter-plugin/logstash-filter-documentdb-aws-guardium/DocumentDBOverCloudwatchPackage/DocumentDB/filter.conf b/filter-plugin/logstash-filter-documentdb-aws-guardium/DocumentDBOverCloudwatchPackage/DocumentDB/filter.conf index 061cd3674..042fa0662 100644 --- a/filter-plugin/logstash-filter-documentdb-aws-guardium/DocumentDBOverCloudwatchPackage/DocumentDB/filter.conf +++ b/filter-plugin/logstash-filter-documentdb-aws-guardium/DocumentDBOverCloudwatchPackage/DocumentDB/filter.conf @@ -1,9 +1,13 @@ filter { -if [type] == "docdb" { -mutate { -replace => { "serverHostnamePrefix" => "%{account_id}_%{[cloudwatch_logs][log_stream]}" } -replace => { "event_id" => "%{[cloudwatch_logs][event_id]}" } -} -documentdb_guardium_filter {} -} +if [type] == "docdb" + { + mutate { add_field => { "log_group" => "%{[cloudwatch_logs][log_group]}" } } + grok { match => { "log_group" => "^/aws/docdb/%{DATA:cluster}/%{DATA:logtype}$" } } + mutate { + replace => { "serverHostnamePrefix" => "%{account_id}_%{cluster}" } + replace => { "dbnamePrefix" => "%{account_id}:%{[cloudwatch_logs][log_stream]}" } + replace => { "event_id" => "%{[cloudwatch_logs][event_id]}" } + } + documentdb_guardium_filter {} + } } \ No newline at end of file diff --git a/filter-plugin/logstash-filter-documentdb-aws-guardium/DocumentDBOverCloudwatchPackage/logstash-filter-documentdb_guardium_filter.zip b/filter-plugin/logstash-filter-documentdb-aws-guardium/DocumentDBOverCloudwatchPackage/logstash-filter-documentdb_guardium_filter.zip new file mode 100644 index 000000000..0baef5dfe Binary files /dev/null and b/filter-plugin/logstash-filter-documentdb-aws-guardium/DocumentDBOverCloudwatchPackage/logstash-filter-documentdb_guardium_filter.zip differ diff --git a/filter-plugin/logstash-filter-documentdb-aws-guardium/README.md b/filter-plugin/logstash-filter-documentdb-aws-guardium/README.md index 89e5a1f37..a602d8bd8 100644 --- a/filter-plugin/logstash-filter-documentdb-aws-guardium/README.md +++ b/filter-plugin/logstash-filter-documentdb-aws-guardium/README.md @@ -100,6 +100,9 @@ The Guardium universal connector is the Guardium entry point for native audit/pr 10. After the offline plug-in is installed and the configuration is uploaded and saved in the Guardium machine, restart the Universal Connector using the ```Disable/Enable``` button. ## 6. Limitations +- Due to native limitations in Amazon DocumentDB, audit logs may truncate query details (1 KB limit), and profiler logs only capture slow queries (based on a configurable threshold, around 50 ms). As a result, full query visibility cannot be guaranteed. + - https://docs.aws.amazon.com/documentdb/latest/developerguide/event-auditing.html + - https://docs.aws.amazon.com/documentdb/latest/developerguide/profiling.html - DocumentDB Profiler logs capture any database operations that take longer than some period of time(e. g. 100 ms). If the threshold value is not configurable and set value is too high, then profiler logs may not get captured for every database operation. - The Following important fields couldn't be mapped with DocumentDB audit/profiler logs - Source program : Only available in case of "aggregate" query @@ -107,6 +110,7 @@ The Guardium universal connector is the Guardium entry point for native audit/pr - Client HostName : Not available with Audit/Profier logs - Server IPs are also not reported because they are not part of the audit stream. That said, the "add_field" clause in the configuration adds a user defined Server Host Name that can be used in reports and policies if desired. - Because Sniffer saves the DB name once when a new session is created, and not with every event, DB name will be updated and populated correctly in Guardium only when everytime a new database connection is established with database name. If Database connection is established without database name, then the database on which the first query for that session runs, will be retained in Guardium. Even if user switches between the databases for the same session. +- Sql Errors are not supported. ## Configuring the DocumentDB Guardium Logstash filters in Guardium Data Security Center diff --git a/filter-plugin/logstash-filter-documentdb-aws-guardium/VERSION b/filter-plugin/logstash-filter-documentdb-aws-guardium/VERSION index 915812ec9..b9fe11d92 100644 --- a/filter-plugin/logstash-filter-documentdb-aws-guardium/VERSION +++ b/filter-plugin/logstash-filter-documentdb-aws-guardium/VERSION @@ -1 +1 @@ -0.1.22 \ No newline at end of file +0.1.24 \ No newline at end of file diff --git a/filter-plugin/logstash-filter-documentdb-aws-guardium/build.gradle b/filter-plugin/logstash-filter-documentdb-aws-guardium/build.gradle index 794a07a6b..5e1004291 100644 --- a/filter-plugin/logstash-filter-documentdb-aws-guardium/build.gradle +++ b/filter-plugin/logstash-filter-documentdb-aws-guardium/build.gradle @@ -2,6 +2,30 @@ import java.nio.file.Files import static java.nio.file.StandardCopyOption.REPLACE_EXISTING apply plugin: 'java' + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + + dependencies { + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' + classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.1.0" + classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' + } +} + +def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" // =========================================================================== @@ -20,10 +44,10 @@ pluginInfo.pluginClass = "DocumentdbGuardiumFilter" pluginInfo.pluginName = "documentdb_guardium_filter" // must match the @LogstashPlugin annotation in the main plugin class // =========================================================================== -sourceCompatibility = 1.8 -targetCompatibility = 1.8 +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 -def jacocoVersion = '0.8.4' +def jacocoVersion = '0.8.11' // minimumCoverage can be set by Travis ENV def minimumCoverageStr = System.getenv("MINIMUM_COVERAGE") ?: "50.0%" if (minimumCoverageStr.endsWith("%")) { @@ -31,27 +55,16 @@ if (minimumCoverageStr.endsWith("%")) { } def minimumCoverage = Float.valueOf(minimumCoverageStr) / 100 -buildscript { - repositories { - maven { - url "https://plugins.gradle.org/m2/" - } - mavenCentral() - jcenter() - } - - dependencies { - classpath 'com.github.jengelman.gradle.plugins:shadow:4.0.4' - classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.0.0-RC2" - classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' - } -} - -def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); -def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) repositories { - mavenCentral() + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } } tasks.register("vendor"){ @@ -62,14 +75,13 @@ tasks.register("vendor"){ File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") projectJarFile.mkdirs() Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) - validatePluginJar(projectJarFile, project.group) } } apply plugin: 'com.github.johnrengelman.shadow' shadowJar { - classifier = null + archiveClassifier = null } dependencies { @@ -77,6 +89,7 @@ dependencies { implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: versions.dependencies.log4jCore implementation 'org.apache.commons:commons-lang3:' + versions.dependencies.commonsLang implementation 'com.google.code.gson:gson:' + versions.dependencies.gson + implementation group: 'commons-beanutils', name: 'commons-beanutils', version: versions.dependencies.commonsBeanutils implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "build/libs/logstash-core-*.*.*.jar") implementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "common-*.*.*.jar") @@ -107,6 +120,17 @@ tasks.withType(JavaCompile) { options.encoding = 'UTF-8' } +test { + jvmArgs = [ + '--add-opens=java.base/java.lang=ALL-UNNAMED', + '--add-opens=java.base/java.util=ALL-UNNAMED', + '--add-opens=java.base/java.text=ALL-UNNAMED', + '--add-opens=java.base/java.lang.reflect=ALL-UNNAMED', + '--add-opens=java.base/sun.util.resources=ALL-UNNAMED', + '--add-opens=java.base/sun.util.cldr=ALL-UNNAMED' + ] +} + tasks.register("generateRubySupportFiles") { @@ -149,17 +173,16 @@ apply plugin: "org.barfuin.gradle.jacocolog" // ------------------------------------ jacoco { toolVersion = "${jacocoVersion}" - reportsDir = file("$buildDir/reports/jacoco") } jacocoTestReport { // You will see "Report -> file://...." at the end of a JaCoCo build // If no output, run this first: ./gradlew test reports { - html.enabled true - xml.enabled true - csv.enabled true - html.destination file("${buildDir}/reports/jacoco") - csv.destination file("${buildDir}/reports/jacoco/all.csv") + html.required = true + xml.required = true + csv.required = true + html.outputLocation = file("${buildDir}/reports/jacoco") + csv.outputLocation = file("${buildDir}/reports/jacoco/all.csv") } executionData.from fileTree(dir: "${buildDir}/jacoco/", includes: [ '**/*.exec' diff --git a/filter-plugin/logstash-filter-documentdb-aws-guardium/documentDBCloudwatch.conf b/filter-plugin/logstash-filter-documentdb-aws-guardium/documentDBCloudwatch.conf index afc79582f..3923740b1 100644 --- a/filter-plugin/logstash-filter-documentdb-aws-guardium/documentDBCloudwatch.conf +++ b/filter-plugin/logstash-filter-documentdb-aws-guardium/documentDBCloudwatch.conf @@ -19,21 +19,24 @@ secret_access_key => event_filter => '' -#Insert the account id of the AWS account +#Insert the account id and cluster name of the AWS account add_field => {"account_id" => ""} +add_field => {"cluster" => ""} type => "docdb" } } filter { -if [type] == "docdb" { -mutate { -replace => { "serverHostnamePrefix" => "%{account_id}_%{[cloudwatch_logs][log_stream]}" } -replace => { "dbnamePrefix" => "%{account_id}:%{[cloudwatch_logs][log_stream]}" } -replace => { "event_id" => "%{[cloudwatch_logs][event_id]}" } -} -documentdb_guardium_filter {} -} + if [type] == "docdb" { + mutate { add_field => { "log_group" => "%{[cloudwatch_logs][log_group]}" } } + grok { match => { "log_group" => "^/aws/docdb/%{DATA:cluster}/%{DATA:logtype}$" } } + mutate { + replace => { "serverHostnamePrefix" => "%{account_id}_%{cluster}" } + replace => { "dbnamePrefix" => "%{account_id}:%{[cloudwatch_logs][log_stream]}" } + replace => { "event_id" => "%{[cloudwatch_logs][event_id]}" } + } + documentdb_guardium_filter {} + } } output { stdout { codec => rubydebug } diff --git a/filter-plugin/logstash-filter-documentdb-aws-guardium/gradle/wrapper/gradle-wrapper.properties b/filter-plugin/logstash-filter-documentdb-aws-guardium/gradle/wrapper/gradle-wrapper.properties index 097ff3d3c..122a8692a 100644 --- a/filter-plugin/logstash-filter-documentdb-aws-guardium/gradle/wrapper/gradle-wrapper.properties +++ b/filter-plugin/logstash-filter-documentdb-aws-guardium/gradle/wrapper/gradle-wrapper.properties @@ -1,7 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/filter-plugin/logstash-filter-documentdb-aws-guardium/mainREADME.md b/filter-plugin/logstash-filter-documentdb-aws-guardium/mainREADME.md new file mode 100644 index 000000000..8a93ced46 --- /dev/null +++ b/filter-plugin/logstash-filter-documentdb-aws-guardium/mainREADME.md @@ -0,0 +1,10 @@ +# DocumentDB Universal Connector + +## Follow this link to set up and use DocumentDB Universal Connector over CloudWatch Logstash Plugin + +[DocumentDBOverCloudwatch](./README.md) + +## Follow this link to set up and use DocumentDB Universal Connector over CloudWatch Connect + +[DocumentDBOverConnectCloudwatch](../../docs/KafkaBasedUCs/DocumentDBCloudwatchKafkaConnect.md) + diff --git a/filter-plugin/logstash-filter-documentdb-aws-guardium/src/main/java/com/ibm/guardium/documentdb/Constants.java b/filter-plugin/logstash-filter-documentdb-aws-guardium/src/main/java/com/ibm/guardium/documentdb/Constants.java new file mode 100644 index 000000000..89f8bd29a --- /dev/null +++ b/filter-plugin/logstash-filter-documentdb-aws-guardium/src/main/java/com/ibm/guardium/documentdb/Constants.java @@ -0,0 +1,132 @@ +/* +Copyright 2022-2023 IBM Inc. All rights reserved +SPDX-License-Identifier: Apache-2.0 +*/ +package com.ibm.guardium.documentdb; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +/** + * Central repository for all constants used in DocumentDB Guardium filter plugin. + * This class consolidates magic strings, numbers, and configuration values to improve + * maintainability and reduce duplication. + */ +public final class Constants { + + // Prevent instantiation + private Constants() { + throw new AssertionError("Constants class should not be instantiated"); + } + + // Protocol and Server Type + public static final String DATA_PROTOCOL = "DocumentDB"; + public static final String SERVER_TYPE = "DocumentDB"; + + // Default Values + public static final String UNKNOWN_STRING = ""; + public static final String NOT_AVAILABLE = "N.A."; + public static final String DEFAULT_IP = "0.0.0.0"; + public static final String DEFAULT_IPV6 = "0000:0000:0000:0000:0000:FFFF:0000:0000"; + public static final String COMPOUND_OBJECT = "[json-object]"; + public static final String DOCUMENT_INTERNAL_API_IP = "(NONE)"; + + // Exception Types + public static final String EXCEPTION_TYPE_AUTHORIZATION = "SQL_ERROR"; + public static final String EXCEPTION_TYPE_AUTHENTICATION = "LOGIN_FAILED"; + public static final String UC_PARSER_ERROR = "UC_PARSER_ERROR"; + public static final String UC_AUDIT_ERROR = "UC_AUDIT_ERROR"; + + // Logstash Tags + public static final String LOGSTASH_TAG_JSON_PARSE_ERROR = "_documentdbguardium_json_parse_error"; + public static final String LOGSTASH_TAG_JSON_DEPTH_ERROR = "_documentdbguardium_json_depth_error"; + public static final String LOGSTASH_TAG_SKIP = "_documentdbguardium_skip"; + + // DocumentDB Event Signals + public static final String DOCUMENTDB_AUDIT_SIGNAL = "\"atype\""; + public static final String DOCUMENTDB_PROFILER_SIGNAL = "command"; + + // DocumentDB Operation Keys + public static final String AGGR_KEY = "aggregate"; + public static final String COUNT_KEY = "count"; + public static final String DELETE_KEY = "remove"; + public static final String INSERT_KEY = "insert"; + public static final String UPDATE_KEY = "update"; + public static final String DISTINCT_KEY = "distinct"; + public static final String FIND_KEY = "find"; + public static final String FINDANDMODIFY_KEY = "findAndModify"; + + // Profiler Keys Set for efficient checking (immutable) + public static final Set PROFILER_KEYS = Collections.unmodifiableSet(new HashSet<>( + Arrays.asList( + AGGR_KEY, + COUNT_KEY, + DELETE_KEY, + INSERT_KEY, + UPDATE_KEY, + DISTINCT_KEY, + FIND_KEY, + FINDANDMODIFY_KEY + ) + )); + + // Local IP addresses (immutable) + public static final Set LOCAL_IP_LIST = Collections.unmodifiableSet(new HashSet<>( + Arrays.asList("127.0.0.1", "0:0:0:0:0:0:0:1") + )); + + // Redaction ignore strings - arguments that won't be redacted (immutable) + public static final Set REDACTION_IGNORE_STRINGS = Collections.unmodifiableSet(new HashSet<>( + Arrays.asList("from", "localField", "foreignField", "as", "connectFromField", "connectToField") + )); + + // JSON Field Names + public static final String FIELD_ATYPE = "atype"; + public static final String FIELD_PARAM = "param"; + public static final String FIELD_NS = "ns"; + public static final String FIELD_USER = "user"; + public static final String FIELD_ERROR = "error"; + public static final String FIELD_MESSAGE = "message"; + public static final String FIELD_COMMAND = "command"; + public static final String FIELD_TS = "ts"; + public static final String FIELD_CLIENT = "client"; + public static final String FIELD_REMOTE_IP = "remote_ip"; + public static final String FIELD_APP_NAME = "appName"; + public static final String FIELD_OP = "op"; + + // Event Field Names + public static final String EVENT_FIELD_MESSAGE = "message"; + public static final String EVENT_FIELD_SERVER_IP = "server_ip"; + public static final String EVENT_FIELD_SERVER_HOSTNAME_PREFIX = "serverHostnamePrefix"; + + // Authentication Types + public static final String AUTH_TYPE_AUTHENTICATE = "authenticate"; + public static final String AUTH_TYPE_AUTHCHECK = "authCheck"; + public static final String AUTH_TYPE_CREATE_ROLE = "createRole"; + public static final String AUTH_TYPE_DROP_ROLE = "dropRole"; + + // Error Codes + public static final String ERROR_CODE_SUCCESS = "0"; + public static final String ERROR_CODE_UNAUTHORIZED = "13"; + public static final String ERROR_CODE_AUTH_FAILED = "18"; + + // Server Configuration + public static final String SERVER_HOSTNAME_SUFFIX = ".aws.com"; + public static final String DEFAULT_SERVER_HOSTNAME = "documentdb.amazonaws.com"; + + // Limits and Thresholds + public static final int MAX_SQL_STRING_LENGTH = 10000; + public static final int STRING_BUILDER_INITIAL_CAPACITY = 256; + + // Error Messages + public static final String ERROR_JSON_VALIDATION_FAILED = "DocumentDB filter: JSON validation failed (truncated or too large)"; + public static final String ERROR_INVALID_AUTHENTICATE_LOG = "DocumentDB filter: Invalid authenticate log"; + public static final String ERROR_JSON_NESTING_TOO_DEEP = "DocumentDB filter: JSON nesting too deep (StackOverflow), skipping event"; + public static final String ERROR_INSUFFICIENT_MEMORY = "DocumentDB filter: Insufficient memory to process event, skipping"; + public static final String ERROR_PARSING_AUDIT_EVENT = "DocumentDB filter: Error parsing docDb audit event"; + public static final String ERROR_PARSING_PROFILER_EVENT = "DocumentDB filter: Error parsing docDb profiler event"; + public static final String ERROR_MISSING_DB_NAME = "DocumentDB filter: Missing DB name"; + public static final String ERROR_FAILED_TO_SERIALIZE_EVENT = "{ error: failed to serialize event }"; +} diff --git a/filter-plugin/logstash-filter-documentdb-aws-guardium/src/main/java/com/ibm/guardium/documentdb/DocumentdbGuardiumFilter.java b/filter-plugin/logstash-filter-documentdb-aws-guardium/src/main/java/com/ibm/guardium/documentdb/DocumentdbGuardiumFilter.java index c07c5b431..53ff63a11 100644 --- a/filter-plugin/logstash-filter-documentdb-aws-guardium/src/main/java/com/ibm/guardium/documentdb/DocumentdbGuardiumFilter.java +++ b/filter-plugin/logstash-filter-documentdb-aws-guardium/src/main/java/com/ibm/guardium/documentdb/DocumentdbGuardiumFilter.java @@ -1,29 +1,21 @@ /* -© Copyright IBM Corp. 2021, 2022 All rights reserved. +Copyright 2022-2023 IBM Inc. All rights reserved SPDX-License-Identifier: Apache-2.0 */ package com.ibm.guardium.documentdb; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.HashSet; -import java.util.Map; -import java.util.Properties; -import java.util.Set; +import java.util.List; -import org.apache.commons.validator.routines.InetAddressValidator; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.JsonObject; -import com.google.gson.JsonParser; +import com.google.gson.JsonSyntaxException; import com.ibm.guardium.universalconnector.commons.GuardConstants; import com.ibm.guardium.universalconnector.commons.Util; import com.ibm.guardium.universalconnector.commons.structures.Record; @@ -37,230 +29,352 @@ import co.elastic.logstash.api.LogstashPlugin; import co.elastic.logstash.api.PluginConfigSpec; +/** + * DocumentDB Guardium Filter plugin for Logstash. Processes DocumentDB audit and profiler logs and + * converts them to Guardium Record format. + * + *

This refactored version uses utility classes and constants for better maintainability. + */ @LogstashPlugin(name = "documentdb_guardium_filter") public class DocumentdbGuardiumFilter implements Filter { - public static final PluginConfigSpec SOURCE_CONFIG = PluginConfigSpec.stringSetting("source", "message"); - private final static String DOCUMENTDB_AUDIT_SIGNAL = "\"atype\""; - private final static String DOCUMENTDB_PROFILER_SIGNAL = "command"; - private final static String AGGR_KEY="aggregate"; - private final static String COUNT_KEY="count"; - private final static String DELETE_KEY="remove"; - private final static String INSERT_KEY="insert"; - private final static String UPDATE_KEY="update"; - private final static String DISTINCT_KEY="distinct"; - private final static String FIND_KEY="find"; - private final static String FINDANDMODIFY_KEY="findAndModify"; - public static final String LOGSTASH_TAG_JSON_PARSE_ERROR = "_documentdbguardium_json_parse_error"; - private final static Set LOCAL_IP_LIST = new HashSet<>(Arrays.asList("127.0.0.1", "0:0:0:0:0:0:0:1")); - private final static String DOCUMENT_INTERNAL_API_IP = "(NONE)"; - /* - * skipping non-relevant log events like - * "successful authenticate", and other events - * with blank db name - */ - public static final String LOGSTASH_TAG_SKIP = "_documentdbguardium_skip"; - private static final InetAddressValidator inetAddressValidator = InetAddressValidator.getInstance(); - private static Logger log ; - private String id; - - public DocumentdbGuardiumFilter(String id, Configuration config, Context context) { - // constructors should validate configuration options - this.id = id; - } - - @Override - public Collection> configSchema() { - // should return a list of all configuration options for this plugin - return Collections.singletonList(SOURCE_CONFIG); - } - - @Override - public String getId() { - return this.id; - } - - /** - * Filter event to create Guard record object(s) for each Adit/Profiler event - */ - @Override - public Collection filter(Collection events, FilterMatchListener matchListener) { - if (log == null) { - // instantiate default logger, as other Context can not coexsit alongside Guardium Universal Connector Logger context. - log = LogManager.getLogger(DocumentdbGuardiumFilter.class); - } - String messageString=null; - ArrayList skippedEvents = new ArrayList<>(); - for (Event e : events) { - if (e.getField("message") instanceof String) { - messageString = e.getField("message").toString(); - - - if (messageString.contains(DOCUMENTDB_AUDIT_SIGNAL)) {// This is an audit event - try { - JsonObject inputJSON = new Gson().fromJson(messageString, JsonObject.class); - final String atype = inputJSON.get("atype").getAsString(); - final JsonObject param = inputJSON.get("param").getAsJsonObject(); - if ((atype.equals("authenticate") && param.get("error").getAsString().equals("0") ) || - (param.has("ns") && param.get("ns").getAsString().isEmpty())) - { - e.tag(LOGSTASH_TAG_SKIP); - skippedEvents.add(e); - continue; - } - Record record = Parser.parseAuditRecord(inputJSON); - if(e.getField("serverHostnamePrefix") !=null && e.getField("serverHostnamePrefix") instanceof String) { - record.getAccessor().setServerHostName(e.getField("serverHostnamePrefix").toString()+".aws.com"); - String dbName=record.getDbName(); - record.setDbName(!dbName.isEmpty()?e.getField("serverHostnamePrefix").toString()+":"+dbName:e.getField("serverHostnamePrefix").toString()); - } - record.getAccessor().setServiceName(record.getDbName()); - if(e.getField("event_id") !=null && e.getField("event_id") instanceof String) { - record.setSessionId(record.getSessionId()+e.getField("event_id").toString()); - } - this.correctIPs(e, record); - final GsonBuilder builder = new GsonBuilder(); - builder.serializeNulls(); - final Gson gson = builder.create(); - e.setField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME, gson.toJson(record)); - matchListener.filterMatched(e); // Flag OK for filter input/parsing/out - } catch (Exception exception) { - // don't let event pass filter - // events.remove(e); - log.error("DocumentDB filter: Error parsing docDb audit event " + logEvent(e), exception); - e.tag(LOGSTASH_TAG_JSON_PARSE_ERROR); - } - } else if (messageString.contains(DOCUMENTDB_PROFILER_SIGNAL)) {// This is a profiler event - try { - if(messageString.contains(AGGR_KEY) || messageString.contains(COUNT_KEY) || messageString.contains(DELETE_KEY) || messageString.contains(DISTINCT_KEY)|| messageString.contains(FIND_KEY)|| messageString.contains(FINDANDMODIFY_KEY)||messageString.contains(INSERT_KEY)||messageString.contains(UPDATE_KEY)) - { - JsonObject inputJSON = new Gson().fromJson(messageString, JsonObject.class); - if ((!inputJSON.has("ns")) || (inputJSON.has("ns") && inputJSON.get("ns").getAsString().isEmpty()) ) { - e.tag(LOGSTASH_TAG_SKIP); - skippedEvents.add(e); - continue; - } - Record record = Parser.parseProfilerRecord(inputJSON); - if(e.getField("serverHostnamePrefix") !=null && e.getField("serverHostnamePrefix") instanceof String) { - record.getAccessor().setServerHostName(e.getField("serverHostnamePrefix").toString()+".aws.com"); - String dbName=record.getDbName(); - record.setDbName(!dbName.isEmpty()?e.getField("serverHostnamePrefix").toString()+":"+dbName:e.getField("serverHostnamePrefix").toString()); - } - record.getAccessor().setServiceName(record.getDbName()); - if(e.getField("event_id") !=null && e.getField("event_id") instanceof String) { - record.setSessionId(record.getSessionId()+e.getField("event_id").toString()); - } - this.correctIPs(e, record); - final GsonBuilder builder = new GsonBuilder(); - builder.serializeNulls(); - final Gson gson = builder.disableHtmlEscaping().create(); - e.setField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME, gson.toJson(record)); - if (record.getDbName().equals(Parser.UNKOWN_STRING)) { - e.tag(LOGSTASH_TAG_SKIP); - skippedEvents.add(e); - continue; - } - matchListener.filterMatched(e); // Flag OK for filter input/parsing/out - } - }catch (Exception exception) { - // don't let event pass filter - // events.remove(e); - log.error("DocumentDB filter: Error parsing docDb profiler event " + logEvent(e), exception); - e.tag(LOGSTASH_TAG_JSON_PARSE_ERROR); - } - } - } - } - events.removeAll(skippedEvents); - return events; - } - - /** - * Overrides DocumentDB local/remote IP 127.0.0.1, if Logstash Event contains - * "server_ip". - * Override "(NONE)" IP, if not filterd, as it's internal command by DocumentDB. - * Note: IP needs to be in ipv4/ipv6 format - * - * @param e - Logstash Event - * @param record - Record after parsing. - */ - private void correctIPs(Event e, Record record) { - // Override "(NONE)" IP, if not filterd, as it's internal command by DocumentDB. - // Note: IP needs to be in ipv4/ipv6 format - SessionLocator sessionLocator = record.getSessionLocator(); - String sessionServerIp = sessionLocator.getServerIp(); - - if (isDocumentInternalCommandIp(sessionServerIp)) { - String ip = getValidatedEventServerIp(e); - if (ip != null) { - if (Util.isIPv6(ip)) { - sessionLocator.setServerIpv6(ip); - sessionLocator.setIpv6(true); - } else { - sessionLocator.setServerIp(ip); - sessionLocator.setIpv6(false); - } - } else if (sessionServerIp.equalsIgnoreCase(DOCUMENT_INTERNAL_API_IP)) { - sessionLocator.setServerIp("0.0.0.0"); - } - } - - if (isDocumentInternalCommandIp(sessionLocator.getClientIp())) { - if (sessionLocator.isIpv6()) { - sessionLocator.setClientIpv6(sessionLocator.getServerIpv6()); - } else { - sessionLocator.setClientIp(sessionLocator.getServerIp()); - } - } - } - - /** - * Validates server IP - * @param e - * @return - */ - private String getValidatedEventServerIp(Event e) { - if (e.getField("server_ip") instanceof String) { - String ip = e.getField("server_ip").toString(); - if (ip != null && inetAddressValidator.isValid(ip)) { - return ip; - } - } - return null; - } - - /** - * Checks if the IP address is local or remote, returns true/false in case of local address - * @param ip - * @return - */ - private boolean isDocumentInternalCommandIp(String ip) { - return ip != null && (LOCAL_IP_LIST.contains(ip) || ip.trim().equalsIgnoreCase(DOCUMENT_INTERNAL_API_IP)); - } - - /** - * Creates log string to be printed - * @param event - * @return - */ - private static String logEvent(Event event) { - StringBuffer sb = new StringBuffer(); - try { - sb.append("{ "); - boolean first = true; - for (Map.Entry stringObjectEntry : event.getData().entrySet()) { - if (!first) { - sb.append(","); - } - sb.append("\"" + stringObjectEntry.getKey() + "\" : \"" + stringObjectEntry.getValue() + "\""); - first = false; - } - sb.append(" }"); - return sb.toString(); - } catch (Exception e) { - log.error("DocumentDB filter: Failed to create event log string", e); - return null; - } - } + public static final PluginConfigSpec SOURCE_CONFIG = + PluginConfigSpec.stringSetting("source", "message"); + // Reuse Gson instances to avoid creating new ones for every event (Performance Optimization) + private static final Gson GSON_PARSER = new Gson(); + private static final Gson GSON_SERIALIZER = new GsonBuilder().serializeNulls().create(); + private static final Gson GSON_SERIALIZER_NO_ESCAPE = + new GsonBuilder().serializeNulls().disableHtmlEscaping().create(); + + private static final Logger log = LogManager.getLogger(DocumentdbGuardiumFilter.class); + + private final Parser parser; + private final String id; + private final List skippedEvents = new ArrayList<>(); + + public DocumentdbGuardiumFilter(String id, Configuration config, Context context) { + this.id = id; + this.parser = new Parser(); + } + + /** + * Formats JsonSyntaxException message, truncating everything from the troubleshooting link + * onwards. + * + * @param jse The JsonSyntaxException to format + * @return Formatted exception message + */ + private static String formatJsonSyntaxException(JsonSyntaxException jse) { + String message = jse.toString(); + int linkIndex = message.indexOf("See https://github.com/google/gson"); + if (linkIndex != -1) { + return message.substring(0, linkIndex).trim(); + } + return message; + } + + @Override + public Collection> configSchema() { + return Collections.singletonList(SOURCE_CONFIG); + } + + @Override + public String getId() { + return this.id; + } + + /** Filter event to create Guard record object(s) for each Audit/Profiler event. */ + @Override + public Collection filter(Collection events, FilterMatchListener matchListener) { + skippedEvents.clear(); // Clear from previous invocation + for (Event e : events) { + processEvent(e, matchListener); + } + events.removeAll(skippedEvents); + return events; + } + + /** + * Processes a single event and converts it to a Guardium record. + * + * @param e The event to process + * @param matchListener The filter match listener + */ + private void processEvent(Event e, FilterMatchListener matchListener) { + String messageString = EventUtils.getMessageField(e); + if (messageString == null) { + return; + } + + if (!ValidationUtils.isProperlyClosedJson(messageString)) { + handleInvalidJson(e, messageString, matchListener); + return; + } + + if (messageString.contains(Constants.DOCUMENTDB_AUDIT_SIGNAL)) { + processAuditEvent(e, messageString, matchListener); + } else if (messageString.contains(Constants.DOCUMENTDB_PROFILER_SIGNAL)) { + processProfilerEvent(e, messageString, matchListener); + } else { + } + } + + /** Handles invalid JSON by creating an exception record. */ + private void handleInvalidJson(Event e, String messageString, FilterMatchListener matchListener) { + String errorMsg = Constants.ERROR_JSON_VALIDATION_FAILED; + Record record = parser.parseRecordException(null, errorMsg, messageString); + updateEventWithException( + record, errorMsg, messageString, e, Constants.LOGSTASH_TAG_JSON_DEPTH_ERROR, matchListener); + } + + /** Processes an audit event. */ + private void processAuditEvent(Event e, String messageString, FilterMatchListener matchListener) { + try { + JsonObject inputJSON = GSON_PARSER.fromJson(messageString, JsonObject.class); + if (shouldSkipAuditEvent(inputJSON)) { + handleSkippedEvent(e, messageString, matchListener); + return; + } + Record record = parser.parseAuditRecord(inputJSON); + enrichRecordWithServerInfo(e, record); + correctIPs(e, record); + String recordJson = GSON_SERIALIZER.toJson(record); + e.setField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME, recordJson); + matchListener.filterMatched(e); + } catch (StackOverflowError soe) { + handleStackOverflowError(e, messageString, matchListener); + } catch (OutOfMemoryError oom) { + handleOutOfMemoryError(e, messageString, matchListener); + } catch (JsonSyntaxException jse) { + handleJsonSyntaxError(e, messageString, jse, matchListener, true); + } catch (Exception exception) { + handleGenericError(e, messageString, exception, matchListener, true); + } + } + + /** Processes a profiler event. */ + private void processProfilerEvent( + Event e, String messageString, FilterMatchListener matchListener) { + try { + if (!StringUtils.containsAnyProfilerKey(messageString)) { + return; + } + + JsonObject inputJSON = GSON_PARSER.fromJson(messageString, JsonObject.class); + + if (shouldSkipProfilerEvent(inputJSON)) { + handleSkippedEvent(e, messageString, matchListener); + return; + } + + Record record = parser.parseProfilerRecord(inputJSON); + enrichRecordWithServerInfo(e, record); + correctIPs(e, record); + + e.setField( + GuardConstants.GUARDIUM_RECORD_FIELD_NAME, GSON_SERIALIZER_NO_ESCAPE.toJson(record)); + + if (record.getDbName().equals(Constants.UNKNOWN_STRING)) { + handleMissingDbName(e, messageString, record, matchListener); + return; + } + + matchListener.filterMatched(e); + + } catch (StackOverflowError soe) { + handleStackOverflowError(e, messageString, matchListener); + } catch (OutOfMemoryError oom) { + handleOutOfMemoryError(e, messageString, matchListener); + } catch (JsonSyntaxException jse) { + handleJsonSyntaxError(e, messageString, jse, matchListener, false); + } catch (Exception exception) { + handleGenericError(e, messageString, exception, matchListener, false); + } + } + + /** Checks if an audit event should be skipped. */ + private boolean shouldSkipAuditEvent(JsonObject inputJSON) { + final String atype = inputJSON.get(Constants.FIELD_ATYPE).getAsString(); + // Never skip authCheck events + if (atype.equals(Constants.AUTH_TYPE_AUTHCHECK)) { + return false; + } + + final JsonObject param = inputJSON.get(Constants.FIELD_PARAM).getAsJsonObject(); + + boolean shouldSkip = (atype.equals(Constants.AUTH_TYPE_AUTHENTICATE) + && param.get(Constants.FIELD_ERROR).getAsString().equals(Constants.ERROR_CODE_SUCCESS)) + || (param.has(Constants.FIELD_NS) && param.get(Constants.FIELD_NS).getAsString().isEmpty()); + return shouldSkip; + } + + /** Checks if a profiler event should be skipped. */ + private boolean shouldSkipProfilerEvent(JsonObject inputJSON) { + return (!inputJSON.has(Constants.FIELD_NS)) + || (inputJSON.has(Constants.FIELD_NS) + && inputJSON.get(Constants.FIELD_NS).getAsString().isEmpty()); + } + + /** Enriches a record with server hostname information from the event. */ + private void enrichRecordWithServerInfo(Event e, Record record) { + String serverHostnamePrefix = EventUtils.getServerHostnamePrefix(e); + if (serverHostnamePrefix != null) { + record + .getAccessor() + .setServerHostName(serverHostnamePrefix + Constants.SERVER_HOSTNAME_SUFFIX); + String dbName = record.getDbName(); + record.setDbName( + !dbName.isEmpty() ? serverHostnamePrefix + ":" + dbName : serverHostnamePrefix); + } + record.getAccessor().setServiceName(record.getDbName()); + } + + /** Corrects IP addresses in the record based on event data. */ + private void correctIPs(Event e, Record record) { + SessionLocator sessionLocator = record.getSessionLocator(); + String sessionServerIp = sessionLocator.getServerIp(); + + + if (ValidationUtils.isDocumentInternalCommandIp(sessionServerIp)) { + String ip = EventUtils.getValidatedEventServerIp(e); + if (ip != null) { + if (Util.isIPv6(ip)) { + sessionLocator.setServerIpv6(ip); + sessionLocator.setIpv6(true); + } else { + sessionLocator.setServerIp(ip); + sessionLocator.setIpv6(false); + } + } else if (sessionServerIp.equalsIgnoreCase(Constants.DOCUMENT_INTERNAL_API_IP)) { + sessionLocator.setServerIp(Constants.DEFAULT_IP); + } + } + + if (ValidationUtils.isDocumentInternalCommandIp(sessionLocator.getClientIp())) { + // Store the original client port before updating IP + int originalClientPort = sessionLocator.getClientPort(); + if (sessionLocator.isIpv6()) { + sessionLocator.setClientIpv6(sessionLocator.getServerIpv6()); + } else { + sessionLocator.setClientIp(sessionLocator.getServerIp()); + + } + + // Restore the client port (important for authCheck events where port is parsed from remote_ip) + sessionLocator.setClientPort(originalClientPort); + } + + } + + /** Handles a skipped event. */ + private void handleSkippedEvent( + Event e, String messageString, FilterMatchListener matchListener) { + e.tag(Constants.LOGSTASH_TAG_SKIP); + skippedEvents.add(e); + String errorMsg = Constants.ERROR_INVALID_AUTHENTICATE_LOG; + Record record = parser.parseRecordException(null, errorMsg, messageString); + updateEventWithException( + record, errorMsg, messageString, e, Constants.LOGSTASH_TAG_SKIP, matchListener); + } + + /** Handles missing database name in profiler event. */ + private void handleMissingDbName( + Event e, String messageString, Record record, FilterMatchListener matchListener) { + e.tag(Constants.LOGSTASH_TAG_SKIP); + String errorMsg = Constants.ERROR_MISSING_DB_NAME; + record = parser.parseRecordException(record, errorMsg, messageString); + updateEventWithException( + record, errorMsg, messageString, e, Constants.LOGSTASH_TAG_SKIP, matchListener); + } + + /** Handles StackOverflowError. */ + private void handleStackOverflowError( + Event e, String messageString, FilterMatchListener matchListener) { + log.error( + "DocumentDB filter: JSON nesting too deep (StackOverflow), skipping event {} ", + EventUtils.logEvent(e)); + String errorMsg = Constants.ERROR_JSON_NESTING_TOO_DEEP; + Record record = parser.parseRecordException(null, errorMsg, messageString); + updateEventWithException( + record, errorMsg, messageString, e, Constants.LOGSTASH_TAG_JSON_DEPTH_ERROR, matchListener); + } + + /** Handles OutOfMemoryError. */ + private void handleOutOfMemoryError( + Event e, String messageString, FilterMatchListener matchListener) { + log.error( + "DocumentDB filter: Insufficient memory to process event, skipping {} ", + EventUtils.logEvent(e)); + String errorMsg = Constants.ERROR_INSUFFICIENT_MEMORY; + Record record = parser.parseRecordException(null, errorMsg, messageString); + updateEventWithException( + record, errorMsg, messageString, e, Constants.LOGSTASH_TAG_JSON_PARSE_ERROR, matchListener); + System.gc(); // Suggest garbage collection + } + + /** Handles JsonSyntaxException. */ + private void handleJsonSyntaxError( + Event e, + String messageString, + JsonSyntaxException jse, + FilterMatchListener matchListener, + boolean isAudit) { + String eventType = isAudit ? "audit" : "profiler"; + log.error( + "DocumentDB filter: Error parsing docDb {} event {} \n {} ", + eventType, + EventUtils.logEvent(e), + formatJsonSyntaxException(jse)); + String errorMsg = + isAudit ? Constants.ERROR_PARSING_AUDIT_EVENT : Constants.ERROR_PARSING_PROFILER_EVENT; + // Append exception message to error description + String exceptionMsg = formatJsonSyntaxException(jse); + if (exceptionMsg != null && !exceptionMsg.isEmpty()) { + errorMsg = errorMsg + " - " + exceptionMsg; + } + Record record = parser.parseRecordException(null, errorMsg, messageString); + updateEventWithException( + record, errorMsg, messageString, e, Constants.LOGSTASH_TAG_JSON_PARSE_ERROR, matchListener); + } + + /** Handles generic exceptions. */ + private void handleGenericError( + Event e, + String messageString, + Exception exception, + FilterMatchListener matchListener, + boolean isAudit) { + String eventType = isAudit ? "audit" : "profiler"; + log.error( + "DocumentDB filter: Error parsing docDb {} event {} ", + eventType, + EventUtils.logEvent(e), + exception); + String errorMsg = + isAudit ? Constants.ERROR_PARSING_AUDIT_EVENT : Constants.ERROR_PARSING_PROFILER_EVENT; + // Append exception message to error description + String exceptionMsg = exception.getMessage(); + if (exceptionMsg != null && !exceptionMsg.isEmpty()) { + errorMsg = errorMsg + " - " + exceptionMsg; + } + Record record = parser.parseRecordException(null, errorMsg, messageString); + updateEventWithException( + record, errorMsg, messageString, e, Constants.LOGSTASH_TAG_JSON_PARSE_ERROR, matchListener); + } + + /** Updates an event with exception information. */ + private void updateEventWithException( + Record record, + String errorMsg, + String eventLog, + Event e, + String tag, + FilterMatchListener matchListener) { + record = parser.parseRecordException(record, errorMsg, eventLog); + enrichRecordWithServerInfo(e, record); + e.tag(tag); + e.setField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME, GSON_SERIALIZER.toJson(record)); + matchListener.filterMatched(e); + } } diff --git a/filter-plugin/logstash-filter-documentdb-aws-guardium/src/main/java/com/ibm/guardium/documentdb/EventUtils.java b/filter-plugin/logstash-filter-documentdb-aws-guardium/src/main/java/com/ibm/guardium/documentdb/EventUtils.java new file mode 100644 index 000000000..cc0da4521 --- /dev/null +++ b/filter-plugin/logstash-filter-documentdb-aws-guardium/src/main/java/com/ibm/guardium/documentdb/EventUtils.java @@ -0,0 +1,186 @@ +/* +Copyright 2022-2023 IBM Inc. All rights reserved +SPDX-License-Identifier: Apache-2.0 +*/ +package com.ibm.guardium.documentdb; + +import co.elastic.logstash.api.Event; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.Map; + +/** + * Utility class for Event-related operations in DocumentDB Guardium filter. + * Provides methods for event logging, field extraction, and manipulation. + */ +public final class EventUtils { + + private static final Logger log = LogManager.getLogger(EventUtils.class); + + // Reuse Gson instance for performance (only used in stdin case) + private static final com.google.gson.Gson GSON_RECONSTRUCTOR = new com.google.gson.Gson(); + + // Prevent instantiation + private EventUtils() { + throw new AssertionError("EventUtils class should not be instantiated"); + } + + /** + * Creates a log string representation of an event (optimized with StringBuilder). + * + * @param event The event to log + * @return String representation of the event + */ + public static String logEvent(Event event) { + StringBuilder sb = new StringBuilder(Constants.STRING_BUILDER_INITIAL_CAPACITY); + try { + sb.append("{ "); + boolean first = true; + for (Map.Entry entry : event.getData().entrySet()) { + if (!first) { + sb.append(", "); + } + sb.append('"') + .append(entry.getKey()) + .append("\": \"") + .append(entry.getValue()) + .append('"'); + first = false; + } + sb.append(" }"); + return sb.toString(); + } catch (Exception e) { + log.error("DocumentDB filter: Failed to create event log string", e); + return Constants.ERROR_FAILED_TO_SERIALIZE_EVENT; + } + } + + /** + * Safely extracts a string field from an event. + * + * @param event The event + * @param fieldName The field name to extract + * @return The field value as string, or null if not found or fieldName is null + */ + public static String getStringField(Event event, String fieldName) { + if (fieldName == null) { + return null; + } + Object field = event.getField(fieldName); + return (field != null) ? field.toString() : null; + } + + /** + * Validates and extracts server IP from event. + * + * @param event The event containing server_ip field + * @return Valid IP address or null if invalid + */ + public static String getValidatedEventServerIp(Event event) { + String ip = getStringField(event, Constants.EVENT_FIELD_SERVER_IP); + return ValidationUtils.isValidIpAddress(ip) ? ip : null; + } + + /** + * Extracts the message field from an event. + * Handles two cases: + * 1. CloudWatch logs: message field contains JSON string (fast path - 99% of cases) + * 2. stdin with json codec: JSON already parsed into event fields (slow path - testing only) + * + * @param event The event + * @return The message string, or reconstructed JSON if already parsed + */ + public static String getMessageField(Event event) { + // Fast path: CloudWatch logs with message field (99% of cases) + String message = getStringField(event, Constants.EVENT_FIELD_MESSAGE); + if (message != null) { + return message; + } + + // Slow path: stdin with json codec - only for testing + // Check if JSON is already parsed by looking for DocumentDB audit signal + Object atype = event.getField(Constants.FIELD_ATYPE); + if (atype != null) { + // JSON already parsed, reconstruct it manually to avoid Gson reflection issues + try { + return reconstructJsonFromEvent(event); + } catch (Exception e) { + return null; + } + } + + return null; + } + + /** + * Reconstructs JSON string from parsed event fields. + * Only includes DocumentDB-relevant fields, avoiding Logstash internal fields. + * This is only used for stdin testing, not production CloudWatch logs. + * + * @param event The event with parsed JSON fields + * @return JSON string representation + */ + private static String reconstructJsonFromEvent(Event event) { + com.google.gson.JsonObject json = new com.google.gson.JsonObject(); + + + // Add all simple fields from the event, skipping Logstash internals + for (Map.Entry entry : event.getData().entrySet()) { + String key = entry.getKey(); + Object value = entry.getValue(); + + // Skip Logstash internal fields + if (key.startsWith("@") || key.equals("host") || key.equals("event")) { + continue; + } + + try { + + // Convert value to JsonElement + if (value == null) { + json.add(key, com.google.gson.JsonNull.INSTANCE); + } else if (value instanceof String) { + json.addProperty(key, (String) value); + } else if (value instanceof Number) { + json.addProperty(key, (Number) value); + } else if (value instanceof Boolean) { + json.addProperty(key, (Boolean) value); + } else if (value instanceof Map || value instanceof java.util.List) { + // For complex objects, try Gson first, fall back to string representation if it fails + try { + json.add(key, GSON_RECONSTRUCTOR.toJsonTree(value)); + } catch (com.google.gson.JsonIOException jioEx) { + // Gson reflection failed (Java module restrictions) + // Convert to string and try to parse as JSON + try { + String valueStr = value.toString(); + // Try to parse it as JSON + json.add(key, com.google.gson.JsonParser.parseString(valueStr)); + } catch (Exception parseEx) { + // If parsing fails, just add as string + json.addProperty(key, value.toString()); + } + } + } else { + // For other types, convert to string + json.addProperty(key, value.toString()); + } + } catch (Exception e) { + } + } + + String result = GSON_RECONSTRUCTOR.toJson(json); + return result; + } + + /** + * Extracts the server hostname prefix from an event. + * + * @param event The event + * @return The server hostname prefix, or null if not found + */ + public static String getServerHostnamePrefix(Event event) { + return getStringField(event, Constants.EVENT_FIELD_SERVER_HOSTNAME_PREFIX); + } +} diff --git a/filter-plugin/logstash-filter-documentdb-aws-guardium/src/main/java/com/ibm/guardium/documentdb/Parser.java b/filter-plugin/logstash-filter-documentdb-aws-guardium/src/main/java/com/ibm/guardium/documentdb/Parser.java index fe756fd30..62ba899dd 100644 --- a/filter-plugin/logstash-filter-documentdb-aws-guardium/src/main/java/com/ibm/guardium/documentdb/Parser.java +++ b/filter-plugin/logstash-filter-documentdb-aws-guardium/src/main/java/com/ibm/guardium/documentdb/Parser.java @@ -5,17 +5,13 @@ package com.ibm.guardium.documentdb; import java.text.ParseException; -import java.util.Arrays; import java.util.Date; -import java.util.HashSet; import java.util.Iterator; import java.util.Map.Entry; -import java.util.Set; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.ibm.guardium.universalconnector.commons.structures.Accessor; @@ -28,366 +24,841 @@ import com.ibm.guardium.universalconnector.commons.structures.SessionLocator; import com.ibm.guardium.universalconnector.commons.structures.Time; +/** + * Parser class for DocumentDB audit and profiler logs. + * Converts DocumentDB log events into Guardium Record structures. + */ public class Parser { - private static Logger log = LogManager.getLogger(Parser.class); - - public static final String DATA_PROTOCOL_STRING = "DocumentDB"; - public static final String UNKOWN_STRING = ""; - public static final String SERVER_TYPE_STRING = "DocumentDB"; - public static final String COMPOUND_OBJECT_STRING = "[json-object]"; - public static final String EXCEPTION_TYPE_AUTHORIZATION_STRING = "SQL_ERROR"; - public static final String EXCEPTION_TYPE_AUTHENTICATION_STRING = "LOGIN_FAILED"; - public static final String NA_STRING = "N/A"; - /** - * These arguments will not be redacted, as they only contain collection/field - * names rather than sensitive values. - */ - public static Set REDACTION_IGNORE_STRINGS = new HashSet<>( - Arrays.asList("from", "localField", "foreignField", "as", "connectFromField", "connectToField")); - - /** - * Parses Audit logs and returns a Guard Record object - * @param data - * @return - * @throws ParseException - */ - public static Record parseAuditRecord(final JsonObject data) throws ParseException { - Record record = new Record(); - final JsonObject param = data.get("param").getAsJsonObject(); - if (param.get("error") != null) { - param.get("error").getAsString(); - } - - // Setting db name - String dbName = Parser.UNKOWN_STRING; - if (param != null && param.has("ns")) { - final String ns = param.get("ns").getAsString(); - dbName = ns.split("\\.")[0]; // sometimes contains "."; fallback OK. - } - record.setDbName(dbName); - - record.setAppUserName(Parser.UNKOWN_STRING); - - // Setting sessionLocator object - record.setSessionLocator(Parser.parseSessionLocatorDocumentDb(data)); - - // Setting accessor - record.setAccessor(Parser.parseAccessorDocumentDb(data)); - - - if(data.get("atype").getAsString().equalsIgnoreCase("authenticate")) { - String result = param.get("error").getAsString(); - if (result.equals("0")) { - record.setData(Parser.parseDataDocumentDb(data)); - } else { - record.setException(Parser.parseException(data, result)); - } - }else { - record.setData(Parser.parseDataDocumentDb(data)); - } - - // Setting timestamp - String dateString = Parser.getTimestampStringDocumentDb(data); - record.setSessionId(dateString); - Time time = Parser.parseTimeDocumentDb(dateString); + private static final Logger log = LogManager.getLogger(Parser.class); + + // Deprecated constants - use Constants class instead + @Deprecated + public static final String DATA_PROTOCOL_STRING = Constants.DATA_PROTOCOL; + @Deprecated + public static final String UNKOWN_STRING = Constants.UNKNOWN_STRING; + @Deprecated + public static final String SERVER_TYPE_STRING = Constants.SERVER_TYPE; + @Deprecated + public static final String COMPOUND_OBJECT_STRING = Constants.COMPOUND_OBJECT; + @Deprecated + public static final String EXCEPTION_TYPE_AUTHORIZATION_STRING = Constants.EXCEPTION_TYPE_AUTHORIZATION; + @Deprecated + public static final String EXCEPTION_TYPE_AUTHENTICATION_STRING = Constants.EXCEPTION_TYPE_AUTHENTICATION; + @Deprecated + public static final String NA_STRING = Constants.NOT_AVAILABLE; + @Deprecated + public static final String DEFAULT_IP = Constants.DEFAULT_IP; + @Deprecated + public static final String DEFAULT_IPV6 = Constants.DEFAULT_IPV6; + @Deprecated + public static final String LOGIN_FAILED = Constants.EXCEPTION_TYPE_AUTHENTICATION; + @Deprecated + public static final String SQL_ERROR = Constants.EXCEPTION_TYPE_AUTHORIZATION; + @Deprecated + public static final String UC_PARSER_ERROR = Constants.UC_PARSER_ERROR; + @Deprecated + public static final String UC_AUDIT_ERROR = Constants.UC_AUDIT_ERROR; + @Deprecated + public static final java.util.Set REDACTION_IGNORE_STRINGS = Constants.REDACTION_IGNORE_STRINGS; + + /** + * Parses Audit logs and returns a Guard Record object + * + * @param data + * @return + * @throws ParseException + */ + public Record parseAuditRecord(final JsonObject data) throws ParseException { + String atype = data.get(Constants.FIELD_ATYPE).getAsString(); + // Handle authCheck events separately + if (atype.equalsIgnoreCase(Constants.AUTH_TYPE_AUTHCHECK)) { + return parseAuthCheckRecord(data); + } + + Record record = new Record(); + final JsonObject param = data.get(Constants.FIELD_PARAM).getAsJsonObject(); + if (param.get(Constants.FIELD_ERROR) != null) { + param.get(Constants.FIELD_ERROR).getAsString(); + } + + // Setting db name + String dbName = Constants.UNKNOWN_STRING; + if (param != null && param.has(Constants.FIELD_NS)) { + final String ns = param.get(Constants.FIELD_NS).getAsString(); + dbName = StringUtils.extractDbNameFromNs(ns); + } + record.setDbName(dbName); + + record.setAppUserName(Constants.UNKNOWN_STRING); + + // Setting sessionLocator object + record.setSessionLocator(parseSessionLocatorDocumentDb(data)); + + // Setting accessor + record.setAccessor(parseAccessorDocumentDb(data)); + + if(atype.equalsIgnoreCase(Constants.AUTH_TYPE_AUTHENTICATE)) { + String result = param.get(Constants.FIELD_ERROR).getAsString(); + if (result.equals(Constants.ERROR_CODE_SUCCESS)) { + record.setData(parseDataDocumentDb(data)); + } else { + record.setException(parseException(data, result)); + } + }else { + record.setData(parseDataDocumentDb(data)); + } + + record.setSessionId(Constants.UNKNOWN_STRING); + // Setting timestamp + String dateString = getTimestampStringDocumentDb(data); + + Time time = parseTimeDocumentDb(dateString); + record.setTime(time); + + return record; + } + + /** + * Parses authCheck audit logs and returns a Guard Record object + * + * @param data + * @return + * @throws ParseException + */ + public Record parseAuthCheckRecord(final JsonObject data) throws ParseException { + Record record = new Record(); + + // Get param object - cache to avoid multiple lookups + JsonObject param = extractParamObject(data); + + // Setting db name from param.ns + String dbName = Constants.UNKNOWN_STRING; + if (param != null && param.has(Constants.FIELD_NS)) { + final String ns = param.get(Constants.FIELD_NS).getAsString(); + dbName = StringUtils.extractDbNameFromNs(ns); + } + record.setDbName(dbName); + + // Extract app user name from users array at top level + String appUserName = extractAppUserName(data); + record.setAppUserName(appUserName); + + // Setting sessionLocator object + SessionLocator sessionLocator = parseSessionLocatorAuthCheck(data); + + record.setSessionLocator(sessionLocator); + + // Setting accessor + Accessor accessor = parseAccessorAuthCheck(data); + + record.setAccessor(accessor); + + // Setting data + Data recordData = parseDataAuthCheck(data); + record.setData(recordData); + + record.setSessionId(Constants.UNKNOWN_STRING); + + // Setting timestamp + String dateString = getTimestampStringDocumentDb(data); + Time time = parseTimeDocumentDb(dateString); + + record.setTime(time); + return record; + } + + /** + * Extracts app user name from users array in authCheck events + * + * @param data + * @return + */ + private String extractAppUserName(JsonObject data) { + if (!data.has("users")) { + return Constants.UNKNOWN_STRING; + } + + JsonElement usersElement = data.get("users"); + com.google.gson.JsonArray usersArray = null; + + if (usersElement.isJsonArray()) { + usersArray = usersElement.getAsJsonArray(); + } else if (usersElement.isJsonPrimitive() && usersElement.getAsJsonPrimitive().isString()) { + // users is a string (from stdin toString()), try to parse it + try { + usersArray = com.google.gson.JsonParser.parseString(usersElement.getAsString()).getAsJsonArray(); + } catch (Exception e) { + // Parsing failed, return unknown + return Constants.UNKNOWN_STRING; + } + } + + // Extract username from first user in array + if (usersArray != null && usersArray.size() > 0) { + JsonObject firstUser = usersArray.get(0).getAsJsonObject(); + if (firstUser.has(Constants.FIELD_USER)) { + return firstUser.get(Constants.FIELD_USER).getAsString(); + } + } + + return Constants.UNKNOWN_STRING; + } + + /** + * Parses the query and returns a Data instance for authCheck events + * + * @param inputJSON + * @return + */ + public Data parseDataAuthCheck(JsonObject inputJSON) { + Data data = new Data(); + try { + Construct construct = parseAsConstructAuthCheck(inputJSON); + if (construct != null) { + data.setConstruct(construct); + if (construct.getFullSql() == null) { + construct.setFullSql(Constants.UNKNOWN_STRING); + } + if (construct.getRedactedSensitiveDataSql() == null) { + construct.setRedactedSensitiveDataSql(Constants.UNKNOWN_STRING); + } + + } + } catch (Exception e) { + + throw e; + } + return data; + } + + /** + * Parses authCheck event and returns Construct + * + * @param data + * @return + */ + public Construct parseAsConstructAuthCheck(final JsonObject data) { + try { + final Sentence sentence = parseSentenceAuthCheck(data); + final Construct construct = new Construct(); + construct.sentences.add(sentence); + String fullSql = "\"atype\": "+data.get(Constants.FIELD_ATYPE).toString()+","+data.get(Constants.FIELD_PARAM); + construct.setFullSql(fullSql); + construct.setRedactedSensitiveDataSql(fullSql); + + return construct; + } catch (final Exception e) { + + throw e; + } + } + + /** + * Parses authCheck audit log JsonObject and returns Sentence + * + * @param data + * @return + */ + protected Sentence parseSentenceAuthCheck(final JsonObject data) { + JsonObject param = extractParamObject(data); + String command = param.has(Constants.FIELD_COMMAND) ? param.get(Constants.FIELD_COMMAND).getAsString() : "authCheck"; + Sentence sentence = new Sentence(command); + sentence.getObjects().add(parseSentenceObjectAuthCheck(param)); + return sentence; + } + + /** + * Extracts param object from data, handling different formats (JsonObject, String, or fallback to data) + * This method centralizes param extraction logic to avoid code duplication + * + * @param data + * @return JsonObject representing param + */ + private JsonObject extractParamObject(final JsonObject data) { + if (data.has(Constants.FIELD_PARAM) && !data.get(Constants.FIELD_PARAM).isJsonNull()) { + JsonElement paramElement = data.get(Constants.FIELD_PARAM); + if (paramElement.isJsonObject()) { + return paramElement.getAsJsonObject(); + } else if (paramElement.isJsonPrimitive() && paramElement.getAsJsonPrimitive().isString()) { + // param is a string (from stdin toString()), try to parse it + try { + return com.google.gson.JsonParser.parseString(paramElement.getAsString()).getAsJsonObject(); + } catch (Exception e) { + return data; + } + } + } + // If no param field or invalid format, use data itself as fallback + return data; + } + + /** + * Parses JsonObject passed as argument and returns SentenceObject for authCheck + * + * @param param + * @return + */ + protected SentenceObject parseSentenceObjectAuthCheck(JsonObject param) { + SentenceObject sentenceObject; + if(param.has(Constants.FIELD_NS)) { + String collection = StringUtils.extractCollectionFromNs(param.get(Constants.FIELD_NS).getAsString()); + sentenceObject = new SentenceObject(collection); + } else { + + sentenceObject = new SentenceObject(param.toString()); + } + sentenceObject.setType("collection"); + return sentenceObject; + } + + /** + * Creates Accessor object for authCheck events + * + * @param data + * @return + */ + public Accessor parseAccessorAuthCheck(JsonObject data) { + Accessor accessor = new Accessor(); + + accessor.setDbProtocol(Constants.DATA_PROTOCOL); + accessor.setServerType(Constants.SERVER_TYPE); + + // Get dbUser from users array - cache result to avoid duplicate extraction + String dbUsers = extractAppUserName(data); + accessor.setDbUser(dbUsers.isEmpty() ? Constants.NOT_AVAILABLE : dbUsers); + + // Extract source program + String sourceProgram = Constants.UNKNOWN_STRING; + if (data.has(Constants.FIELD_APP_NAME)) { + sourceProgram = StringUtils.removeWhitespace(data.get(Constants.FIELD_APP_NAME).getAsString().trim()); + } + accessor.setSourceProgram(sourceProgram); + + // Set static fields + accessor.setServerHostName(Constants.DEFAULT_SERVER_HOSTNAME); + accessor.setLanguage(Accessor.LANGUAGE_FREE_TEXT_STRING); + accessor.setDataType(Accessor.DATA_TYPE_GUARDIUM_SHOULD_NOT_PARSE_SQL); + accessor.setClient_mac(Constants.UNKNOWN_STRING); + accessor.setClientHostName(Constants.UNKNOWN_STRING); + accessor.setClientOs(Constants.UNKNOWN_STRING); + accessor.setCommProtocol(Constants.UNKNOWN_STRING); + accessor.setDbProtocolVersion(Constants.UNKNOWN_STRING); + accessor.setOsUser(Constants.UNKNOWN_STRING); + accessor.setServerDescription(Constants.UNKNOWN_STRING); + accessor.setServerOs(Constants.UNKNOWN_STRING); + + // Extract and set service name (db name) - use centralized param extraction + String dbName = Constants.UNKNOWN_STRING; + JsonObject param = extractParamObject(data); + if (param.has(Constants.FIELD_NS)) { + dbName = StringUtils.extractDbNameFromNs(param.get(Constants.FIELD_NS).getAsString()); + } + accessor.setServiceName(dbName); + + return accessor; + } + + /** + * Parses JSON object and returns session locator for authCheck events + * + * @param data + * @return + */ + private SessionLocator parseSessionLocatorAuthCheck(JsonObject data) { + SessionLocator sessionLocator = new SessionLocator(); + sessionLocator.setIpv6(false); + + sessionLocator.setClientIp(Constants.DEFAULT_IP); + sessionLocator.setClientPort(SessionLocator.PORT_DEFAULT); + sessionLocator.setClientIpv6(Constants.UNKNOWN_STRING); + + String remote = data.has(Constants.FIELD_REMOTE_IP) ? data.get(Constants.FIELD_REMOTE_IP).getAsString() : null; + if (remote != null && remote.indexOf(':') > -1) { + String[] remoteobjects = remote.split(":"); + if(remoteobjects.length > 1) { + sessionLocator.setClientIp(remoteobjects[0]); + // For authCheck, parse the port from remote_ip + try { + int port = Integer.parseInt(remoteobjects[1]); + sessionLocator.setClientPort(port); + } catch (NumberFormatException e) { + + sessionLocator.setClientPort(SessionLocator.PORT_DEFAULT); + } + } + } + + sessionLocator.setServerIp(Constants.DEFAULT_IP); + sessionLocator.setServerPort(SessionLocator.PORT_DEFAULT); + + return sessionLocator; + } + + /** + * Parses Profiler logs and returns a Guard Record object + * + * @param data + * @return + * @throws ParseException + */ + public Record parseProfilerRecord(final JsonObject data) throws ParseException { + Record record = new Record(); + + // Setting db name + String dbName = Constants.UNKNOWN_STRING; + + if (data != null && data.has(Constants.FIELD_NS)) { + final String ns = data.get(Constants.FIELD_NS).getAsString(); + dbName = StringUtils.extractDbNameFromNs(ns); + } + record.setDbName(dbName); + + record.setAppUserName(Constants.UNKNOWN_STRING); + + // Setting sessionLocator object + record.setSessionLocator(parseSessionLocatorDocumentDb(data)); + + // Setting accessor + record.setAccessor(parseAccessorDocumentDb(data)); + + record.setData(parseDataDocumentDb(data)); + record.setSessionId(Constants.UNKNOWN_STRING); + + // Setting timestamp + String dateString = getTimestampStringDocumentDb(data); + Time time = parseTimeDocumentDb(dateString); record.setTime(time); - return record; } - /** - * Parses Profiler logs and returns a Guard Record object - * @param data - * @return - * @throws ParseException - */ - public static Record parseProfilerRecord(final JsonObject data) throws ParseException { - Record record = new Record(); - final JsonObject param = data.get("command").getAsJsonObject(); - - // Setting session ID - String sessionId = Parser.UNKOWN_STRING; - if (param != null && param.has("lsid")) { - final JsonObject lsid = param.getAsJsonObject("lsid"); - sessionId = lsid.getAsJsonObject("id").get("$binary").getAsString(); - } - - // Setting db name - String dbName = Parser.UNKOWN_STRING; - - if (data != null && data.has("ns")) { - final String ns = data.get("ns").getAsString(); - dbName = ns.split("\\.")[0]; // sometimes contains "."; fallback OK. - } - record.setDbName(dbName); - - record.setAppUserName(Parser.UNKOWN_STRING); - - // Setting sessionLocator object - record.setSessionLocator(Parser.parseSessionLocatorDocumentDb(data)); - - - // Setting accessor - record.setAccessor(Parser.parseAccessorDocumentDb(data)); - - - record.setData(Parser.parseDataDocumentDb(data)); - - // Setting timestamp - String dateString = Parser.getTimestampStringDocumentDb(data); - record.setSessionId(dateString); - Time time = Parser.parseTimeDocumentDb(dateString); - record.setTime(time); - return record; - } - - /** - * Converts timestamp jsonObject and returns a string - * @param data - * @return - */ - public static String getTimestampStringDocumentDb(final JsonObject data) { - String dateString = null; - if (data.has("ts")) { - dateString = data.get("ts").getAsString(); - } - return dateString; - } - - /** - * This method will parse timestamp String into Time object - * @param dateString - * @return - */ - public static Time parseTimeDocumentDb(String dateString) { + /** + * Converts timestamp jsonObject and returns a string + * + * @param data + * @return + */ + public String getTimestampStringDocumentDb(final JsonObject data) { + String dateString = null; + if (data.has(Constants.FIELD_TS)) { + dateString = data.get(Constants.FIELD_TS).getAsString(); + } + return dateString; + } + + /** + * This method will parse timestamp String into Time object + * + * @param dateString + * @return + */ + public Time parseTimeDocumentDb(String dateString) { Date date = new java.util.Date(Long.parseLong(dateString)); return new Time(date.getTime(), date.getTimezoneOffset(), 0); } - /** - * Parses audit/profiler log JsonObject and returns Sentence - * @param data - * @return - */ - protected static Sentence parseSentenceDocumentDb(final JsonObject data) { - - Sentence sentence = null; - // + main object - String atype = ""; - if (data.has("atype")) { - atype = data.get("atype").getAsString(); - } - if (!atype.isEmpty()) {// Audit Logs - JsonObject param = data.getAsJsonObject("param"); - sentence = new Sentence(atype); - sentence.getObjects().add(parseSentenceObjectDocumentDbAudit(param,atype)); - } else {// Profiler logs - final JsonObject command = data.get("command").getAsJsonObject(); - if (data.has("op") && data.get("op").getAsString().equals("update") - || data.has("op") && data.get("op").getAsString().equals("remove")) { - String key = data.get("op").getAsString(); - sentence = new Sentence(key); - } else { - for (Iterator> iterator = command.entrySet().iterator(); iterator.hasNext();) { - String key = iterator.next().getKey(); - sentence = new Sentence(key); - break; - } - } - sentence.getObjects().add(parseSentenceObjectDocumentDbProfiler(data)); - - } - - return sentence; - } - - /** - * Parses JsonObject passed as argument and returns Sentence object - * @param command - * @return - */ - protected static SentenceObject parseSentenceObjectDocumentDbAudit(JsonObject command, String aType) { - SentenceObject sentenceObject; - if(aType.equalsIgnoreCase("authenticate") && command.has("user")) { - sentenceObject = new SentenceObject(command.get("user").getAsString()); - }else if(command.has("ns")) { - sentenceObject = new SentenceObject(command.get("ns").getAsString().contains(".")?command.get("ns").getAsString().split("\\.")[1]:command.get("ns").getAsString()); - }else if(command.has("userName")){ - sentenceObject = new SentenceObject(command.get("userName").getAsString()); - }else if(aType.equalsIgnoreCase("createRole") && command.has("role")) { - sentenceObject = new SentenceObject(command.get("role").getAsString()); - }else if(aType.equalsIgnoreCase("dropRole") && command.has("roleName")) { - sentenceObject = new SentenceObject(command.get("roleName").getAsString()); - }else { - sentenceObject = new SentenceObject(command.toString()); - } - sentenceObject.setType("collection"); // this used to be default value, but since sentence is defined in - - return sentenceObject; - } - - /** - * Parses JsonObject passed as argument and returns Sentence object - * @param command - * @return - */ - protected static SentenceObject parseSentenceObjectDocumentDbProfiler(JsonObject command) { - SentenceObject sentenceObject = new SentenceObject(UNKOWN_STRING); - if (command != null && command.has("ns")) { - sentenceObject = new SentenceObject(command.get("ns").getAsString().contains(".")?command.get("ns").getAsString().split("\\.")[1]:command.get("ns").getAsString()); - } - else { - sentenceObject = new SentenceObject(command.toString()); - } - sentenceObject.setType("collection"); // this used to be default value, but since sentence is defined in - - return sentenceObject; - } - - public static Construct parseAsConstructDocumentDb(final JsonObject data) { - try { - final Sentence sentence = Parser.parseSentenceDocumentDb(data); - final Construct construct = new Construct(); - construct.sentences.add(sentence); - if(data.has("param")) { - construct.setFullSql("\"atype\": "+data.get("atype").toString()+","+data.get("param")); - construct.setRedactedSensitiveDataSql("\"atype\": "+data.get("atype").toString()+","+data.get("param")); - }else if(data.has("command")) { - construct.setFullSql(data.get("command").toString()); - construct.setRedactedSensitiveDataSql(data.get("command").toString()); - } - return construct; - } catch (final Exception e) { - throw e; - } - } - - /** - * Parses the query and returns a Data instance. - * - * @param inputJSON - * @return - * - * @see Data - */ - public static Data parseDataDocumentDb(JsonObject inputJSON) { - Data data = new Data(); - try { - Construct construct = parseAsConstructDocumentDb(inputJSON); - if (construct != null) { - data.setConstruct(construct); - - if (construct.getFullSql() == null) { - construct.setFullSql(UNKOWN_STRING); - } - if (construct.getRedactedSensitiveDataSql() == null) { - construct.setRedactedSensitiveDataSql(Parser.UNKOWN_STRING); - } - } - } catch (Exception e) { - log.error("DocumentDB filter: Error parsing JSon " + inputJSON, e); - throw e; - } - return data; - } - - /** - * Creates an ExceptionRecord to be used in Record, instead of Data. - * - * @param data - * @param resultCode - * @return - */ - private static ExceptionRecord parseException(JsonObject data, String resultCode) { - ExceptionRecord exceptionRecord = new ExceptionRecord(); - if (resultCode.equals("13")) { - exceptionRecord.setExceptionTypeId(Parser.EXCEPTION_TYPE_AUTHORIZATION_STRING); - exceptionRecord.setDescription("Unauthorized to perform the operation (13)"); - - } else if (resultCode.equals("18")) { - exceptionRecord.setExceptionTypeId(Parser.EXCEPTION_TYPE_AUTHENTICATION_STRING); - exceptionRecord.setDescription("Authentication Failed (18)"); - } else { // prep for unknown error code - exceptionRecord.setExceptionTypeId(Parser.EXCEPTION_TYPE_AUTHORIZATION_STRING); - exceptionRecord.setDescription("Error (" + resultCode + ")"); - } - - exceptionRecord.setSqlString(data.getAsJsonObject("param").get("message").getAsString()); - return exceptionRecord; - } - - /** - * Creates Accessor object to be used in Guard Record - * @param data - * @return - */ - public static Accessor parseAccessorDocumentDb(JsonObject data) { - Accessor accessor = new Accessor(); - - - - accessor.setDbProtocol(Parser.DATA_PROTOCOL_STRING); - accessor.setServerType(Parser.SERVER_TYPE_STRING); - String dbUsers = NA_STRING; - if (data.has("user")) { - dbUsers = (data.get("user")==null || data.get("user").getAsString().isEmpty())?NA_STRING:data.get("user").getAsString(); - }if(data.has("param") && data.get("param").getAsJsonObject().has("user")) { - String usr=data.get("param").getAsJsonObject().get("user").getAsString(); - dbUsers = (usr==null || usr.isEmpty())?NA_STRING:usr; - } - accessor.setDbUser(dbUsers); - - - String sourceProgram = Parser.UNKOWN_STRING; - if (data.has("appName")) { - sourceProgram = data.get("appName").getAsString().trim().replaceAll("\\s", ""); - } - accessor.setSourceProgram(sourceProgram); - accessor.setServerHostName("documentdb.amazonaws.com"); - accessor.setLanguage(Parser.UNKOWN_STRING); - accessor.setDataType(Accessor.DATA_TYPE_GUARDIUM_SHOULD_NOT_PARSE_SQL); - accessor.setClient_mac(Parser.UNKOWN_STRING); - accessor.setClientHostName(Parser.UNKOWN_STRING); - accessor.setClientOs(Parser.UNKOWN_STRING); - accessor.setCommProtocol(Parser.UNKOWN_STRING); - accessor.setDbProtocolVersion(Parser.UNKOWN_STRING); - accessor.setOsUser(Parser.UNKOWN_STRING); - accessor.setServerDescription(Parser.UNKOWN_STRING); - accessor.setServerOs(Parser.UNKOWN_STRING); - - - - return accessor; - } - - /** - * Parses JSON object and returns session locator - * @param data - * @return - */ - private static SessionLocator parseSessionLocatorDocumentDb(JsonObject data) { - SessionLocator sessionLocator = new SessionLocator(); - sessionLocator.setIpv6(false); - - sessionLocator.setClientIp("0.0.0.0");//Default value for Client IP - sessionLocator.setClientPort(SessionLocator.PORT_DEFAULT); - sessionLocator.setClientIpv6(Parser.UNKOWN_STRING); - - String remote = data.has("client") ? data.get("client").getAsString() - : (data.has("remote_ip") ? data.get("remote_ip").getAsString() : null); - if (remote != null && remote.indexOf(':') > -1) { - String[] remoteobjects = remote.split(":"); - - if(remoteobjects.length>1) { - sessionLocator.setClientIp(remoteobjects[0]); - sessionLocator.setClientPort(Integer.parseInt(remoteobjects[1])); - }else { - sessionLocator.setClientIp("0.0.0.0"); - sessionLocator.setClientPort(SessionLocator.PORT_DEFAULT); - } - } - sessionLocator.setServerIp("0.0.0.0");// In AWS databases setting this field to 0.0.0.0 - sessionLocator.setServerPort(SessionLocator.PORT_DEFAULT);// In AWS databases setting this field to -1 - return sessionLocator; - } + /** + * Parses audit/profiler log JsonObject and returns Sentence + * + * @param data + * @return + */ + protected Sentence parseSentenceDocumentDb(final JsonObject data) { + + Sentence sentence = null; + // + main object + String atype = ""; + if (data.has(Constants.FIELD_ATYPE)) { + atype = data.get(Constants.FIELD_ATYPE).getAsString(); + } + if (!atype.isEmpty()) {// Audit Logs + JsonObject param = data.getAsJsonObject(Constants.FIELD_PARAM); + sentence = new Sentence(atype); + sentence.getObjects().add(parseSentenceObjectDocumentDbAudit(param,atype)); + } else {// Profiler logs + final JsonObject command = data.get(Constants.FIELD_COMMAND).getAsJsonObject(); + if (data.has(Constants.FIELD_OP) && data.get(Constants.FIELD_OP).getAsString().equals(Constants.UPDATE_KEY) + || data.has(Constants.FIELD_OP) && data.get(Constants.FIELD_OP).getAsString().equals(Constants.DELETE_KEY)) { + String key = data.get(Constants.FIELD_OP).getAsString(); + sentence = new Sentence(key); + } else { + for (Iterator> iterator = command.entrySet().iterator(); iterator.hasNext();) { + String key = iterator.next().getKey(); + sentence = new Sentence(key); + break; + } + } + sentence.getObjects().add(parseSentenceObjectDocumentDbProfiler(data)); + + } + + return sentence; + } + + /** + * Parses JsonObject passed as argument and returns Sentence object + * + * @param command + * @return + */ + protected SentenceObject parseSentenceObjectDocumentDbAudit(JsonObject command, String aType) { + SentenceObject sentenceObject; + if(aType.equalsIgnoreCase(Constants.AUTH_TYPE_AUTHENTICATE) && command.has(Constants.FIELD_USER)) { + sentenceObject = new SentenceObject(command.get(Constants.FIELD_USER).getAsString()); + }else if(command.has(Constants.FIELD_NS)) { + sentenceObject = new SentenceObject(StringUtils.extractCollectionFromNs(command.get(Constants.FIELD_NS).getAsString())); + }else if(command.has("userName")){ + sentenceObject = new SentenceObject(command.get("userName").getAsString()); + }else if(aType.equalsIgnoreCase(Constants.AUTH_TYPE_CREATE_ROLE) && command.has("role")) { + sentenceObject = new SentenceObject(command.get("role").getAsString()); + }else if(aType.equalsIgnoreCase(Constants.AUTH_TYPE_DROP_ROLE) && command.has("roleName")) { + sentenceObject = new SentenceObject(command.get("roleName").getAsString()); + }else { + sentenceObject = new SentenceObject(command.toString()); + } + sentenceObject.setType("collection"); // this used to be default value, but since sentence is defined in + + return sentenceObject; + } + + /** + * Parses JsonObject passed as argument and returns Sentence object + * + * @param command + * @return + */ + protected SentenceObject parseSentenceObjectDocumentDbProfiler(JsonObject command) { + SentenceObject sentenceObject = new SentenceObject(Constants.UNKNOWN_STRING); + if (command != null && command.has(Constants.FIELD_NS)) { + sentenceObject = new SentenceObject(StringUtils.extractCollectionFromNs(command.get(Constants.FIELD_NS).getAsString())); + } + else { + sentenceObject = new SentenceObject(command.toString()); + } + sentenceObject.setType("collection"); // this used to be default value, but since sentence is defined in + + return sentenceObject; + } + + public Construct parseAsConstructDocumentDb(final JsonObject data) { + try { + final Sentence sentence = parseSentenceDocumentDb(data); + final Construct construct = new Construct(); + construct.sentences.add(sentence); + if(data.has(Constants.FIELD_PARAM)) { + construct.setFullSql("\"atype\": "+data.get(Constants.FIELD_ATYPE).toString()+","+data.get(Constants.FIELD_PARAM)); + construct.setRedactedSensitiveDataSql("\"atype\": "+data.get(Constants.FIELD_ATYPE).toString()+","+data.get(Constants.FIELD_PARAM)); + }else if(data.has(Constants.FIELD_COMMAND)) { + construct.setFullSql(data.get(Constants.FIELD_COMMAND).toString()); + construct.setRedactedSensitiveDataSql(data.get(Constants.FIELD_COMMAND).toString()); + } + return construct; + } catch (final Exception e) { + throw e; + } + } + + /** + * Parses the query and returns a Data instance. + * + * @param inputJSON + * @return + * @see Data + */ + public Data parseDataDocumentDb(JsonObject inputJSON) { + Data data = new Data(); + try { + Construct construct = parseAsConstructDocumentDb(inputJSON); + if (construct != null) { + data.setConstruct(construct); + + if (construct.getFullSql() == null) { + construct.setFullSql(Constants.UNKNOWN_STRING); + } + if (construct.getRedactedSensitiveDataSql() == null) { + construct.setRedactedSensitiveDataSql(Constants.UNKNOWN_STRING); + } + } + } catch (Exception e) { + log.error("DocumentDB filter: Error parsing JSon {}", inputJSON, e); + throw e; + } + return data; + } + + /** + * Creates an ExceptionRecord to be used in Record, instead of Data. + * + * @param data + * @param resultCode + * @return + */ + private ExceptionRecord parseException(JsonObject data, String resultCode) { + ExceptionRecord exceptionRecord = new ExceptionRecord(); + if (resultCode.equals(Constants.ERROR_CODE_UNAUTHORIZED)) { + exceptionRecord.setExceptionTypeId(Constants.EXCEPTION_TYPE_AUTHORIZATION); + exceptionRecord.setDescription("Unauthorized to perform the operation (13)"); + + } else if (resultCode.equals(Constants.ERROR_CODE_AUTH_FAILED)) { + exceptionRecord.setExceptionTypeId(Constants.EXCEPTION_TYPE_AUTHENTICATION); + exceptionRecord.setDescription("Authentication Failed (18)"); + } else { // prep for unknown error code + exceptionRecord.setExceptionTypeId(Constants.EXCEPTION_TYPE_AUTHORIZATION); + exceptionRecord.setDescription("Error (" + resultCode + ")"); + } + + exceptionRecord.setSqlString(data.getAsJsonObject(Constants.FIELD_PARAM).get(Constants.FIELD_MESSAGE).getAsString()); + return exceptionRecord; + } + + /** + * Creates Accessor object to be used in Guard Record + * + * @param data + * @return + */ + public Accessor parseAccessorDocumentDb(JsonObject data) { + Accessor accessor = new Accessor(); + + accessor.setDbProtocol(Constants.DATA_PROTOCOL); + accessor.setServerType(Constants.SERVER_TYPE); + String dbUsers = Constants.NOT_AVAILABLE; + if (data.has(Constants.FIELD_USER)) { + dbUsers = (data.get(Constants.FIELD_USER)==null || data.get(Constants.FIELD_USER).getAsString().isEmpty())?Constants.NOT_AVAILABLE:data.get(Constants.FIELD_USER).getAsString(); + }if(data.has(Constants.FIELD_PARAM) && data.get(Constants.FIELD_PARAM).getAsJsonObject().has(Constants.FIELD_USER)) { + String usr=data.get(Constants.FIELD_PARAM).getAsJsonObject().get(Constants.FIELD_USER).getAsString(); + dbUsers = (usr==null || usr.isEmpty())?Constants.NOT_AVAILABLE:usr; + } + accessor.setDbUser(dbUsers); + + String sourceProgram = Constants.UNKNOWN_STRING; + if (data.has(Constants.FIELD_APP_NAME)) { + sourceProgram = StringUtils.removeWhitespace(data.get(Constants.FIELD_APP_NAME).getAsString().trim()); + } + accessor.setSourceProgram(sourceProgram); + accessor.setServerHostName(Constants.DEFAULT_SERVER_HOSTNAME); + accessor.setLanguage(Accessor.LANGUAGE_FREE_TEXT_STRING); + accessor.setDataType(Accessor.DATA_TYPE_GUARDIUM_SHOULD_NOT_PARSE_SQL); + accessor.setClient_mac(Constants.UNKNOWN_STRING); + accessor.setClientHostName(Constants.UNKNOWN_STRING); + accessor.setClientOs(Constants.UNKNOWN_STRING); + accessor.setCommProtocol(Constants.UNKNOWN_STRING); + accessor.setDbProtocolVersion(Constants.UNKNOWN_STRING); + accessor.setOsUser(Constants.UNKNOWN_STRING); + accessor.setServerDescription(Constants.UNKNOWN_STRING); + accessor.setServerOs(Constants.UNKNOWN_STRING); + + String dbName = Constants.UNKNOWN_STRING; + + if (data != null && data.has(Constants.FIELD_NS)) { + final String ns = data.get(Constants.FIELD_NS).getAsString(); + dbName = StringUtils.extractDbNameFromNs(ns); + } + accessor.setServiceName(dbName); + + return accessor; + } + + /** + * Parses JSON object and returns session locator + * + * @param data + * @return + */ + private SessionLocator parseSessionLocatorDocumentDb(JsonObject data) { + SessionLocator sessionLocator = new SessionLocator(); + sessionLocator.setIpv6(false); + + sessionLocator.setClientIp(Constants.DEFAULT_IP); // Default value for Client IP + sessionLocator.setClientPort(SessionLocator.PORT_DEFAULT); + sessionLocator.setClientIpv6(Constants.UNKNOWN_STRING); + + String remote = data.has(Constants.FIELD_CLIENT) ? data.get(Constants.FIELD_CLIENT).getAsString() + : (data.has(Constants.FIELD_REMOTE_IP) ? data.get(Constants.FIELD_REMOTE_IP).getAsString() : null); + if (remote != null && remote.indexOf(':') > -1) { + String[] remoteobjects = remote.split(":"); + + if(remoteobjects.length>1) { + sessionLocator.setClientIp(remoteobjects[0]); + } + sessionLocator.setClientPort(SessionLocator.PORT_DEFAULT); + } + sessionLocator.setServerIp(Constants.DEFAULT_IP); // In AWS databases setting this field to 0.0.0.0 + sessionLocator.setServerPort(SessionLocator.PORT_DEFAULT);// In AWS databases setting this field to -1 + return sessionLocator; + } + /** + * Attempts to extract partial information from a malformed audit log JSON string. + * This method uses lenient parsing to extract whatever fields are available, + * even if the JSON is not fully valid. This helps provide better error context + * in exception records. + * + * @param record The record to populate with extracted information + * @param auditLogJsonString The potentially malformed JSON string + */ + private void extractPartialInformation(Record record, String auditLogJsonString) { + if (auditLogJsonString == null || auditLogJsonString.trim().isEmpty()) { + return; + } + + try { + // Try lenient parsing - may succeed even with some malformed fields like invalid timestamp + JsonObject partialJson = com.google.gson.JsonParser.parseString(auditLogJsonString).getAsJsonObject(); + + // Check event type and use appropriate existing parsing method + String atype = partialJson.has(Constants.FIELD_ATYPE) + ? partialJson.get(Constants.FIELD_ATYPE).getAsString() : null; + + if (Constants.AUTH_TYPE_AUTHCHECK.equals(atype)) { + // Use existing authCheck parsing methods + record.setSessionLocator(parseSessionLocatorAuthCheck(partialJson)); + record.setAccessor(parseAccessorAuthCheck(partialJson)); + record.setAppUserName(extractAppUserName(partialJson)); + // Extract db name + JsonObject param = extractParamObject(partialJson); + if (param != null && param.has(Constants.FIELD_NS)) { + record.setDbName(StringUtils.extractDbNameFromNs(param.get(Constants.FIELD_NS).getAsString())); + } + } else if (atype != null) { + // Use existing audit parsing methods + record.setSessionLocator(parseSessionLocatorDocumentDb(partialJson)); + record.setAccessor(parseAccessorDocumentDb(partialJson)); + // Extract db name from param.ns + if (partialJson.has(Constants.FIELD_PARAM)) { + JsonObject param = partialJson.getAsJsonObject(Constants.FIELD_PARAM); + if (param.has(Constants.FIELD_NS)) { + record.setDbName(StringUtils.extractDbNameFromNs(param.get(Constants.FIELD_NS).getAsString())); + } + } + } else if (partialJson.has(Constants.FIELD_COMMAND)) { + // Use existing profiler parsing methods + record.setSessionLocator(parseSessionLocatorDocumentDb(partialJson)); + record.setAccessor(parseAccessorDocumentDb(partialJson)); + // Extract db name from ns + if (partialJson.has(Constants.FIELD_NS)) { + record.setDbName(StringUtils.extractDbNameFromNs(partialJson.get(Constants.FIELD_NS).getAsString())); + } + } + } catch (Exception e) { + // Silently fail - defaults will be used by buildExceptionRecord + log.debug("Could not extract partial information from malformed log", e); + } + } + + + private Record buildExceptionRecord( + Record record, String error, String auditLogJsonString, String exceptionType) { + if (record == null) { + record = new Record(); + } + // Always try to extract partial information from the JSON string + // This ensures fields like dbName are populated even when record has existing data + extractPartialInformation(record, auditLogJsonString); + + Data data = record.getData(); + ExceptionRecord exceptionRecord = new ExceptionRecord(); + + // Set exception type + exceptionRecord.setExceptionTypeId(exceptionType); + + // Determine SQL string based on whether data was partially parsed + if (data != null + && data.getOriginalSqlCommand() != null + && !data.getOriginalSqlCommand().isEmpty()) { + exceptionRecord.setSqlString(data.getOriginalSqlCommand()); + } else { + // Truncate audit log string if too long to avoid memory issues + String sqlString = auditLogJsonString; + if (sqlString != null && sqlString.length() > Constants.MAX_SQL_STRING_LENGTH) { + sqlString = StringUtils.truncate(sqlString, Constants.MAX_SQL_STRING_LENGTH, "... [truncated]"); + } + exceptionRecord.setSqlString(sqlString != null ? sqlString : Constants.UNKNOWN_STRING); + } + + // Clear data and set exception + record.setData(null); + exceptionRecord.setDescription(error != null ? error : "Unknown parsing error"); + record.setException(exceptionRecord); + + // Set default values for required fields + record.setDbName(ValidationUtils.getValueOrDefault(record.getDbName(), Constants.NOT_AVAILABLE)); + record.setAppUserName(ValidationUtils.getValueOrDefault(record.getAppUserName(), Constants.NOT_AVAILABLE)); + record.setSessionId(ValidationUtils.getValueOrDefault(record.getSessionId(), Constants.UNKNOWN_STRING)); + + // Ensure time is set (required field to avoid NullPointerException) + if (record.getTime() == null) { + record.setTime(new Time(System.currentTimeMillis(), 0, 0)); + } + + // Ensure accessor and session locator are properly initialized + Accessor accessor = getAccessor(record); + record.setAccessor(accessor); + + SessionLocator sessionLocator = getSessionLocator(record); + record.setSessionLocator(sessionLocator); + + return record; + } + + public Record parseRecordException(Record record, String error, String auditLogJsonString) { + // For backward compatibility, determine exception type based on data availability + Data data = record != null ? record.getData() : null; + if (data != null + && data.getOriginalSqlCommand() != null + && !data.getOriginalSqlCommand().isEmpty()) { + return buildExceptionRecord(record, error, auditLogJsonString, Constants.UC_PARSER_ERROR); + } else { + return buildExceptionRecord(record, error, auditLogJsonString, Constants.UC_AUDIT_ERROR); + } + } + + protected SessionLocator getSessionLocator(Record record) { + SessionLocator sessionLocator = record.getSessionLocator(); + if (sessionLocator == null) { + sessionLocator = new SessionLocator(); + sessionLocator.setIpv6(false); + } + + sessionLocator.setClientIp( + ValidationUtils.getValueOrDefault(sessionLocator.getClientIp(), Constants.DEFAULT_IP)); + sessionLocator.setClientIpv6( + ValidationUtils.getValueOrDefault(sessionLocator.getClientIpv6(), Constants.DEFAULT_IPV6)); + sessionLocator.setServerIp( + ValidationUtils.getValueOrDefault(sessionLocator.getServerIp(), Constants.DEFAULT_IP)); + sessionLocator.setServerIpv6( + ValidationUtils.getValueOrDefault(sessionLocator.getServerIpv6(), Constants.DEFAULT_IPV6)); + sessionLocator.setIpv6(Boolean.TRUE.equals(sessionLocator.isIpv6())); + sessionLocator.setServerPort(sessionLocator.getServerPort()); + sessionLocator.setClientPort(sessionLocator.getClientPort()); + return sessionLocator; + } + + @Deprecated + protected String getValueOrSetDefault(String value, String defaultValue) { + return ValidationUtils.getValueOrDefault(value, defaultValue); + } + + private Accessor getAccessor(Record record) { + Accessor accessor = record.getAccessor(); + if (accessor == null) accessor = new Accessor(); + accessor.setDbUser(ValidationUtils.getValueOrDefault(accessor.getDbUser(), Constants.NOT_AVAILABLE)); + accessor.setDbProtocol(Constants.DATA_PROTOCOL); + accessor.setServerType(Constants.SERVER_TYPE); + accessor.setServerHostName(ValidationUtils.getValueOrDefault(accessor.getServerHostName(), Constants.NOT_AVAILABLE)); + accessor.setLanguage(Accessor.LANGUAGE_FREE_TEXT_STRING); + accessor.setDataType(Accessor.DATA_TYPE_GUARDIUM_SHOULD_NOT_PARSE_SQL); + accessor.setServiceName(ValidationUtils.getValueOrDefault(accessor.getServiceName(), Constants.NOT_AVAILABLE)); + + // Set all remaining null fields to UNKNOWN_STRING to avoid null values in JSON output + accessor.setServerOs(ValidationUtils.getValueOrDefault(accessor.getServerOs(), Constants.UNKNOWN_STRING)); + accessor.setClientOs(ValidationUtils.getValueOrDefault(accessor.getClientOs(), Constants.UNKNOWN_STRING)); + accessor.setClientHostName(ValidationUtils.getValueOrDefault(accessor.getClientHostName(), Constants.UNKNOWN_STRING)); + accessor.setCommProtocol(ValidationUtils.getValueOrDefault(accessor.getCommProtocol(), Constants.UNKNOWN_STRING)); + accessor.setDbProtocolVersion(ValidationUtils.getValueOrDefault(accessor.getDbProtocolVersion(), Constants.UNKNOWN_STRING)); + accessor.setOsUser(ValidationUtils.getValueOrDefault(accessor.getOsUser(), Constants.UNKNOWN_STRING)); + accessor.setSourceProgram(ValidationUtils.getValueOrDefault(accessor.getSourceProgram(), Constants.UNKNOWN_STRING)); + accessor.setClient_mac(ValidationUtils.getValueOrDefault(accessor.getClient_mac(), Constants.UNKNOWN_STRING)); + accessor.setServerDescription(ValidationUtils.getValueOrDefault(accessor.getServerDescription(), Constants.UNKNOWN_STRING)); + + return accessor; + } } \ No newline at end of file diff --git a/filter-plugin/logstash-filter-documentdb-aws-guardium/src/main/java/com/ibm/guardium/documentdb/StringUtils.java b/filter-plugin/logstash-filter-documentdb-aws-guardium/src/main/java/com/ibm/guardium/documentdb/StringUtils.java new file mode 100644 index 000000000..1cfdaae8a --- /dev/null +++ b/filter-plugin/logstash-filter-documentdb-aws-guardium/src/main/java/com/ibm/guardium/documentdb/StringUtils.java @@ -0,0 +1,103 @@ +/* +Copyright 2022-2023 IBM Inc. All rights reserved +SPDX-License-Identifier: Apache-2.0 +*/ +package com.ibm.guardium.documentdb; + +/** + * Utility class for string manipulation operations in DocumentDB Guardium filter. + * Provides efficient methods for common string operations. + */ +public final class StringUtils { + + // Prevent instantiation + private StringUtils() { + throw new AssertionError("StringUtils class should not be instantiated"); + } + + /** + * Removes all whitespace from a string. + * More efficient than regex replaceAll for simple character removal. + * + * @param str The string to process + * @return String with all whitespace removed, or original if null/empty + */ + public static String removeWhitespace(String str) { + if (str == null || str.isEmpty()) { + return str; + } + + StringBuilder sb = new StringBuilder(str.length()); + for (int i = 0; i < str.length(); i++) { + char c = str.charAt(i); + if (!Character.isWhitespace(c)) { + sb.append(c); + } + } + return sb.toString(); + } + + /** + * Extracts database name from namespace string (e.g., "db.collection" -> "db"). + * Uses indexOf instead of regex split for better performance. + * + * @param ns The namespace string + * @return The database name, or empty string if invalid + */ + public static String extractDbNameFromNs(String ns) { + if (ns == null || ns.isEmpty()) { + return Constants.UNKNOWN_STRING; + } + int dotIndex = ns.indexOf('.'); + return dotIndex > 0 ? ns.substring(0, dotIndex) : ns; + } + + /** + * Extracts collection name from namespace string (e.g., "db.collection" -> "collection"). + * Uses indexOf instead of regex split for better performance. + * + * @param ns The namespace string + * @return The collection name, or original string if no dot found + */ + public static String extractCollectionFromNs(String ns) { + if (ns == null || ns.isEmpty()) { + return ns; + } + int dotIndex = ns.indexOf('.'); + return dotIndex > 0 ? ns.substring(dotIndex + 1) : ns; + } + + /** + * Truncates a string to the specified maximum length and appends a suffix. + * + * @param str The string to truncate + * @param maxLength The maximum length + * @param suffix The suffix to append (e.g., "... [truncated]") + * @return The truncated string with suffix, or original if shorter than maxLength + */ + public static String truncate(String str, int maxLength, String suffix) { + if (str == null || str.length() <= maxLength) { + return str; + } + return str.substring(0, maxLength) + suffix; + } + + /** + * Checks if a message contains any of the profiler keys. + * More efficient than multiple contains() calls. + * + * @param message The message to check + * @return true if any profiler key is found, false otherwise + */ + public static boolean containsAnyProfilerKey(String message) { + if (message == null || message.isEmpty()) { + return false; + } + for (String key : Constants.PROFILER_KEYS) { + if (message.contains(key)) { + return true; + } + } + return false; + } +} diff --git a/filter-plugin/logstash-filter-documentdb-aws-guardium/src/main/java/com/ibm/guardium/documentdb/ValidationUtils.java b/filter-plugin/logstash-filter-documentdb-aws-guardium/src/main/java/com/ibm/guardium/documentdb/ValidationUtils.java new file mode 100644 index 000000000..f987b376d --- /dev/null +++ b/filter-plugin/logstash-filter-documentdb-aws-guardium/src/main/java/com/ibm/guardium/documentdb/ValidationUtils.java @@ -0,0 +1,88 @@ +/* +Copyright 2022-2023 IBM Inc. All rights reserved +SPDX-License-Identifier: Apache-2.0 +*/ +package com.ibm.guardium.documentdb; + +import org.apache.commons.validator.routines.InetAddressValidator; + +/** + * Utility class for validation operations in DocumentDB Guardium filter. + * Provides methods for JSON validation, IP address validation, and other common checks. + */ +public final class ValidationUtils { + + private static final InetAddressValidator INET_ADDRESS_VALIDATOR = InetAddressValidator.getInstance(); + + // Prevent instantiation + private ValidationUtils() { + throw new AssertionError("ValidationUtils class should not be instantiated"); + } + + /** + * Checks if a JSON string is properly closed with matching braces/brackets. + * + * @param json The JSON string to validate + * @return true if the JSON has matching opening and closing braces/brackets, false otherwise + */ + public static boolean isProperlyClosedJson(String json) { + if (json == null || json.isEmpty()) { + return false; + } + + String trimmed = json.trim(); + if (trimmed.isEmpty()) { + return false; + } + + char first = trimmed.charAt(0); + char last = trimmed.charAt(trimmed.length() - 1); + + // Check if starts with { or [ and ends with matching } or ] + return (first == '{' && last == '}') || (first == '[' && last == ']'); + } + + /** + * Validates if a string is a valid IP address (IPv4 or IPv6). + * + * @param ip The IP address string to validate + * @return true if valid IP address, false otherwise + */ + public static boolean isValidIpAddress(String ip) { + return ip != null && INET_ADDRESS_VALIDATOR.isValid(ip); + } + + /** + * Checks if the IP address is a DocumentDB internal command IP. + * This includes local IPs (127.0.0.1, ::1) and the special "(NONE)" marker. + * + * @param ip The IP address to check + * @return true if it's an internal/local IP, false otherwise + */ + public static boolean isDocumentInternalCommandIp(String ip) { + return ip != null && + (Constants.LOCAL_IP_LIST.contains(ip) || + ip.trim().equalsIgnoreCase(Constants.DOCUMENT_INTERNAL_API_IP)); + } + + /** + * Checks if a string is null or empty. + * + * @param str The string to check + * @return true if null or empty, false otherwise + */ + public static boolean isNullOrEmpty(String str) { + return str == null || str.isEmpty(); + } + + /** + * Returns the provided value if not null/empty, otherwise returns the default value. + * + * @param value The value to check + * @param defaultValue The default value to return if value is null/empty + * @return The value or default value + */ + public static String getValueOrDefault(String value, String defaultValue) { + return (value != null && !value.isEmpty()) ? value : defaultValue; + } +} diff --git a/filter-plugin/logstash-filter-documentdb-aws-guardium/src/test/java/com/ibm/guardium/documentdb/AuthCheckParserTest.java b/filter-plugin/logstash-filter-documentdb-aws-guardium/src/test/java/com/ibm/guardium/documentdb/AuthCheckParserTest.java new file mode 100644 index 000000000..eb1e2a488 --- /dev/null +++ b/filter-plugin/logstash-filter-documentdb-aws-guardium/src/test/java/com/ibm/guardium/documentdb/AuthCheckParserTest.java @@ -0,0 +1,337 @@ +package com.ibm.guardium.documentdb; + +import static org.junit.jupiter.api.Assertions.*; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.logstash.plugins.ContextImpl; + +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.ibm.guardium.universalconnector.commons.GuardConstants; +import com.ibm.guardium.universalconnector.commons.structures.Record; + +import co.elastic.logstash.api.Context; +import co.elastic.logstash.api.Event; +import co.elastic.logstash.api.FilterMatchListener; + +/** + * Test class for authCheck audit log parsing in DocumentDB Guardium filter. + * Tests the parsing of authCheck events from CSV file containing 200 real audit logs. + */ +public class AuthCheckParserTest { + + private static final Context context = new ContextImpl(null, null); + private static final Gson gson = new Gson(); + + /** + * Helper method to read authCheck audit logs from CSV file + */ + private List readAuditLogsFromCSV() throws IOException { + List auditLogs = new ArrayList<>(); + + try (InputStream is = getClass().getClassLoader() + .getResourceAsStream("documentdb/authcheck_audit_logs.csv"); + BufferedReader reader = new BufferedReader(new InputStreamReader(is))) { + + // Skip header line + reader.readLine(); + + String line; + while ((line = reader.readLine()) != null) { + // Split by comma, but handle the JSON message field which contains commas + int firstComma = line.indexOf(','); + if (firstComma > 0) { + String message = line.substring(firstComma + 1); + // Remove surrounding quotes if present + if (message.startsWith("\"") && message.endsWith("\"")) { + message = message.substring(1, message.length() - 1); + } + // Unescape double quotes + message = message.replace("\"\"", "\""); + auditLogs.add(message); + } + } + } + + return auditLogs; + } + + /** + * Helper method to parse a specific record by index + */ + private Record parseRecordByIndex(int index) throws Exception { + List auditLogs = readAuditLogsFromCSV(); + assertTrue(index > 0 && index <= auditLogs.size(), + "Index " + index + " out of range. Total records: " + auditLogs.size()); + + String auditLog = auditLogs.get(index - 1); // Convert to 0-based index + + DocumentdbGuardiumFilter filter = new DocumentdbGuardiumFilter("test-id", null, context); + Event e = new org.logstash.Event(); + TestMatchListener matchListener = new TestMatchListener(); + e.setField("message", auditLog); + e.setField("serverHostnamePrefix", "test-cluster"); + + Collection results = filter.filter(Collections.singletonList(e), matchListener); + + assertEquals(1, results.size(), "Should have 1 result"); + assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME), + "GuardRecord should not be null"); + + String recordString = e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME).toString(); + return gson.fromJson(recordString, Record.class); + } + + /** + * Main test: Parse all 200 authCheck audit logs from CSV + */ + @Test + public void testParseMultipleAuthCheckAuditLogs() throws Exception { + List auditLogs = readAuditLogsFromCSV(); + + assertEquals(200, auditLogs.size(), "Should have 200 audit logs in CSV"); + + DocumentdbGuardiumFilter filter = new DocumentdbGuardiumFilter("test-id", null, context); + TestMatchListener matchListener = new TestMatchListener(); + + int successCount = 0; + List errors = new ArrayList<>(); + + for (int i = 0; i < auditLogs.size(); i++) { + String auditLog = auditLogs.get(i); + try { + Event e = new org.logstash.Event(); + e.setField("message", auditLog); + e.setField("serverHostnamePrefix", "test-cluster"); + + Collection results = filter.filter(Collections.singletonList(e), matchListener); + + assertNotNull(results, "Results should not be null for record " + (i + 1)); + assertEquals(1, results.size(), "Should have 1 result for record " + (i + 1)); + + Object guardRecord = e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME); + assertNotNull(guardRecord, "GuardRecord should not be null for record " + (i + 1)); + + // Verify it's valid JSON + String recordString = guardRecord.toString(); + Record record = gson.fromJson(recordString, Record.class); + assertNotNull(record, "Parsed record should not be null for record " + (i + 1)); + + successCount++; + } catch (Exception ex) { + errors.add("Record " + (i + 1) + ": " + ex.getMessage()); + } + } + + if (!errors.isEmpty()) { + fail("Failed to parse " + errors.size() + " records:\n" + String.join("\n", errors)); + } + + assertEquals(200, successCount, "All 200 records should parse successfully"); + assertEquals(200, matchListener.getMatchCount(), "All 200 records should match"); + } + + /** + * Test parsing of record #7 - validates specific field values + */ + @Test + public void testAuthCheckRecord7() throws Exception { + Record record = parseRecordByIndex(7); + + assertNotNull(record); + assertEquals("readwrite", record.getAccessor().getDbUser()); + // dbName includes serverHostnamePrefix in Logstash version + assertEquals("test-cluster:admin", record.getDbName()); + assertNotNull(record.getSessionLocator()); + assertEquals("203.0.113.10", record.getSessionLocator().getClientIp()); + // Verify port was parsed (actual value may vary per record) + assertTrue(record.getSessionLocator().getClientPort() > 0, + "Client port should be parsed and positive"); + } + + /** + * Test parsing of record #20 + */ + @Test + public void testAuthCheckRecord20() throws Exception { + Record record = parseRecordByIndex(20); + + assertNotNull(record); + assertEquals("readwrite", record.getAccessor().getDbUser()); + assertNotNull(record.getData()); + assertNotNull(record.getData().getConstruct()); + } + + /** + * Test parsing of record #25 + */ + @Test + public void testAuthCheckRecord25() throws Exception { + Record record = parseRecordByIndex(25); + + assertNotNull(record); + assertEquals("readwrite", record.getAccessor().getDbUser()); + assertEquals("203.0.113.10", record.getSessionLocator().getClientIp()); + } + + /** + * Test parsing of record #30 + */ + @Test + public void testAuthCheckRecord30() throws Exception { + Record record = parseRecordByIndex(30); + + assertNotNull(record); + assertNotNull(record.getAccessor()); + assertNotNull(record.getSessionLocator()); + } + + /** + * Test parsing of record #54 + */ + @Test + public void testAuthCheckRecord54() throws Exception { + Record record = parseRecordByIndex(54); + + assertNotNull(record); + assertEquals("readwrite", record.getAccessor().getDbUser()); + } + + /** + * Test parsing of record #167 + */ + @Test + public void testAuthCheckRecord167() throws Exception { + Record record = parseRecordByIndex(167); + + assertNotNull(record); + assertNotNull(record.getData()); + } + + /** + * Test parsing of record #200 (last record) + */ + @Test + public void testAuthCheckRecord200() throws Exception { + Record record = parseRecordByIndex(200); + + assertNotNull(record); + assertEquals("readwrite", record.getAccessor().getDbUser()); + assertNotNull(record.getSessionLocator()); + } + + /** + * Test that authCheck events are not skipped + */ + @Test + public void testAuthCheckNotSkipped() throws Exception { + String authCheckLog = "{\"atype\":\"authCheck\",\"ts\":1779145197084," + + "\"timestamp_utc\":\"2026-05-18 22:59:57.084\",\"remote_ip\":\"203.0.113.10:10862\"," + + "\"users\":[{\"user\":\"readwrite\",\"db\":\"admin\"}]," + + "\"param\":{\"command\":\"find\",\"ns\":\"admin.test\"," + + "\"args\":{\"find\":\"test\"},\"result\":0}}"; + + DocumentdbGuardiumFilter filter = new DocumentdbGuardiumFilter("test-id", null, context); + Event e = new org.logstash.Event(); + TestMatchListener matchListener = new TestMatchListener(); + e.setField("message", authCheckLog); + + Collection results = filter.filter(Collections.singletonList(e), matchListener); + + assertEquals(1, results.size(), "authCheck event should not be skipped"); + assertEquals(1, matchListener.getMatchCount(), "authCheck event should match"); + } + + /** + * Test client port parsing from remote_ip field + */ + @Test + public void testClientPortParsing() throws Exception { + Record record = parseRecordByIndex(1); + + assertNotNull(record); + assertNotNull(record.getSessionLocator()); + // Verify port was parsed correctly from remote_ip + assertTrue(record.getSessionLocator().getClientPort() > 0, + "Client port should be parsed from remote_ip"); + } + + /** + * Test that users array is correctly parsed for app user name + */ + @Test + public void testUsersArrayParsing() throws Exception { + Record record = parseRecordByIndex(1); + + assertNotNull(record); + assertEquals("readwrite", record.getAccessor().getDbUser(), + "User should be extracted from users array"); + } + + /** + * Test database name extraction from param.ns + */ + @Test + public void testDatabaseNameExtraction() throws Exception { + Record record = parseRecordByIndex(1); + + assertNotNull(record); + // dbName includes serverHostnamePrefix in Logstash version + assertEquals("test-cluster:admin", record.getDbName(), + "Database name should be extracted from param.ns and prefixed with serverHostnamePrefix"); + } + + /** + * Test that Data construct is properly created + */ + @Test + public void testDataConstructCreation() throws Exception { + Record record = parseRecordByIndex(1); + + assertNotNull(record); + assertNotNull(record.getData(), "Data should not be null"); + assertNotNull(record.getData().getConstruct(), "Construct should not be null"); + assertNotNull(record.getData().getConstruct().getFullSql(), + "FullSql should not be null"); + } + + /** + * Test that timestamp is correctly parsed + */ + @Test + public void testTimestampParsing() throws Exception { + Record record = parseRecordByIndex(1); + + assertNotNull(record); + assertNotNull(record.getTime(), "Time should not be null"); + assertTrue(record.getTime().getTimstamp() > 0, "Timestamp should be positive"); + } + + /** + * Inner class for test match listener + */ + private static class TestMatchListener implements FilterMatchListener { + private int matchCount = 0; + + @Override + public void filterMatched(Event event) { + matchCount++; + } + + public int getMatchCount() { + return matchCount; + } + } +} + +// Made with Bob diff --git a/filter-plugin/logstash-filter-documentdb-aws-guardium/src/test/java/com/ibm/guardium/documentdb/ConstantsTest.java b/filter-plugin/logstash-filter-documentdb-aws-guardium/src/test/java/com/ibm/guardium/documentdb/ConstantsTest.java new file mode 100644 index 000000000..90ffed156 --- /dev/null +++ b/filter-plugin/logstash-filter-documentdb-aws-guardium/src/test/java/com/ibm/guardium/documentdb/ConstantsTest.java @@ -0,0 +1,205 @@ +package com.ibm.guardium.documentdb; + +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.*; + +/** + * Unit tests for Constants class. + * Verifies that all constants are properly defined and accessible. + */ +public class ConstantsTest { + + @Test + public void testProtocolConstants() { + assertEquals("DocumentDB", Constants.DATA_PROTOCOL); + assertEquals("DocumentDB", Constants.SERVER_TYPE); + } + + @Test + public void testDefaultValues() { + assertEquals("", Constants.UNKNOWN_STRING); + assertEquals("N.A.", Constants.NOT_AVAILABLE); + assertEquals("0.0.0.0", Constants.DEFAULT_IP); + assertEquals("0000:0000:0000:0000:0000:FFFF:0000:0000", Constants.DEFAULT_IPV6); + assertEquals("[json-object]", Constants.COMPOUND_OBJECT); + assertEquals("(NONE)", Constants.DOCUMENT_INTERNAL_API_IP); + } + + @Test + public void testExceptionTypes() { + assertEquals("SQL_ERROR", Constants.EXCEPTION_TYPE_AUTHORIZATION); + assertEquals("LOGIN_FAILED", Constants.EXCEPTION_TYPE_AUTHENTICATION); + assertEquals("UC_PARSER_ERROR", Constants.UC_PARSER_ERROR); + assertEquals("UC_AUDIT_ERROR", Constants.UC_AUDIT_ERROR); + } + + @Test + public void testLogstashTags() { + assertEquals("_documentdbguardium_json_parse_error", Constants.LOGSTASH_TAG_JSON_PARSE_ERROR); + assertEquals("_documentdbguardium_json_depth_error", Constants.LOGSTASH_TAG_JSON_DEPTH_ERROR); + assertEquals("_documentdbguardium_skip", Constants.LOGSTASH_TAG_SKIP); + } + + @Test + public void testDocumentDBSignals() { + assertEquals("\"atype\"", Constants.DOCUMENTDB_AUDIT_SIGNAL); + assertEquals("command", Constants.DOCUMENTDB_PROFILER_SIGNAL); + } + + @Test + public void testOperationKeys() { + assertEquals("aggregate", Constants.AGGR_KEY); + assertEquals("count", Constants.COUNT_KEY); + assertEquals("remove", Constants.DELETE_KEY); + assertEquals("insert", Constants.INSERT_KEY); + assertEquals("update", Constants.UPDATE_KEY); + assertEquals("distinct", Constants.DISTINCT_KEY); + assertEquals("find", Constants.FIND_KEY); + assertEquals("findAndModify", Constants.FINDANDMODIFY_KEY); + } + + @Test + public void testProfilerKeysSet() { + assertNotNull(Constants.PROFILER_KEYS); + assertEquals(8, Constants.PROFILER_KEYS.size()); + assertTrue(Constants.PROFILER_KEYS.contains("aggregate")); + assertTrue(Constants.PROFILER_KEYS.contains("count")); + assertTrue(Constants.PROFILER_KEYS.contains("remove")); + assertTrue(Constants.PROFILER_KEYS.contains("insert")); + assertTrue(Constants.PROFILER_KEYS.contains("update")); + assertTrue(Constants.PROFILER_KEYS.contains("distinct")); + assertTrue(Constants.PROFILER_KEYS.contains("find")); + assertTrue(Constants.PROFILER_KEYS.contains("findAndModify")); + } + + @Test + public void testLocalIPList() { + assertNotNull(Constants.LOCAL_IP_LIST); + assertEquals(2, Constants.LOCAL_IP_LIST.size()); + assertTrue(Constants.LOCAL_IP_LIST.contains("127.0.0.1")); + assertTrue(Constants.LOCAL_IP_LIST.contains("0:0:0:0:0:0:0:1")); + } + + @Test + public void testRedactionIgnoreStrings() { + assertNotNull(Constants.REDACTION_IGNORE_STRINGS); + assertEquals(6, Constants.REDACTION_IGNORE_STRINGS.size()); + assertTrue(Constants.REDACTION_IGNORE_STRINGS.contains("from")); + assertTrue(Constants.REDACTION_IGNORE_STRINGS.contains("localField")); + assertTrue(Constants.REDACTION_IGNORE_STRINGS.contains("foreignField")); + assertTrue(Constants.REDACTION_IGNORE_STRINGS.contains("as")); + assertTrue(Constants.REDACTION_IGNORE_STRINGS.contains("connectFromField")); + assertTrue(Constants.REDACTION_IGNORE_STRINGS.contains("connectToField")); + } + + @Test + public void testFieldNames() { + assertEquals("atype", Constants.FIELD_ATYPE); + assertEquals("param", Constants.FIELD_PARAM); + assertEquals("ns", Constants.FIELD_NS); + assertEquals("user", Constants.FIELD_USER); + assertEquals("error", Constants.FIELD_ERROR); + assertEquals("message", Constants.FIELD_MESSAGE); + assertEquals("command", Constants.FIELD_COMMAND); + assertEquals("ts", Constants.FIELD_TS); + assertEquals("client", Constants.FIELD_CLIENT); + assertEquals("remote_ip", Constants.FIELD_REMOTE_IP); + assertEquals("appName", Constants.FIELD_APP_NAME); + assertEquals("op", Constants.FIELD_OP); + } + + @Test + public void testEventFieldNames() { + assertEquals("message", Constants.EVENT_FIELD_MESSAGE); + assertEquals("server_ip", Constants.EVENT_FIELD_SERVER_IP); + assertEquals("serverHostnamePrefix", Constants.EVENT_FIELD_SERVER_HOSTNAME_PREFIX); + } + + @Test + public void testAuthenticationTypes() { + assertEquals("authenticate", Constants.AUTH_TYPE_AUTHENTICATE); + assertEquals("createRole", Constants.AUTH_TYPE_CREATE_ROLE); + assertEquals("dropRole", Constants.AUTH_TYPE_DROP_ROLE); + } + + @Test + public void testErrorCodes() { + assertEquals("0", Constants.ERROR_CODE_SUCCESS); + assertEquals("13", Constants.ERROR_CODE_UNAUTHORIZED); + assertEquals("18", Constants.ERROR_CODE_AUTH_FAILED); + } + + @Test + public void testServerConfiguration() { + assertEquals(".aws.com", Constants.SERVER_HOSTNAME_SUFFIX); + assertEquals("documentdb.amazonaws.com", Constants.DEFAULT_SERVER_HOSTNAME); + } + + @Test + public void testLimitsAndThresholds() { + assertEquals(10000, Constants.MAX_SQL_STRING_LENGTH); + assertEquals(256, Constants.STRING_BUILDER_INITIAL_CAPACITY); + } + + @Test + public void testErrorMessages() { + assertNotNull(Constants.ERROR_JSON_VALIDATION_FAILED); + assertNotNull(Constants.ERROR_INVALID_AUTHENTICATE_LOG); + assertNotNull(Constants.ERROR_JSON_NESTING_TOO_DEEP); + assertNotNull(Constants.ERROR_INSUFFICIENT_MEMORY); + assertNotNull(Constants.ERROR_PARSING_AUDIT_EVENT); + assertNotNull(Constants.ERROR_PARSING_PROFILER_EVENT); + assertNotNull(Constants.ERROR_MISSING_DB_NAME); + assertNotNull(Constants.ERROR_FAILED_TO_SERIALIZE_EVENT); + + assertTrue(Constants.ERROR_JSON_VALIDATION_FAILED.contains("JSON")); + assertTrue(Constants.ERROR_INVALID_AUTHENTICATE_LOG.contains("authenticate")); + assertTrue(Constants.ERROR_JSON_NESTING_TOO_DEEP.contains("nesting")); + assertTrue(Constants.ERROR_INSUFFICIENT_MEMORY.contains("memory")); + assertTrue(Constants.ERROR_PARSING_AUDIT_EVENT.contains("audit")); + assertTrue(Constants.ERROR_PARSING_PROFILER_EVENT.contains("profiler")); + assertTrue(Constants.ERROR_MISSING_DB_NAME.contains("DB name")); + } + + @Test + public void testConstantsClassCannotBeInstantiated() { + // Verify that the Constants class cannot be instantiated + // This is a design pattern test to ensure utility class is properly designed + try { + java.lang.reflect.Constructor constructor = Constants.class.getDeclaredConstructor(); + constructor.setAccessible(true); + constructor.newInstance(); + fail("Expected AssertionError to be thrown"); + } catch (Exception e) { + // Expected - constructor should throw AssertionError + assertTrue(e.getCause() instanceof AssertionError); + } + } + + @Test + public void testSetsAreImmutable() { + // Verify that the sets cannot be modified + try { + Constants.PROFILER_KEYS.add("newKey"); + fail("Expected UnsupportedOperationException"); + } catch (UnsupportedOperationException e) { + // Expected - sets should be immutable + } + + try { + Constants.LOCAL_IP_LIST.add("192.168.1.1"); + fail("Expected UnsupportedOperationException"); + } catch (UnsupportedOperationException e) { + // Expected - sets should be immutable + } + + try { + Constants.REDACTION_IGNORE_STRINGS.add("newField"); + fail("Expected UnsupportedOperationException"); + } catch (UnsupportedOperationException e) { + // Expected - sets should be immutable + } + } +} + +// Made with Bob diff --git a/filter-plugin/logstash-filter-documentdb-aws-guardium/src/test/java/com/ibm/guardium/documentdb/ErrorHandlingTest.java b/filter-plugin/logstash-filter-documentdb-aws-guardium/src/test/java/com/ibm/guardium/documentdb/ErrorHandlingTest.java new file mode 100644 index 000000000..e2358c768 --- /dev/null +++ b/filter-plugin/logstash-filter-documentdb-aws-guardium/src/test/java/com/ibm/guardium/documentdb/ErrorHandlingTest.java @@ -0,0 +1,187 @@ +/* + * Copyright 2022-2023 IBM Inc. All rights reserved + * SPDX-License-Identifier: Apache-2.0 + */ +package com.ibm.guardium.documentdb; + +import com.ibm.guardium.universalconnector.commons.structures.Construct; +import com.ibm.guardium.universalconnector.commons.structures.Data; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.*; + +/** + * Test class for error handling in DocumentDB parser. + * Tests UC_AUDIT_ERROR and UC_PARSER_ERROR scenarios. + */ +public class ErrorHandlingTest { + + private final Parser parser = new Parser(); + + @Test + public void testUCAuditError_WithNullRecord() { + // Test UC_AUDIT_ERROR when record is null + String error = Constants.ERROR_PARSING_AUDIT_EVENT; + String jsonString = "{\"atype\":\"authCheck\",\"ts\":\"invalid_timestamp\",\"remote_ip\":\"203.0.113.10:10862\",\"users\":[{\"user\":\"testuser\",\"db\":\"admin\"}],\"param\":{\"command\":\"find\",\"ns\":\"admin.test_collection\"}}"; + + Record result = parser.parseRecordException(null, error, jsonString); + + // Verify it's UC_AUDIT_ERROR + assertNotNull(result, "Result should not be null"); + assertNotNull(result.getException(), "Exception should not be null"); + assertEquals(Constants.UC_AUDIT_ERROR, result.getException().getExceptionTypeId(), "Should be UC_AUDIT_ERROR"); + assertTrue(result.getException().getDescription().contains(error), "Description should contain error message"); + assertNotNull(result.getException().getSqlString(), "SQL string should not be null"); + + // Verify partial information was extracted + assertEquals("admin", result.getDbName(), "DB name should be extracted"); + assertEquals("testuser", result.getAppUserName(), "App user name should be extracted"); + assertEquals("203.0.113.10", result.getSessionLocator().getClientIp(), "Client IP should be extracted"); + assertEquals(10862, result.getSessionLocator().getClientPort(), "Client port should be extracted"); + + // Verify accessor fields are not null + assertNotNull(result.getAccessor(), "Accessor should not be null"); + assertEquals("testuser", result.getAccessor().getDbUser(), "DB user should be extracted"); + assertEquals("admin", result.getAccessor().getServiceName(), "Service name should be extracted"); + assertEquals(Constants.UNKNOWN_STRING, result.getAccessor().getServerOs(), "Server OS should be empty string"); + assertEquals(Constants.UNKNOWN_STRING, result.getAccessor().getClientOs(), "Client OS should be empty string"); + assertEquals(Constants.UNKNOWN_STRING, result.getAccessor().getSourceProgram(), "Source program should be empty string"); + } + + @Test + public void testUCAuditError_WithMalformedJSON() { + // Test UC_AUDIT_ERROR with completely malformed JSON + String error = Constants.ERROR_PARSING_AUDIT_EVENT; + String jsonString = "{\"atype\":\"authCheck\",\"malformed_structure\":}"; + + Record result = parser.parseRecordException(null, error, jsonString); + + // Verify it's UC_AUDIT_ERROR + assertNotNull(result, "Result should not be null"); + assertNotNull(result.getException(), "Exception should not be null"); + assertEquals(Constants.UC_AUDIT_ERROR, result.getException().getExceptionTypeId(), "Should be UC_AUDIT_ERROR"); + + // Verify default values are used when extraction fails + assertEquals(Constants.NOT_AVAILABLE, result.getDbName(), "DB name should be N.A."); + assertEquals(Constants.NOT_AVAILABLE, result.getAppUserName(), "App user name should be N.A."); + assertEquals(Constants.DEFAULT_IP, result.getSessionLocator().getClientIp(), "Client IP should be default"); + + // Verify accessor fields are not null + assertNotNull(result.getAccessor(), "Accessor should not be null"); + assertEquals(Constants.UNKNOWN_STRING, result.getAccessor().getServerOs(), "Server OS should be empty string"); + assertEquals(Constants.UNKNOWN_STRING, result.getAccessor().getClientOs(), "Client OS should be empty string"); + } + + @Test + public void testUCParserError_WithDataSet() { + // Test UC_PARSER_ERROR when record has data with originalSqlCommand + Record record = new Record(); + Data data = new Data(); + Construct construct = new Construct(); + construct.setFullSql("test sql command"); + data.setConstruct(construct); + data.setOriginalSqlCommand("test sql command"); + record.setData(data); + + String error = "Test parser error"; + String jsonString = "{\"atype\":\"authCheck\",\"ts\":\"123456789\",\"remote_ip\":\"192.168.1.1:5000\",\"users\":[{\"user\":\"parseruser\",\"db\":\"testdb\"}],\"param\":{\"command\":\"update\",\"ns\":\"testdb.collection\"}}"; + + Record result = parser.parseRecordException(record, error, jsonString); + + // Verify it's UC_PARSER_ERROR + assertNotNull(result, "Result should not be null"); + assertNotNull(result.getException(), "Exception should not be null"); + assertEquals(Constants.UC_PARSER_ERROR, result.getException().getExceptionTypeId(), "Should be UC_PARSER_ERROR"); + assertEquals("test sql command", result.getException().getSqlString(), "SQL string should be from original data"); + assertTrue(result.getException().getDescription().contains(error), "Description should contain error message"); + + // Verify partial information was still extracted from JSON + assertEquals("testdb", result.getDbName(), "DB name should be extracted"); + assertEquals("parseruser", result.getAppUserName(), "App user name should be extracted"); + assertEquals("192.168.1.1", result.getSessionLocator().getClientIp(), "Client IP should be extracted"); + assertEquals(5000, result.getSessionLocator().getClientPort(), "Client port should be extracted"); + + // Verify accessor fields are not null + assertNotNull(result.getAccessor(), "Accessor should not be null"); + assertEquals("parseruser", result.getAccessor().getDbUser(), "DB user should be extracted"); + assertEquals("testdb", result.getAccessor().getServiceName(), "Service name should be extracted"); + assertEquals(Constants.UNKNOWN_STRING, result.getAccessor().getServerOs(), "Server OS should be empty string"); + assertEquals(Constants.UNKNOWN_STRING, result.getAccessor().getClientOs(), "Client OS should be empty string"); + } + + @Test + public void testUCParserError_WithEmptyOriginalSqlCommand() { + // Test that empty originalSqlCommand results in UC_AUDIT_ERROR + Record record = new Record(); + Data data = new Data(); + data.setOriginalSqlCommand(""); + record.setData(data); + + String error = "Test error"; + String jsonString = "{\"atype\":\"authCheck\"}"; + + Record result = parser.parseRecordException(record, error, jsonString); + + // Should be UC_AUDIT_ERROR because originalSqlCommand is empty + assertEquals(Constants.UC_AUDIT_ERROR, result.getException().getExceptionTypeId(), "Should be UC_AUDIT_ERROR"); + } + + @Test + public void testErrorDescription_ContainsOriginalError() { + // Test that error description contains the original error message + String error = "Custom error message for testing"; + String jsonString = "{\"atype\":\"authCheck\"}"; + + Record result = parser.parseRecordException(null, error, jsonString); + + assertNotNull(result.getException(), "Exception should not be null"); + assertTrue(result.getException().getDescription().contains(error), "Description should contain original error"); + } + + @Test + public void testPartialExtraction_AuthCheckEvent() { + // Test partial extraction for authCheck event with various fields + String error = "Test error"; + String jsonString = "{\"atype\":\"authCheck\",\"ts\":\"invalid\",\"remote_ip\":\"10.0.0.1:8080\",\"users\":[{\"user\":\"admin\",\"db\":\"mydb\"}],\"param\":{\"command\":\"delete\",\"ns\":\"mydb.mycollection\"},\"appName\":\"MyApp\"}"; + + Record result = parser.parseRecordException(null, error, jsonString); + + // Verify all extractable fields were extracted + assertEquals("mydb", result.getDbName(), "DB name"); + assertEquals("admin", result.getAppUserName(), "App user name"); + assertEquals("10.0.0.1", result.getSessionLocator().getClientIp(), "Client IP"); + assertEquals(8080, result.getSessionLocator().getClientPort(), "Client port"); + assertEquals("admin", result.getAccessor().getDbUser(), "DB user"); + assertEquals("mydb", result.getAccessor().getServiceName(), "Service name"); + assertEquals("MyApp", result.getAccessor().getSourceProgram(), "Source program"); + } + + @Test + public void testPartialExtraction_AuditEvent() { + // Test partial extraction for regular audit event + String error = "Test error"; + String jsonString = "{\"atype\":\"createCollection\",\"ts\":\"invalid\",\"user\":\"dbadmin\",\"param\":{\"ns\":\"production.orders\"}}"; + + Record result = parser.parseRecordException(null, error, jsonString); + + // Verify extraction for audit event + assertEquals("production", result.getDbName(), "DB name"); + assertNotNull(result.getAccessor(), "Accessor should not be null"); + assertEquals("dbadmin", result.getAccessor().getDbUser(), "DB user"); + } + + @Test + public void testPartialExtraction_ProfilerEvent() { + // Test partial extraction for profiler event + String error = "Test error"; + String jsonString = "{\"command\":{\"find\":\"users\"},\"ns\":\"appdb.users\",\"user\":\"appuser\"}"; + + Record result = parser.parseRecordException(null, error, jsonString); + + // Verify extraction for profiler event + assertEquals("appdb", result.getDbName(), "DB name"); + assertNotNull(result.getAccessor(), "Accessor should not be null"); + } +} + +// Made with Bob diff --git a/filter-plugin/logstash-filter-documentdb-aws-guardium/src/test/java/com/ibm/guardium/documentdb/EventUtilsTest.java b/filter-plugin/logstash-filter-documentdb-aws-guardium/src/test/java/com/ibm/guardium/documentdb/EventUtilsTest.java new file mode 100644 index 000000000..b6dca42f9 --- /dev/null +++ b/filter-plugin/logstash-filter-documentdb-aws-guardium/src/test/java/com/ibm/guardium/documentdb/EventUtilsTest.java @@ -0,0 +1,180 @@ +package com.ibm.guardium.documentdb; + +import co.elastic.logstash.api.Event; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Unit tests for EventUtils class. + */ +public class EventUtilsTest { + + @Test + public void testLogEvent_ValidEvent() { + Event event = new org.logstash.Event(); + event.setField("key1", "value1"); + event.setField("key2", "value2"); + + String result = EventUtils.logEvent(event); + + assertNotNull(result); + assertTrue(result.contains("key1")); + assertTrue(result.contains("value1")); + assertTrue(result.contains("key2")); + assertTrue(result.contains("value2")); + assertTrue(result.startsWith("{")); + assertTrue(result.endsWith("}")); + } + + @Test + public void testLogEvent_EmptyEvent() { + Event event = new org.logstash.Event(); + + String result = EventUtils.logEvent(event); + + assertNotNull(result); + assertTrue(result.contains("{")); + assertTrue(result.contains("}")); + } + + @Test + public void testGetStringField_ValidString() { + Event event = new org.logstash.Event(); + event.setField("testField", "testValue"); + + String result = EventUtils.getStringField(event, "testField"); + + assertEquals("testValue", result); + } + + @Test + public void testGetStringField_NonExistentField() { + Event event = new org.logstash.Event(); + + String result = EventUtils.getStringField(event, "nonExistent"); + + assertNull(result); + } + + @Test + public void testGetStringField_NonStringField() { + Event event = new org.logstash.Event(); + event.setField("numberField", 123); + + String result = EventUtils.getStringField(event, "numberField"); + + // Should return string representation or null depending on implementation + assertNotNull(result); + } + + @Test + public void testGetValidatedEventServerIp_ValidIPv4() { + Event event = new org.logstash.Event(); + event.setField(Constants.EVENT_FIELD_SERVER_IP, "192.168.1.1"); + + String result = EventUtils.getValidatedEventServerIp(event); + + assertEquals("192.168.1.1", result); + } + + @Test + public void testGetValidatedEventServerIp_ValidIPv6() { + Event event = new org.logstash.Event(); + event.setField(Constants.EVENT_FIELD_SERVER_IP, "::1"); + + String result = EventUtils.getValidatedEventServerIp(event); + + assertEquals("::1", result); + } + + @Test + public void testGetValidatedEventServerIp_InvalidIP() { + Event event = new org.logstash.Event(); + event.setField(Constants.EVENT_FIELD_SERVER_IP, "not-an-ip"); + + String result = EventUtils.getValidatedEventServerIp(event); + + assertNull(result); + } + + @Test + public void testGetValidatedEventServerIp_NoField() { + Event event = new org.logstash.Event(); + + String result = EventUtils.getValidatedEventServerIp(event); + + assertNull(result); + } + + @Test + public void testGetMessageField_ValidMessage() { + Event event = new org.logstash.Event(); + event.setField(Constants.EVENT_FIELD_MESSAGE, "test message"); + + String result = EventUtils.getMessageField(event); + + assertEquals("test message", result); + } + + @Test + public void testGetMessageField_NoMessage() { + Event event = new org.logstash.Event(); + + String result = EventUtils.getMessageField(event); + + assertNull(result); + } + + @Test + public void testGetServerHostnamePrefix_ValidPrefix() { + Event event = new org.logstash.Event(); + event.setField(Constants.EVENT_FIELD_SERVER_HOSTNAME_PREFIX, "test-server"); + + String result = EventUtils.getServerHostnamePrefix(event); + + assertEquals("test-server", result); + } + + @Test + public void testGetServerHostnamePrefix_NoPrefix() { + Event event = new org.logstash.Event(); + + String result = EventUtils.getServerHostnamePrefix(event); + + assertNull(result); + } + + @Test + public void testGetStringField_NullFieldName() { + Event event = new org.logstash.Event(); + event.setField("test", "value"); + + String result = EventUtils.getStringField(event, null); + + assertNull(result); + } + + @Test + public void testLogEvent_WithSpecialCharacters() { + Event event = new org.logstash.Event(); + event.setField("key", "value with \"quotes\" and \n newlines"); + + String result = EventUtils.logEvent(event); + + assertNotNull(result); + assertTrue(result.contains("key")); + } + + @Test + public void testGetStringField_EmptyString() { + Event event = new org.logstash.Event(); + event.setField("emptyField", ""); + + String result = EventUtils.getStringField(event, "emptyField"); + + assertEquals("", result); + } +} + +// Made with Bob diff --git a/filter-plugin/logstash-filter-documentdb-aws-guardium/src/test/java/com/ibm/guardium/documentdb/MaliciousInputTest.java b/filter-plugin/logstash-filter-documentdb-aws-guardium/src/test/java/com/ibm/guardium/documentdb/MaliciousInputTest.java new file mode 100644 index 000000000..0bd667ee3 --- /dev/null +++ b/filter-plugin/logstash-filter-documentdb-aws-guardium/src/test/java/com/ibm/guardium/documentdb/MaliciousInputTest.java @@ -0,0 +1,224 @@ +package com.ibm.guardium.documentdb; + +import co.elastic.logstash.api.Context; +import co.elastic.logstash.api.Event; +import co.elastic.logstash.api.FilterMatchListener; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.logstash.plugins.ContextImpl; + +import java.util.Collection; +import java.util.Collections; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** Tests for malicious inputs that could cause infinite loops or performance issues */ +@Disabled +class MaliciousInputTest { + + static final Context context = new ContextImpl(null, null); + static final DocumentdbGuardiumFilter filter = + new DocumentdbGuardiumFilter("test-id", null, context); + + /** + * Test with extremely deeply nested JSON objects (10,000 levels) This tests Gson's recursion + * handling and toString() on deeply nested structures EXPECTED: StackOverflowError - this is a + * known vulnerability in Gson without depth limits + */ + @Test + @Timeout(value = 10, unit = TimeUnit.SECONDS) + void testExtremelyDeeplyNestedJson() { + StringBuilder deepJson = new StringBuilder(); + deepJson.append( + "{\"atype\":\"createDatabase\",\"ts\":1629364973923,\"remote_ip\":\"172.31.32.149:57308\",\"user\":\"test\",\"param\":{\"ns\":\"test\",\"data\":"); + + // Create 10,000 levels of nesting + for (int i = 0; i < 10000; i++) { + deepJson.append("{\"level").append(i).append("\":"); + } + deepJson.append("\"value\""); + for (int i = 0; i < 10000; i++) { + deepJson.append("}"); + } + deepJson.append("}}"); + + Event e = new org.logstash.Event(); + e.setField("message", deepJson.toString()); + TestMatchListener matchListener = new TestMatchListener(); + + // With JsonValidator, deeply nested JSON is now rejected before parsing + Collection results = filter.filter(Collections.singletonList(e), matchListener); + assertNotNull(results); + + // Event should be tagged as having excessive depth + Object tags = e.getField("tags"); + assertTrue( + tags != null && tags.toString().contains("_documentdbguardium_json_depth_error"), + "Deeply nested JSON should be detected and tagged by JsonValidator"); + System.out.println( + "✓ JsonValidator successfully prevented StackOverflowError by rejecting deeply nested JSON"); + } + + /** + * Test with command.toString() on extremely large nested object Lines 222, 240 call + * command.toString() which could be slow on large objects + */ + @Test + @Timeout(value = 10, unit = TimeUnit.SECONDS) + void testLargeNestedObjectToString() { + StringBuilder largeObject = new StringBuilder(); + largeObject.append( + "{\"atype\":\"createDatabase\",\"ts\":1629364973923,\"remote_ip\":\"172.31.32.149:57308\",\"user\":\"test\",\"param\":{"); + + // Create object with 10,000 fields + for (int i = 0; i < 10000; i++) { + if (i > 0) largeObject.append(","); + largeObject + .append("\"field") + .append(i) + .append("\":{\"nested\":{\"data\":\"value") + .append(i) + .append("\"}}"); + } + largeObject.append("}}"); + + Event e = new org.logstash.Event(); + e.setField("message", largeObject.toString()); + TestMatchListener matchListener = new TestMatchListener(); + + try { + Collection results = filter.filter(Collections.singletonList(e), matchListener); + assertNotNull(results); + } catch (Exception ex) { + assertTrue(true, "Parser handled large object: " + ex.getMessage()); + } + } + + /** Test with special regex characters in strings that could cause replaceAll issues */ + @Test + @Timeout(value = 5, unit = TimeUnit.SECONDS) + void testSpecialRegexCharacters() { + // Create appName with regex special characters repeated many times + StringBuilder specialChars = new StringBuilder(); + String regexChars = ".*+?^${}()|[]\\"; + for (int i = 0; i < 10000; i++) { + specialChars.append(regexChars); + } + + String json = + "{\"op\":\"command\",\"ts\":1641978528311,\"ns\":\"test.collection\",\"command\":{\"aggregate\":\"cases\"},\"client\":\"172.31.40.18:38230\",\"appName\":\"" + + specialChars.toString().replace("\\", "\\\\").replace("\"", "\\\"") + + "\",\"user\":\"test\"}"; + + Event e = new org.logstash.Event(); + e.setField("message", json); + TestMatchListener matchListener = new TestMatchListener(); + + try { + Collection results = filter.filter(Collections.singletonList(e), matchListener); + assertNotNull(results); + } catch (Exception ex) { + assertTrue(true, "Parser handled special characters: " + ex.getMessage()); + } + } + + /** + * Test with circular-looking references in param object This tests if toString() on param causes + * issues + */ + @Test + @Timeout(value = 5, unit = TimeUnit.SECONDS) + void testComplexParamObject() { + StringBuilder complexParam = new StringBuilder(); + complexParam.append( + "{\"atype\":\"createDatabase\",\"ts\":1629364973923,\"remote_ip\":\"172.31.32.149:57308\",\"user\":\"test\",\"param\":{"); + + // Create complex nested structure in param + for (int i = 0; i < 1000; i++) { + if (i > 0) complexParam.append(","); + complexParam.append("\"obj").append(i).append("\":{"); + for (int j = 0; j < 10; j++) { + if (j > 0) complexParam.append(","); + complexParam.append("\"field").append(j).append("\":\"value").append(j).append("\""); + } + complexParam.append("}"); + } + complexParam.append("}}"); + + Event e = new org.logstash.Event(); + e.setField("message", complexParam.toString()); + TestMatchListener matchListener = new TestMatchListener(); + + try { + Collection results = filter.filter(Collections.singletonList(e), matchListener); + assertNotNull(results); + } catch (Exception ex) { + assertTrue(true, "Parser handled complex param: " + ex.getMessage()); + } + } + + /** Test with Unicode and special characters that could cause string processing issues */ + @Test + @Timeout(value = 5, unit = TimeUnit.SECONDS) + void testUnicodeAndSpecialCharacters() { + StringBuilder unicode = new StringBuilder(); + // Add various Unicode characters, emojis, and special chars + for (int i = 0; i < 10000; i++) { + unicode.append("\\u0000\\u0001\\u0002\\uFFFF😀🔥💻"); + } + + String json = + "{\"atype\":\"createDatabase\",\"ts\":1629364973923,\"remote_ip\":\"172.31.32.149:57308\",\"user\":\"test\",\"param\":{\"ns\":\"test\",\"data\":\"" + + unicode.toString() + + "\"}}"; + + Event e = new org.logstash.Event(); + e.setField("message", json); + TestMatchListener matchListener = new TestMatchListener(); + + try { + Collection results = filter.filter(Collections.singletonList(e), matchListener); + assertNotNull(results); + } catch (Exception ex) { + assertTrue(true, "Parser handled Unicode: " + ex.getMessage()); + } + } + + /** Test with extremely long single field value */ + @Test + @Timeout(value = 5, unit = TimeUnit.SECONDS) + void testExtremelyLongFieldValue() { + StringBuilder longValue = new StringBuilder(); + // Create 1MB string + for (int i = 0; i < 1000000; i++) { + longValue.append("x"); + } + + String json = + "{\"atype\":\"createDatabase\",\"ts\":1629364973923,\"remote_ip\":\"172.31.32.149:57308\",\"user\":\"test\",\"param\":{\"ns\":\"test\",\"data\":\"" + + longValue.toString() + + "\"}}"; + + Event e = new org.logstash.Event(); + e.setField("message", json); + TestMatchListener matchListener = new TestMatchListener(); + + try { + Collection results = filter.filter(Collections.singletonList(e), matchListener); + assertNotNull(results); + } catch (Exception ex) { + assertTrue(true, "Parser handled long value: " + ex.getMessage()); + } + } + + // Helper class for test listener + private static class TestMatchListener implements FilterMatchListener { + @Override + public void filterMatched(Event event) { + // No-op + } + } +} diff --git a/filter-plugin/logstash-filter-documentdb-aws-guardium/src/test/java/com/ibm/guardium/documentdb/MemoryIssuesTest.java b/filter-plugin/logstash-filter-documentdb-aws-guardium/src/test/java/com/ibm/guardium/documentdb/MemoryIssuesTest.java new file mode 100644 index 000000000..38610af68 --- /dev/null +++ b/filter-plugin/logstash-filter-documentdb-aws-guardium/src/test/java/com/ibm/guardium/documentdb/MemoryIssuesTest.java @@ -0,0 +1,124 @@ +package com.ibm.guardium.documentdb; + +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import java.util.concurrent.TimeUnit; + +/** Demonstrates the difference between StackOverflowError and OutOfMemoryError */ +@Disabled +class MemoryIssuesTest { + + /** + * StackOverflowError: Caused by deep recursion (stack depth) Stack memory is limited (typically + * 1-2MB per thread) + */ + @Test + @Timeout(value = 5, unit = TimeUnit.SECONDS) + void testStackOverflowError() { + System.out.println("\n=== StackOverflowError Test ==="); + + // Create deeply nested JSON (10,000 levels) + StringBuilder deepJson = new StringBuilder("{"); + for (int i = 0; i < 10000; i++) { + deepJson.append("\"level\":").append("{"); + } + deepJson.append("\"value\":1"); + for (int i = 0; i < 10000; i++) { + deepJson.append("}"); + } + deepJson.append("}"); + + System.out.println("JSON string size: " + deepJson.length() + " bytes"); + System.out.println("Nesting depth: 10,000 levels"); + + try { + JsonObject obj = new Gson().fromJson(deepJson.toString(), JsonObject.class); + System.out.println("✗ Unexpectedly succeeded"); + } catch (StackOverflowError soe) { + System.out.println("✓ StackOverflowError caught (as expected)"); + System.out.println(" Cause: Recursive parsing exceeded stack depth"); + System.out.println(" Stack memory exhausted, but heap memory is fine"); + } + } + + /** + * OutOfMemoryError: Caused by too much data (heap exhaustion) Heap memory is much larger (GBs + * available) + */ + @Test + @Timeout(value = 10, unit = TimeUnit.SECONDS) + void testPotentialOutOfMemoryScenario() { + System.out.println("\n=== Potential OutOfMemoryError Scenario ==="); + + // Create JSON with MANY fields (not deeply nested, but wide) + // This consumes heap memory, not stack + StringBuilder wideJson = new StringBuilder("{"); + int fieldCount = 100000; // 100K fields + + for (int i = 0; i < fieldCount; i++) { + if (i > 0) wideJson.append(","); + wideJson.append("\"field").append(i).append("\":\"value").append(i).append("\""); + } + wideJson.append("}"); + + System.out.println("JSON string size: " + (wideJson.length() / 1024) + " KB"); + System.out.println("Field count: " + fieldCount); + System.out.println("Nesting depth: 1 (shallow)"); + + try { + // Get memory before parsing + Runtime runtime = Runtime.getRuntime(); + long memBefore = runtime.totalMemory() - runtime.freeMemory(); + + JsonObject obj = new Gson().fromJson(wideJson.toString(), JsonObject.class); + + long memAfter = runtime.totalMemory() - runtime.freeMemory(); + long memUsed = (memAfter - memBefore) / 1024 / 1024; // MB + + System.out.println("✓ Parsing succeeded"); + System.out.println(" Memory used: ~" + memUsed + " MB"); + System.out.println(" This uses HEAP memory, not stack"); + System.out.println(" With millions of fields, could cause OOM"); + } catch (OutOfMemoryError oom) { + System.out.println("✗ OutOfMemoryError occurred"); + System.out.println(" Heap memory exhausted"); + } catch (Exception e) { + System.out.println("Other exception: " + e.getMessage()); + } + } + + /** Demonstrates the difference in memory usage */ + @Test + void testMemoryDifference() { + System.out.println("\n=== Memory Usage Comparison ==="); + + // Stack memory per thread (JVM default) + System.out.println("Stack memory per thread: ~1-2 MB (JVM default -Xss)"); + System.out.println(" - Used for method calls, local variables"); + System.out.println(" - StackOverflowError when exceeded"); + System.out.println(); + + // Heap memory + Runtime runtime = Runtime.getRuntime(); + long maxHeap = runtime.maxMemory() / 1024 / 1024; // MB + long totalHeap = runtime.totalMemory() / 1024 / 1024; // MB + long freeHeap = runtime.freeMemory() / 1024 / 1024; // MB + + System.out.println("Heap memory:"); + System.out.println(" - Max heap: " + maxHeap + " MB (JVM -Xmx)"); + System.out.println(" - Total heap: " + totalHeap + " MB"); + System.out.println(" - Free heap: " + freeHeap + " MB"); + System.out.println(" - Used for objects, arrays, data structures"); + System.out.println(" - OutOfMemoryError when exceeded"); + System.out.println(); + + System.out.println("Key Difference:"); + System.out.println(" - Deep nesting (10K levels) → StackOverflowError"); + System.out.println(" - Wide data (millions of fields) → OutOfMemoryError"); + System.out.println(" - Both are security concerns!"); + } +} diff --git a/filter-plugin/logstash-filter-documentdb-aws-guardium/src/test/java/com/ibm/guardium/documentdb/ParserInfiniteLoopTest.java b/filter-plugin/logstash-filter-documentdb-aws-guardium/src/test/java/com/ibm/guardium/documentdb/ParserInfiniteLoopTest.java new file mode 100644 index 000000000..274036189 --- /dev/null +++ b/filter-plugin/logstash-filter-documentdb-aws-guardium/src/test/java/com/ibm/guardium/documentdb/ParserInfiniteLoopTest.java @@ -0,0 +1,326 @@ +package com.ibm.guardium.documentdb; + +import co.elastic.logstash.api.Context; +import co.elastic.logstash.api.Event; +import co.elastic.logstash.api.FilterMatchListener; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.logstash.plugins.ContextImpl; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Test suite to verify that malformed events cannot cause infinite loops in the DocumentDB parser. + * All tests have timeouts to detect infinite loops. + */ +@Disabled +class ParserInfiniteLoopTest { + + static final Context context = new ContextImpl(null, null); + static final DocumentdbGuardiumFilter filter = + new DocumentdbGuardiumFilter("test-id", null, context); + + /** + * Test 1: Deeply nested JSON (1000 levels) This should NOT cause infinite loop - Gson will handle + * it or throw exception + */ + @Test + // @Timeout(value = 5, unit = TimeUnit.SECONDS) + void testDeeplyNestedJson() { + StringBuilder deepJson = new StringBuilder(); + deepJson.append( + "{\"atype\":\"createDatabase\",\"ts\":1629364973923,\"remote_ip\":\"172.31.32.149:57308\",\"user\":\"test\",\"param\":{\"ns\":\"test\",\"nested\":"); + + // Create 1000 levels of nesting + for (int i = 0; i < 1000; i++) { + deepJson.append("{\"level").append(i).append("\":"); + } + deepJson.append("\"value\""); + for (int i = 0; i < 1000; i++) { + deepJson.append("}"); + } + deepJson.append("}}"); + + Event e = new org.logstash.Event(); + e.setField("message", deepJson.toString()); + TestMatchListener matchListener = new TestMatchListener(); + + try { + Collection results = filter.filter(Collections.singletonList(e), matchListener); + // Should complete without infinite loop (may succeed or fail with exception) + assertNotNull(results); + } catch (Exception ex) { + // Exception is acceptable - just shouldn't hang + assertTrue(true, "Parser threw exception instead of hanging: " + ex.getMessage()); + } + } + + /** + * Test 2: Truncated JSON (missing closing braces) Should be caught by isValidJsonStructure() + * check + */ + @Test + @Timeout(value = 5, unit = TimeUnit.SECONDS) + void testTruncatedJson() { + String truncatedJson = + "{\"atype\":\"createDatabase\",\"ts\":1629364973923,\"remote_ip\":\"172.31.32.149:57308\",\"user\":\"test\",\"param\":{\"ns\":\"test\""; + + Event e = new org.logstash.Event(); + e.setField("message", truncatedJson); + TestMatchListener matchListener = new TestMatchListener(); + + ArrayList events = new ArrayList<>(); + events.add(e); + Collection results = filter.filter(events, matchListener); + + // Should be tagged as truncated by JsonValidator + assertNotNull(results); + Object tags = e.getField("tags"); + assertTrue( + (tags != null && tags.toString().contains("_documentdbguardium_json_depth_error")), + "Truncated JSON should be detected and tagged"); + } + + /** + * Test 3: Very long string with many dots (tests split optimization) Should NOT cause infinite + * loop with our indexOf() optimization + */ + @Test + // @Timeout(value = 5, unit = TimeUnit.SECONDS) + void testManyDotsInNamespace() { + StringBuilder manyDots = new StringBuilder("db"); + for (int i = 0; i < 10000; i++) { + manyDots.append(".collection").append(i); + } + + String jsonWithManyDots = + "{\"atype\":\"createDatabase\",\"ts\":1629364973923,\"remote_ip\":\"172.31.32.149:57308\",\"user\":\"test\",\"param\":{\"ns\":\"" + + manyDots.toString() + + "\"}}"; + + Event e = new org.logstash.Event(); + e.setField("message", jsonWithManyDots); + TestMatchListener matchListener = new TestMatchListener(); + + try { + Collection results = filter.filter(Collections.singletonList(e), matchListener); + assertNotNull(results); + // Should complete quickly with our indexOf() optimization + } catch (Exception ex) { + // Exception is acceptable - just shouldn't hang + assertTrue(true, "Parser handled long string without hanging"); + } + } + + /** Test 4: Circular-looking structure (not actually circular in JSON, but tests recursion) */ + @Test + @Timeout(value = 5, unit = TimeUnit.SECONDS) + void testCircularLookingStructure() { + String circularLooking = + "{\"atype\":\"createDatabase\",\"ts\":1629364973923,\"remote_ip\":\"172.31.32.149:57308\",\"user\":\"test\",\"param\":{\"ns\":\"test\",\"a\":{\"b\":{\"c\":{\"d\":{\"e\":{\"f\":{\"g\":{\"h\":{\"i\":{\"j\":\"value\"}}}}}}}}}}}}"; + + Event e = new org.logstash.Event(); + e.setField("message", circularLooking); + TestMatchListener matchListener = new TestMatchListener(); + + Collection results = filter.filter(Collections.singletonList(e), matchListener); + assertNotNull(results); + // Should complete - JSON doesn't support actual circular references + } + + /** Test 5: Very large array (tests array iteration) */ + @Test + @Timeout(value = 5, unit = TimeUnit.SECONDS) + void testVeryLargeArray() { + StringBuilder largeArray = new StringBuilder(); + largeArray.append( + "{\"op\":\"command\",\"ts\":1641978528311,\"ns\":\"test.collection\",\"command\":{\"aggregate\":\"cases\",\"pipeline\":["); + + for (int i = 0; i < 1000; i++) { + if (i > 0) largeArray.append(","); + largeArray + .append("{\"$match\":{\"field") + .append(i) + .append("\":\"value") + .append(i) + .append("\"}}"); + } + + largeArray.append( + "],\"cursor\":{},\"$db\":\"test\"},\"client\":\"172.31.40.18:38230\",\"user\":\"test\"}"); + + Event e = new org.logstash.Event(); + e.setField("message", largeArray.toString()); + TestMatchListener matchListener = new TestMatchListener(); + + try { + Collection results = filter.filter(Collections.singletonList(e), matchListener); + assertNotNull(results); + // Should complete - array iteration is bounded + } catch (Exception ex) { + assertTrue(true, "Parser handled large array without hanging"); + } + } + + /** Test 6: Malformed IP address with many colons (tests IP parsing) */ + @Test + @Timeout(value = 5, unit = TimeUnit.SECONDS) + void testMalformedIpAddress() { + String malformedIp = + "{\"atype\":\"createDatabase\",\"ts\":1629364973923,\"remote_ip\":\"1:2:3:4:5:6:7:8:9:10:11:12:13:14:15\",\"user\":\"test\",\"param\":{\"ns\":\"test\"}}"; + + Event e = new org.logstash.Event(); + e.setField("message", malformedIp); + TestMatchListener matchListener = new TestMatchListener(); + + try { + Collection results = filter.filter(Collections.singletonList(e), matchListener); + assertNotNull(results); + // Should complete - our indexOf() optimization handles this + } catch (Exception ex) { + assertTrue(true, "Parser handled malformed IP without hanging"); + } + } + + /** Test 7: String with excessive whitespace (tests replaceAll optimization) */ + @Test + @Timeout(value = 5, unit = TimeUnit.SECONDS) + void testExcessiveWhitespace() { + StringBuilder excessiveWhitespace = new StringBuilder(); + for (int i = 0; i < 10000; i++) { + excessiveWhitespace.append(" \t\n\r"); + } + + String jsonWithWhitespace = + "{\"op\":\"command\",\"ts\":1641978528311,\"ns\":\"test.collection\",\"command\":{\"aggregate\":\"cases\"},\"client\":\"172.31.40.18:38230\",\"appName\":\"" + + excessiveWhitespace.toString() + + "\",\"user\":\"test\"}"; + + Event e = new org.logstash.Event(); + e.setField("message", jsonWithWhitespace); + TestMatchListener matchListener = new TestMatchListener(); + + try { + Collection results = filter.filter(Collections.singletonList(e), matchListener); + assertNotNull(results); + // Should complete with our replace() optimization + } catch (Exception ex) { + assertTrue(true, "Parser handled excessive whitespace without hanging"); + } + } + + /** Test 8: Maximum size event (256KB - CloudWatch limit) */ + @Test + @Timeout(value = 10, unit = TimeUnit.SECONDS) + void testMaximumSizeEvent() { + StringBuilder maxSize = new StringBuilder(); + maxSize.append( + "{\"atype\":\"createDatabase\",\"ts\":1629364973923,\"remote_ip\":\"172.31.32.149:57308\",\"user\":\"test\",\"param\":{\"ns\":\"test\",\"data\":\""); + + // Fill to near 256KB + int targetSize = 256 * 1024 - 200; // Leave room for JSON structure + while (maxSize.length() < targetSize) { + maxSize.append("x"); + } + maxSize.append("\"}}"); + + Event e = new org.logstash.Event(); + e.setField("message", maxSize.toString()); + TestMatchListener matchListener = new TestMatchListener(); + + try { + Collection results = filter.filter(Collections.singletonList(e), matchListener); + assertNotNull(results); + // Should complete - size is bounded by CloudWatch + } catch (Exception ex) { + assertTrue(true, "Parser handled maximum size event without hanging"); + } + } + + /** Test 9: Concurrent parsing of malformed events Tests thread safety and ensures no deadlocks */ + @Test + @Timeout(value = 10, unit = TimeUnit.SECONDS) + void testConcurrentMalformedEvents() throws InterruptedException { + ExecutorService executor = Executors.newFixedThreadPool(10); + CountDownLatch latch = new CountDownLatch(100); + + for (int i = 0; i < 100; i++) { + final int index = i; + executor.submit( + () -> { + try { + String malformed = + "{\"atype\":\"test" + index + "\",\"ts\":1629364973923,\"param\":{\"ns\":\"test"; + Event e = new org.logstash.Event(); + e.setField("message", malformed); + TestMatchListener matchListener = new TestMatchListener(); + filter.filter(Collections.singletonList(e), matchListener); + } catch (Exception ex) { + // Expected for malformed events + } finally { + latch.countDown(); + } + }); + } + + boolean completed = latch.await(8, TimeUnit.SECONDS); + executor.shutdown(); + assertTrue(completed, "All concurrent parsing tasks should complete without deadlock"); + } + + /** Test 10: Empty and null edge cases */ + @Test + @Timeout(value = 5, unit = TimeUnit.SECONDS) + void testEmptyAndNullCases() { + TestMatchListener matchListener = new TestMatchListener(); + + // Empty string + Event e1 = new org.logstash.Event(); + e1.setField("message", ""); + ArrayList events1 = new ArrayList<>(); + events1.add(e1); + Collection results1 = filter.filter(events1, matchListener); + assertNotNull(results1); + + // Just braces + Event e2 = new org.logstash.Event(); + e2.setField("message", "{}"); + ArrayList events2 = new ArrayList<>(); + events2.add(e2); + Collection results2 = filter.filter(events2, matchListener); + assertNotNull(results2); + + // Just brackets + Event e3 = new org.logstash.Event(); + e3.setField("message", "[]"); + ArrayList events3 = new ArrayList<>(); + events3.add(e3); + Collection results3 = filter.filter(events3, matchListener); + assertNotNull(results3); + } + + // Helper class for test listener + private static class TestMatchListener implements FilterMatchListener { + private AtomicInteger matchCount = new AtomicInteger(0); + + @Override + public void filterMatched(Event event) { + matchCount.incrementAndGet(); + } + + private int getMatchCount() { + return matchCount.get(); + } + } +} diff --git a/filter-plugin/logstash-filter-documentdb-aws-guardium/src/test/java/com/ibm/guardium/documentdb/PerformanceTest.java b/filter-plugin/logstash-filter-documentdb-aws-guardium/src/test/java/com/ibm/guardium/documentdb/PerformanceTest.java new file mode 100644 index 000000000..a1f56e628 --- /dev/null +++ b/filter-plugin/logstash-filter-documentdb-aws-guardium/src/test/java/com/ibm/guardium/documentdb/PerformanceTest.java @@ -0,0 +1,67 @@ +package com.ibm.guardium.documentdb; + +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import java.util.concurrent.TimeUnit; + +/** Direct performance test to measure actual execution time of problematic operations */ +@Disabled +class PerformanceTest { + + @Test + @Timeout(value = 2, unit = TimeUnit.SECONDS) + void testSplitPerformance() { + // Create string with 100,000 dots + StringBuilder sb = new StringBuilder("db"); + for (int i = 0; i < 100000; i++) { + sb.append(".collection").append(i); + } + String ns = sb.toString(); + + long start = System.currentTimeMillis(); + String[] parts = ns.split("\\."); + long end = System.currentTimeMillis(); + + System.out.println("Split with 100,000 dots took: " + (end - start) + "ms"); + System.out.println("Array length: " + parts.length); + System.out.println("First element: " + parts[0]); + } + + @Test + @Timeout(value = 2, unit = TimeUnit.SECONDS) + void testReplaceAllPerformance() { + // Create string with 500,000 whitespace characters + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < 500000; i++) { + sb.append(" \t\n\r"); + } + String whitespace = sb.toString(); + + long start = System.currentTimeMillis(); + String result = whitespace.trim().replaceAll("\\s", ""); + long end = System.currentTimeMillis(); + + System.out.println("ReplaceAll with 2,000,000 whitespace chars took: " + (end - start) + "ms"); + System.out.println("Result length: " + result.length()); + } + + @Test + @Timeout(value = 2, unit = TimeUnit.SECONDS) + void testSplitColonPerformance() { + // Create IP string with many colons + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < 100000; i++) { + sb.append(i).append(":"); + } + String ip = sb.toString(); + + long start = System.currentTimeMillis(); + String[] parts = ip.split(":"); + long end = System.currentTimeMillis(); + + System.out.println("Split colon with 100,000 colons took: " + (end - start) + "ms"); + System.out.println("Array length: " + parts.length); + } +} diff --git a/filter-plugin/logstash-filter-documentdb-aws-guardium/src/test/java/com/ibm/guardium/documentdb/StackOverflowHandlingTest.java b/filter-plugin/logstash-filter-documentdb-aws-guardium/src/test/java/com/ibm/guardium/documentdb/StackOverflowHandlingTest.java new file mode 100644 index 000000000..94f11d8e1 --- /dev/null +++ b/filter-plugin/logstash-filter-documentdb-aws-guardium/src/test/java/com/ibm/guardium/documentdb/StackOverflowHandlingTest.java @@ -0,0 +1,92 @@ +package com.ibm.guardium.documentdb; + +import co.elastic.logstash.api.Context; +import co.elastic.logstash.api.Event; +import co.elastic.logstash.api.FilterMatchListener; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.logstash.plugins.ContextImpl; + +import java.util.Collection; +import java.util.Collections; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.*; + +/** Demonstrates that StackOverflowError CAN be caught and handled gracefully */ +@Disabled +class StackOverflowHandlingTest { + + static final Context context = new ContextImpl(null, null); + static final DocumentdbGuardiumFilter filter = + new DocumentdbGuardiumFilter("test-id", null, context); + + /** Test that demonstrates StackOverflowError can be caught */ + @Test + @Timeout(value = 5, unit = TimeUnit.SECONDS) + void testStackOverflowCanBeCaught() { + // Create deeply nested JSON that will cause StackOverflowError + StringBuilder deepJson = new StringBuilder(); + deepJson.append( + "{\"atype\":\"createDatabase\",\"ts\":1629364973923,\"remote_ip\":\"172.31.32.149:57308\",\"user\":\"test\",\"param\":{\"ns\":\"test\",\"data\":"); + + for (int i = 0; i < 10000; i++) { + deepJson.append("{\"level").append(i).append("\":"); + } + deepJson.append("\"value\""); + for (int i = 0; i < 10000; i++) { + deepJson.append("}"); + } + deepJson.append("}}"); + + Event e = new org.logstash.Event(); + e.setField("message", deepJson.toString()); + TestMatchListener matchListener = new TestMatchListener(); + + // With JsonValidator, deeply nested JSON is now prevented before it can cause + // StackOverflowError + Collection results = filter.filter(Collections.singletonList(e), matchListener); + assertNotNull(results); + + // Event should be tagged as having excessive depth + Object tags = e.getField("tags"); + boolean wasTagged = + tags != null && tags.toString().contains("_documentdbguardium_json_depth_error"); + + assertTrue(wasTagged, "JsonValidator should have detected and tagged deeply nested JSON"); + System.out.println("✓ JsonValidator successfully prevented StackOverflowError"); + System.out.println("✓ Event was tagged instead of causing stack overflow"); + System.out.println("✓ This is the BETTER solution - prevention rather than catching errors"); + } + + /** Demonstrates that after catching StackOverflowError, the JVM continues normally */ + @Test + @Timeout(value = 5, unit = TimeUnit.SECONDS) + void testJvmContinuesAfterStackOverflow() { + // First, cause a StackOverflowError + try { + causeStackOverflow(0); + } catch (StackOverflowError soe) { + System.out.println("✓ Caught StackOverflowError from recursive method"); + } + + // Now prove the JVM is still working fine + String result = "JVM is still working!"; + assertEquals("JVM is still working!", result); + System.out.println("✓ JVM continues to work normally after StackOverflowError"); + } + + // Helper method to intentionally cause StackOverflowError + private void causeStackOverflow(int depth) { + causeStackOverflow(depth + 1); // Infinite recursion + } + + // Helper class for test listener + private static class TestMatchListener implements FilterMatchListener { + @Override + public void filterMatched(Event event) { + // No-op + } + } +} diff --git a/filter-plugin/logstash-filter-documentdb-aws-guardium/src/test/java/com/ibm/guardium/documentdb/StringUtilsTest.java b/filter-plugin/logstash-filter-documentdb-aws-guardium/src/test/java/com/ibm/guardium/documentdb/StringUtilsTest.java new file mode 100644 index 000000000..5f7815e34 --- /dev/null +++ b/filter-plugin/logstash-filter-documentdb-aws-guardium/src/test/java/com/ibm/guardium/documentdb/StringUtilsTest.java @@ -0,0 +1,176 @@ +package com.ibm.guardium.documentdb; + +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.*; + +/** + * Unit tests for StringUtils class. + */ +public class StringUtilsTest { + + @Test + public void testRemoveWhitespace_WithSpaces() { + assertEquals("HelloWorld", StringUtils.removeWhitespace("Hello World")); + assertEquals("test", StringUtils.removeWhitespace(" test ")); + assertEquals("abc", StringUtils.removeWhitespace("a b c")); + } + + @Test + public void testRemoveWhitespace_WithTabs() { + assertEquals("HelloWorld", StringUtils.removeWhitespace("Hello\tWorld")); + assertEquals("test", StringUtils.removeWhitespace("\ttest\t")); + } + + @Test + public void testRemoveWhitespace_WithNewlines() { + assertEquals("HelloWorld", StringUtils.removeWhitespace("Hello\nWorld")); + assertEquals("HelloWorld", StringUtils.removeWhitespace("Hello\r\nWorld")); + } + + @Test + public void testRemoveWhitespace_NullOrEmpty() { + assertNull(StringUtils.removeWhitespace(null)); + assertEquals("", StringUtils.removeWhitespace("")); + } + + @Test + public void testRemoveWhitespace_NoWhitespace() { + assertEquals("HelloWorld", StringUtils.removeWhitespace("HelloWorld")); + } + + @Test + public void testExtractDbNameFromNs_Valid() { + assertEquals("testdb", StringUtils.extractDbNameFromNs("testdb.collection")); + assertEquals("mydb", StringUtils.extractDbNameFromNs("mydb.users")); + assertEquals("db", StringUtils.extractDbNameFromNs("db.col")); + } + + @Test + public void testExtractDbNameFromNs_NoDot() { + assertEquals("testdb", StringUtils.extractDbNameFromNs("testdb")); + assertEquals("single", StringUtils.extractDbNameFromNs("single")); + } + + @Test + public void testExtractDbNameFromNs_MultipleDots() { + assertEquals("db", StringUtils.extractDbNameFromNs("db.collection.subcollection")); + } + + @Test + public void testExtractDbNameFromNs_NullOrEmpty() { + assertEquals(Constants.UNKNOWN_STRING, StringUtils.extractDbNameFromNs(null)); + assertEquals(Constants.UNKNOWN_STRING, StringUtils.extractDbNameFromNs("")); + } + + @Test + public void testExtractCollectionFromNs_Valid() { + assertEquals("collection", StringUtils.extractCollectionFromNs("testdb.collection")); + assertEquals("users", StringUtils.extractCollectionFromNs("mydb.users")); + assertEquals("col", StringUtils.extractCollectionFromNs("db.col")); + } + + @Test + public void testExtractCollectionFromNs_NoDot() { + assertEquals("testdb", StringUtils.extractCollectionFromNs("testdb")); + assertEquals("single", StringUtils.extractCollectionFromNs("single")); + } + + @Test + public void testExtractCollectionFromNs_MultipleDots() { + assertEquals("collection.subcollection", + StringUtils.extractCollectionFromNs("db.collection.subcollection")); + } + + @Test + public void testExtractCollectionFromNs_NullOrEmpty() { + assertNull(StringUtils.extractCollectionFromNs(null)); + assertEquals("", StringUtils.extractCollectionFromNs("")); + } + + @Test + public void testTruncate_ExceedsMaxLength() { + String longString = "This is a very long string that needs to be truncated"; + String result = StringUtils.truncate(longString, 20, "..."); + assertEquals("This is a very long ...".length(), result.length()); + assertTrue(result.endsWith("...")); + assertEquals("This is a very long ...", result); + } + + @Test + public void testTruncate_WithinMaxLength() { + String shortString = "Short"; + String result = StringUtils.truncate(shortString, 20, "..."); + assertEquals("Short", result); + } + + @Test + public void testTruncate_ExactMaxLength() { + String exactString = "12345678901234567890"; + String result = StringUtils.truncate(exactString, 20, "..."); + assertEquals("12345678901234567890", result); + } + + @Test + public void testTruncate_NullString() { + assertNull(StringUtils.truncate(null, 10, "...")); + } + + @Test + public void testContainsAnyProfilerKey_WithAggregateKey() { + assertTrue(StringUtils.containsAnyProfilerKey("{ \"aggregate\": \"collection\" }")); + } + + @Test + public void testContainsAnyProfilerKey_WithCountKey() { + assertTrue(StringUtils.containsAnyProfilerKey("{ \"count\": \"collection\" }")); + } + + @Test + public void testContainsAnyProfilerKey_WithInsertKey() { + assertTrue(StringUtils.containsAnyProfilerKey("{ \"insert\": \"collection\" }")); + } + + @Test + public void testContainsAnyProfilerKey_WithUpdateKey() { + assertTrue(StringUtils.containsAnyProfilerKey("{ \"update\": \"collection\" }")); + } + + @Test + public void testContainsAnyProfilerKey_WithRemoveKey() { + assertTrue(StringUtils.containsAnyProfilerKey("{ \"remove\": \"collection\" }")); + } + + @Test + public void testContainsAnyProfilerKey_WithFindKey() { + assertTrue(StringUtils.containsAnyProfilerKey("{ \"find\": \"collection\" }")); + } + + @Test + public void testContainsAnyProfilerKey_WithDistinctKey() { + assertTrue(StringUtils.containsAnyProfilerKey("{ \"distinct\": \"collection\" }")); + } + + @Test + public void testContainsAnyProfilerKey_WithFindAndModifyKey() { + assertTrue(StringUtils.containsAnyProfilerKey("{ \"findAndModify\": \"collection\" }")); + } + + @Test + public void testContainsAnyProfilerKey_NoProfilerKey() { + assertFalse(StringUtils.containsAnyProfilerKey("{ \"other\": \"value\" }")); + assertFalse(StringUtils.containsAnyProfilerKey("{ \"atype\": \"authenticate\" }")); + } + + @Test + public void testContainsAnyProfilerKey_NullOrEmpty() { + assertFalse(StringUtils.containsAnyProfilerKey(null)); + assertFalse(StringUtils.containsAnyProfilerKey("")); + } + + @Test + public void testContainsAnyProfilerKey_MultipleKeys() { + assertTrue(StringUtils.containsAnyProfilerKey("{ \"insert\": \"col\", \"update\": \"col\" }")); + } +} + +// Made with Bob diff --git a/filter-plugin/logstash-filter-documentdb-aws-guardium/src/test/java/com/ibm/guardium/documentdb/ValidationUtilsTest.java b/filter-plugin/logstash-filter-documentdb-aws-guardium/src/test/java/com/ibm/guardium/documentdb/ValidationUtilsTest.java new file mode 100644 index 000000000..9125df98e --- /dev/null +++ b/filter-plugin/logstash-filter-documentdb-aws-guardium/src/test/java/com/ibm/guardium/documentdb/ValidationUtilsTest.java @@ -0,0 +1,113 @@ +package com.ibm.guardium.documentdb; + +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.*; + +/** + * Unit tests for ValidationUtils class. + */ +public class ValidationUtilsTest { + + @Test + public void testIsProperlyClosedJson_ValidObject() { + assertTrue(ValidationUtils.isProperlyClosedJson("{\"key\":\"value\"}")); + assertTrue(ValidationUtils.isProperlyClosedJson("{ \"key\": \"value\" }")); + assertTrue(ValidationUtils.isProperlyClosedJson(" {\"key\":\"value\"} ")); + } + + @Test + public void testIsProperlyClosedJson_ValidArray() { + assertTrue(ValidationUtils.isProperlyClosedJson("[\"value1\",\"value2\"]")); + assertTrue(ValidationUtils.isProperlyClosedJson("[ \"value1\", \"value2\" ]")); + assertTrue(ValidationUtils.isProperlyClosedJson(" [\"value1\"] ")); + } + + @Test + public void testIsProperlyClosedJson_Invalid() { + assertFalse(ValidationUtils.isProperlyClosedJson("{\"key\":\"value\"")); + assertFalse(ValidationUtils.isProperlyClosedJson("\"key\":\"value\"}")); + assertFalse(ValidationUtils.isProperlyClosedJson("[\"value\"")); + assertFalse(ValidationUtils.isProperlyClosedJson("\"value\"]")); + assertFalse(ValidationUtils.isProperlyClosedJson("{]")); + assertFalse(ValidationUtils.isProperlyClosedJson("[}")); + } + + @Test + public void testIsProperlyClosedJson_NullOrEmpty() { + assertFalse(ValidationUtils.isProperlyClosedJson(null)); + assertFalse(ValidationUtils.isProperlyClosedJson("")); + assertFalse(ValidationUtils.isProperlyClosedJson(" ")); + } + + @Test + public void testIsValidIpAddress_ValidIPv4() { + assertTrue(ValidationUtils.isValidIpAddress("192.168.1.1")); + assertTrue(ValidationUtils.isValidIpAddress("127.0.0.1")); + assertTrue(ValidationUtils.isValidIpAddress("0.0.0.0")); + assertTrue(ValidationUtils.isValidIpAddress("255.255.255.255")); + } + + @Test + public void testIsValidIpAddress_ValidIPv6() { + assertTrue(ValidationUtils.isValidIpAddress("2001:0db8:85a3:0000:0000:8a2e:0370:7334")); + assertTrue(ValidationUtils.isValidIpAddress("::1")); + assertTrue(ValidationUtils.isValidIpAddress("fe80::1")); + } + + @Test + public void testIsValidIpAddress_Invalid() { + assertFalse(ValidationUtils.isValidIpAddress("256.1.1.1")); + assertFalse(ValidationUtils.isValidIpAddress("192.168.1")); + assertFalse(ValidationUtils.isValidIpAddress("not-an-ip")); + assertFalse(ValidationUtils.isValidIpAddress("")); + assertFalse(ValidationUtils.isValidIpAddress(null)); + } + + @Test + public void testIsDocumentInternalCommandIp_LocalIPs() { + assertTrue(ValidationUtils.isDocumentInternalCommandIp("127.0.0.1")); + assertTrue(ValidationUtils.isDocumentInternalCommandIp("0:0:0:0:0:0:0:1")); + } + + @Test + public void testIsDocumentInternalCommandIp_InternalMarker() { + assertTrue(ValidationUtils.isDocumentInternalCommandIp("(NONE)")); + assertTrue(ValidationUtils.isDocumentInternalCommandIp(" (NONE) ")); + assertTrue(ValidationUtils.isDocumentInternalCommandIp("(none)")); + } + + @Test + public void testIsDocumentInternalCommandIp_NotInternal() { + assertFalse(ValidationUtils.isDocumentInternalCommandIp("192.168.1.1")); + assertFalse(ValidationUtils.isDocumentInternalCommandIp("10.0.0.1")); + assertFalse(ValidationUtils.isDocumentInternalCommandIp("")); + assertFalse(ValidationUtils.isDocumentInternalCommandIp(null)); + } + + @Test + public void testIsNullOrEmpty() { + assertTrue(ValidationUtils.isNullOrEmpty(null)); + assertTrue(ValidationUtils.isNullOrEmpty("")); + assertFalse(ValidationUtils.isNullOrEmpty(" ")); + assertFalse(ValidationUtils.isNullOrEmpty("text")); + } + + @Test + public void testGetValueOrDefault_WithValue() { + assertEquals("value", ValidationUtils.getValueOrDefault("value", "default")); + assertEquals("text", ValidationUtils.getValueOrDefault("text", "default")); + } + + @Test + public void testGetValueOrDefault_WithoutValue() { + assertEquals("default", ValidationUtils.getValueOrDefault(null, "default")); + assertEquals("default", ValidationUtils.getValueOrDefault("", "default")); + } + + @Test + public void testGetValueOrDefault_BothNull() { + assertNull(ValidationUtils.getValueOrDefault(null, null)); + } +} + +// Made with Bob diff --git a/filter-plugin/logstash-filter-dynamodb-guardium/DynamoDBOverS3SQS/S3SQS_README.md b/filter-plugin/logstash-filter-dynamodb-guardium/DynamoDBOverS3SQS/S3SQS_README.md new file mode 100644 index 000000000..b5cacb7a1 --- /dev/null +++ b/filter-plugin/logstash-filter-dynamodb-guardium/DynamoDBOverS3SQS/S3SQS_README.md @@ -0,0 +1,221 @@ +# Dynamodb-Guardium Logstash filter plug-in + +This is a [Logstash](https://github.com/elastic/logstash) filter plug-in for the universal connector that is featured in IBM Security Guardium. It parses events and messages from the Amazon DynamoDB audit log into a [Guardium record](https://github.com/IBM/universal-connectors/blob/main/common/src/main/java/com/ibm/guardium/universalconnector/commons/structures/Record.java) instance (which is a standard structure made out of several parts). The information is then sent over to Guardium. Guardium records include the accessor (the person who tried to access the data), the session, data, and exceptions. If there are no errors, the data contains details about the query "construct". The construct details the main action (verb) and collections (objects) involved. + +The plug-in is free and open-source (Apache 2.0). It can be used as a starting point to develop additional filter plug-ins for Guardium universal connector. + + +## 1. Configuring Amazon DynamoDB + +In the AWS web interface, configure the service for Dynamodb. + +### Procedure + +1. Go to https://console.aws.amazon.com/ +2. Click **Services** in the top left menu. +3. Underneath **All services**, click on **Database**. +4. On the right panel, click **DynamoDB**. +5. At the top right, click on the dropdown menu and select your region. +6. Click the orange **Create Table** button. +7. Enter a table name. +8. Enter a partition key. +9. Scroll down and click the orange **Create table** button. + +## 2. Enabling audit logs + +There are different methods for auditing and logging. We will use CloudTrail for this example since it supports all required parameters. The following events are supported for auditing in AWS. + + +### Procedure + +1. Click **Services** in the top left menu. +2. Underneath **All services**, click on **Management & Governance**. +3. On the right panel, click **Cloud Trail**. +4. Click **Create trail** button. +5. Provide a trail name under **Trail name**. +6. Under **Storage location**, verify that **Create new S3 bucket** is selected. +7. Under **Log file SSE-KMS encryption**, clear the Enabled box. +8. Click **Next**. +9. For **Event type**, select **Management events** and **Data events**. +10. Verify that **Read** and **Write** are selected for **API Activity**. +11. In the **Data Events** section, click **Switch to basic event selectors**. +12. Click **Continue** to confirm. +13. Click **Add data event type**. +14. Click **Data event source** and select **DynamoDB**. +15. Click **NEXT**. +16. Verify that all parameters shown are correct. +17. Click **Create trail**. + + +## 3. Viewing the logs on CloudTrail + +### Procedure + +1. Click the Service drop down. +2. On the `Recently visited` panel, click **CloudTrail**. +3. On the left panel, click **Trails**. +4. Click on **Bucket Name** under the Trail that was created above. +5. Traverse through the folders of the S3 bucket. +7. Log files will be displayed as .json.gz files. +8. The log file can be then opened in the browser in readable format or downloaded as per the requirement. + +## 4. Exporting Log File Details Created on S3 to SQS using Event Notification +In order to achieve load balancing of audit logs between different collectors, the audit logs file details must be exported from +S3 to SQS. +To retrieve details of a recently added log file in S3, an event notification must be configured. +This notification will be sent to an SQS queue, allowing us to capture information about the newly added log file in S3. + + +### Creating the SQS queue +The SQS queue created in these steps will receive messages from the Event Notification (configured in the next section). +These messages, generated by monitoring the S3 bucket, will contain details of the recently added S3 log files. + + +#### Procedure +1. Go to https://console.aws.amazon.com/ +2. Click **Services** +3. Search for SQS and click on **Simple Queue Services** +4. Click **Create Queue**. +5. Select the type as **Standard**. +6. Enter the name for the queue +7. Keep the rest of the default settings + +### Creating a policy for the relevant IAM User +Perform the following steps for the IAM user who is accessing the SQS logs in Guardium: + +#### Procedure +1. Go to https://console.aws.amazon.com/ +2. Go to **IAM service** > **Policies** > **Create Policy**. +3. Select **service as SQS**. +4. Check the following checkboxes: + * **ListQueues** + * **DeleteMessage** + * **DeleteMessageBatch** + * **GetQueueAttributes** + * **GetQueueUrl** + * **ReceiveMessage** + * **ChangeMessageVisibility** + * **ChangeMessageVisibilityBatch** +5. In the resources, specify the ARN of the queue created in the above step. +6. Click **Review policy** and specify the policy name. +7. Click **Create policy**. +8. Assign the policy to the user + 1. Log in to the IAM console as an IAM user (https://console.aws.amazon.com/iam/). + 2. Go to **Users** on the console and select the relevant IAM user to whom you want to give permissions. + Click the **username**. + 3. In the **Permissions tab**, click **Add permissions**. + 4. Click **Attach existing policies directly**. + 5. Search for the policy created and check the checkbox next to it. + 6. Click **Next: Review** + 7. Click **Add permissions** + +### Creating the Event Notification +The Event Notification will get triggered when a new Object is added to S3 bucket and will send the events to the SQS queue. +Follow the steps below to configure the Event Notification + +#### Creating Access Policy to allow Notifications +Update the Access Policy of the SQS queue to allow the Notification Service to send messages to the Queue + +__*Procedure*__ +1. Go to https://console.aws.amazon.com/ +2. Go to **SQS** -> **Queues** +3. Click on the Queue that was created in the above step +4. Go to **Access Policy** +5. Click on **Edit** +6. Add the below details to the existing policy + +``` +{ + "Sid": "example-statement-ID", + "Effect": "Allow", + "Principal": { + "Service": "s3.amazonaws.com" + }, + "Action": "SQS:SendMessage", + "Resource": "", + "Condition": { + "StringEquals": { + "aws:SourceAccount": "" + }, + "ArnLike": { + "aws:SourceArn": "" + } + } +} +``` + + +7. Click on **Save** + + +#### Create the Event Notification +__*Procedure*__ +1. Go to https://console.aws.amazon.com/ +2. Go to **Services**. Search for **S3**. +3. Click on the S3 bucket that is associated with the CloudTrail. +4. Click **Properties** +5. Navigate to **Event Notifications** +6. Click on **Create event notification**. +7. Enter **Event name** +8. Enter the **Prefix** though this is optional, this can be set to capture the specific traffic. +9. In **Event Types** Select **All object create events**. +10. In **Destination** Select **SQS queue**. +11. In **Specify SQS Queue** either **Choose from your SQS queues** option select the Queue name from drop down list or **Enter SQS queue ARN** enter the Queue ARN manually. +12. Click on **Save Changes** + + +### Limitations + +1. The Dynamo DB plug-in does not support IPV6. +2. You may need to disable management events in order to avoid heavy traffic and data loss in Guardium. Disabling management events disables logging of the following events: +CreateTable, DeleteTable, ListTable, UpdateTable, DescribeTable events. +3. The following fields couldn't be mapped with the Dynamo audit logs, + 1. Client HostName : Not available with audit logs so set as NA. + + +## 5. Configuring the Dynamodb filters in Guardium + +The Guardium universal connector is the Guardium entry point for native audit logs. The universal connector identifies and parses received events, and then converts them to a standard Guardium format. The output of the universal connector is forwarded to the Guardium sniffer on the collector, for policy and auditing enforcements. Configure Guardium to read the native audit logs by customizing the dynamodb template. + +### Authorizing outgoing traffic from AWS to Guardium + +#### Procedure + +1. Log in to the Guardium Collector's APIs. +2. Issue the following commands: +``` +grdapi add_domain_to_universal_connector_allowed_domains domain=amazonaws.com +grdapi add_domain_to_universal_connector_allowed_domains domain=amazon.com +``` + +#### Before you begin +• Configure the policies you require. See [policies](https://github.com/IBM/universal-connectors/tree/main/docs#policies) for more information. + +• You must have permission for the S-Tap Management role. The admin user includes this role by default. + +• Download the [logstash-filter-dynamodb_guardium_plugin_filter.zip plug-in.](../logstash-filter-dynamodb_guardium_plugin_filter.zip) + +### Procedure +1. On the collector, go to **Setup** > **Tools and Views** > **Configure Universal Connector**. +2. Enable the connector if it is already disabled, before proceeding to upload the UC. +3. Click **Upload File**, + * If the audit logs are to be fetched from S3SQS, select the + 1. [logstash-input-s3_sqs.zip](../../../input-plugin/logstash-input-s3sqs/InputS3SQSPackage/S3SQS/logstash-input-s3_sqs.zip) plug-in. After it is uploaded, click **OK**. This is specifically for S3SQS only. Other types of inputs do not require this file to be uploaded. + * Select [logstash-filter-dynamodb_guardium_plugin_filter.zip](../logstash-filter-dynamodb_guardium_plugin_filter.zip) plug-in. After it is uploaded, click **OK**. +4. Click the Plus sign to open the Connector Configuration dialog box. +5. Type a name in the **Connector name** field. +6. If the audit logs are to be fetched from S3SQS, use the details from the [s3_sqs_input.conf](../DynamoDBOverS3SQS/s3_sqs_input.conf) file. Update the input section to add the details from the corresponding file's input part, omitting the keyword "input{" at the beginning and its corresponding "}" at the end. Please find more detail from [here](../../../input-plugin/logstash-input-s3sqs/README.md). +7. The "type" fields should match in input and filter configuration sections. This field should be unique for every individual connector added. +8. Click **Save**. Guardium validates the new connector, and enables the universal connector if it was + disabled. After it is validated, it appears in the Configure Universal Connector page. + + +## Configuring the dynamodb filters in Guardium Insights + +Depending on your environment, see the instructions for configuring the DynamoDB filters in one of the following +locations, + +* Guardium Insights SaaS, follow [this guide](https://github.com/IBM/universal-connectors/blob/main/docs/Guardium%20Insights/SaaS_1.0/UC_Configuration_GI.md). +* Guardium Insights on-premises, follow [this guide](https://github.com/IBM/universal-connectors/blob/main/docs/Guardium%20Insights/3.2.x/UC_Configuration_GI.md). + +In the input configuration section, refer to the CloudWatch_logs section. diff --git a/filter-plugin/logstash-filter-dynamodb-guardium/DynamoDBOverS3SQS/s3_sqs_input.conf b/filter-plugin/logstash-filter-dynamodb-guardium/DynamoDBOverS3SQS/s3_sqs_input.conf new file mode 100644 index 000000000..48d04dab3 --- /dev/null +++ b/filter-plugin/logstash-filter-dynamodb-guardium/DynamoDBOverS3SQS/s3_sqs_input.conf @@ -0,0 +1,51 @@ +input { + s3_sqs { + queue_url => "" # For i.e https://sqs..amazonaws.com// + region => "" + access_key_id => "" + secret_access_key => "" + role_arn => "" # Leave empty if not using role-based access + max_messages => + wait_time => # Must be >= 0 and <= 20, + polling_frequency => + type => "" + } +} + + +filter { + if [type] == 'S3SQS' and [message] { + + json { + source => "message" + target => "parsed_message" + } + + # Drop event if parsed_message does not contain "Records" + if ![parsed_message][Records] { + drop {} + } + + ruby { + code => ' + parsed_message = event.get("parsed_message") + if parsed_message && parsed_message["Records"].is_a?(Array) + parsed_message["Records"].each do |record| + record["type"] = event.get("type") + record["timestamp"] = event.get("timestamp") || Time.now.to_i + record["fileKey"] = event.get("fileKey") || "unknown" + record["@version"] = event.get("@version") + record["@timestamp"] = event.get("@timestamp") + end + event.set("message", parsed_message["Records"].to_json) + else + event.tag("json_parse_error") + end + ' + } + + dynamodb_guardium_plugin_filter {} + + prune { whitelist_names => [ "GuardRecord" ] } + } +} diff --git a/filter-plugin/logstash-filter-dynamodb-guardium/DynamoDBOverS3SQS/s3_sqs_input_test.conf b/filter-plugin/logstash-filter-dynamodb-guardium/DynamoDBOverS3SQS/s3_sqs_input_test.conf new file mode 100644 index 000000000..604d29a56 --- /dev/null +++ b/filter-plugin/logstash-filter-dynamodb-guardium/DynamoDBOverS3SQS/s3_sqs_input_test.conf @@ -0,0 +1,50 @@ +input{ + generator { + message => '{"Records":[{"eventVersion":"1.10","userIdentity":{"type":"AssumedRole","principalId":"AROAXXXXXXXXXXXXXXXXX:user@example.com","arn":"arn:aws:sts::123456789012:assumed-role/example-role/user@example.com","accountId":"123456789012","accessKeyId":"ASIAXXXXXXXXXXXXXXXX","sessionContext":{"sessionIssuer":{"type":"Role","principalId":"AROAXXXXXXXXXXXXXXXXX","arn":"arn:aws:iam::123456789012:role/example-role","accountId":"123456789012","userName":"example-role"},"attributes":{"creationDate":"2025-02-21T08:24:22Z","mfaAuthenticated":"false"}}},"eventTime":"2025-02-21T08:54:25Z","eventSource":"dynamodb.amazonaws.com","eventName":"Scan","awsRegion":"ap-south-1","sourceIPAddress":"192.0.2.1","userAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36","requestParameters":{"tableName":"TestDynamoDB","limit":1,"consistentRead":false},"responseElements":null,"requestID":"EXAMPLE1DVR44ANSU51S3GAVUKVVV4KQNSO5AEMVJF66Q9ASUAAJG","eventID":"ada136ef-f5a4-4730-9fa9-5c6ea8fbe162","readOnly":true,"resources":[{"accountId":"123456789012","type":"AWS::DynamoDB::Table","ARN":"arn:aws:dynamodb:ap-south-1:123456789012:table/TestDynamoDB"}],"eventType":"AwsApiCall","apiVersion":"2012-08-10","managementEvent":false,"recipientAccountId":"123456789012","eventCategory":"Data","tlsDetails":{"tlsVersion":"TLSv1.3","cipherSuite":"TLS_AES_128_GCM_SHA256","clientProvidedHostHeader":"dynamodb.ap-south-1.amazonaws.com"},"sessionCredentialFromConsole":"true"},{"eventVersion":"1.10","userIdentity":{"type":"AssumedRole","principalId":"AROAXXXXXXXXXXXXXXXXX:user@example.com","arn":"arn:aws:sts::123456789012:assumed-role/example-role/user@example.com","accountId":"123456789012","accessKeyId":"ASIAXXXXXXXXXXXXXXXX","sessionContext":{"sessionIssuer":{"type":"Role","principalId":"AROAXXXXXXXXXXXXXXXXX","arn":"arn:aws:iam::123456789012:role/example-role","accountId":"123456789012","userName":"example-role"},"attributes":{"creationDate":"2025-02-21T08:24:22Z","mfaAuthenticated":"false"}}},"eventTime":"2025-02-21T08:58:54Z","eventSource":"dynamodb.amazonaws.com","eventName":"PutItem","awsRegion":"ap-south-1","sourceIPAddress":"192.0.2.2","userAgent":"aws-cli/2.24.5 md/awscrt#0.23.8 ua/2.0 os/linux#6.1.127-135.201.amzn2023.x86_64 md/arch#x86_64 lang/python#3.12.6 md/pyimpl#CPython exec-env/CloudShell cfg/retry-mode#standard md/installer#exe md/distrib#amzn.2023 md/prompt#off md/command#dynamodb.put-item","requestParameters":{"tableName":"TestDynamoDB","key":{"TestDynamoDB_":"Item001"},"items":["Attribute3","Attribute2","Attribute1","TestDynamoDB_"]},"responseElements":null,"requestID":"EXAMPLE2IR1MQS4M9JNMI0RNKE17VV4KQNSO5AEMVJF66Q9ASUAAJG","eventID":"0934bf16-d19a-4a6a-8509-d401780529f2","readOnly":false,"resources":[{"accountId":"123456789012","type":"AWS::DynamoDB::Table","ARN":"arn:aws:dynamodb:ap-south-1:123456789012:table/TestDynamoDB"}],"eventType":"AwsApiCall","apiVersion":"2012-08-10","managementEvent":false,"recipientAccountId":"123456789012","eventCategory":"Data","tlsDetails":{"tlsVersion":"TLSv1.3","cipherSuite":"TLS_AES_256_GCM_SHA384","clientProvidedHostHeader":"dynamodb.ap-south-1.amazonaws.com"},"sessionCredentialFromConsole":"true"},{"eventVersion":"1.10","userIdentity":{"type":"AssumedRole","principalId":"AROAXXXXXXXXXXXXXXXXX:user@example.com","arn":"arn:aws:sts::123456789012:assumed-role/example-role/user@example.com","accountId":"123456789012","accessKeyId":"ASIAXXXXXXXXXXXXXXXX","sessionContext":{"sessionIssuer":{"type":"Role","principalId":"AROAXXXXXXXXXXXXXXXXX","arn":"arn:aws:iam::123456789012:role/example-role","accountId":"123456789012","userName":"example-role"},"attributes":{"creationDate":"2025-02-21T08:24:22Z","mfaAuthenticated":"false"}}},"eventTime":"2025-02-21T08:59:06Z","eventSource":"dynamodb.amazonaws.com","eventName":"GetItem","awsRegion":"ap-south-1","sourceIPAddress":"192.0.2.2","userAgent":"aws-cli/2.24.5 md/awscrt#0.23.8 ua/2.0 os/linux#6.1.127-135.201.amzn2023.x86_64 md/arch#x86_64 lang/python#3.12.6 md/pyimpl#CPython exec-env/CloudShell cfg/retry-mode#standard md/installer#exe md/distrib#amzn.2023 md/prompt#off md/command#dynamodb.get-item","requestParameters":{"tableName":"TestDynamoDB","key":{"TestDynamoDB_":"Item001"}},"responseElements":null,"requestID":"EXAMPLE3SQH0VGRSI8UA7C1HTHP7VV4KQNSO5AEMVJF66Q9ASUAAJG","eventID":"8c945250-068f-4fd1-856c-55df7d9f3418","readOnly":true,"resources":[{"accountId":"123456789012","type":"AWS::DynamoDB::Table","ARN":"arn:aws:dynamodb:ap-south-1:123456789012:table/TestDynamoDB"}],"eventType":"AwsApiCall","apiVersion":"2012-08-10","managementEvent":false,"recipientAccountId":"123456789012","eventCategory":"Data","tlsDetails":{"tlsVersion":"TLSv1.3","cipherSuite":"TLS_AES_256_GCM_SHA384","clientProvidedHostHeader":"dynamodb.ap-south-1.amazonaws.com"},"sessionCredentialFromConsole":"true"}]}' + add_field => { "include_account_in_host" => "true" } + count => 1 + type => "S3SQS" + } +} + +filter { + if [type] == 'S3SQS' and [message] { + + json { + source => "message" + target => "parsed_message" + } + + # Drop event if parsed_message does not contain "Records" + if ![parsed_message][Records] { + drop {} + } + + ruby { + code => ' + parsed_message = event.get("parsed_message") + if parsed_message && parsed_message["Records"].is_a?(Array) + parsed_message["Records"].each do |record| + record["type"] = event.get("type") + record["timestamp"] = event.get("timestamp") || Time.now.to_i + record["fileKey"] = event.get("fileKey") || "unknown" + record["@version"] = event.get("@version") + record["@timestamp"] = event.get("@timestamp") + end + event.set("message", parsed_message["Records"].to_json) + else + event.tag("json_parse_error") + end + ' + } + + dynamodb_guardium_plugin_filter {} + + prune { whitelist_names => [ "GuardRecord" ] } + } +} + +output { + stdout { codec => rubydebug } +} + diff --git a/filter-plugin/logstash-filter-dynamodb-guardium/build.gradle b/filter-plugin/logstash-filter-dynamodb-guardium/build.gradle index 4a66909ef..969700b7b 100644 --- a/filter-plugin/logstash-filter-dynamodb-guardium/build.gradle +++ b/filter-plugin/logstash-filter-dynamodb-guardium/build.gradle @@ -2,28 +2,52 @@ import java.nio.file.Files import static java.nio.file.StandardCopyOption.REPLACE_EXISTING apply plugin: 'java' + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + + dependencies { + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' + classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.1.0" + classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' + } +} + +def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" // =========================================================================== // plugin info // =========================================================================== -group 'com.ibm.guardium.dynamodb' // must match the package of the main plugin class -version "${file("VERSION").text.trim()}" // read from required VERSION file -description = "DynamoDB-Guardium filter plugin" -pluginInfo.licenses = ['Apache-2.0'] // list of SPDX license IDs +group 'com.ibm.guardium.dynamodb' // must match the package of the main plugin class +version "${file("VERSION").text.trim()}" // read from required VERSION file +description = "DynamoDB-Guardium filter plugin" +pluginInfo.licenses = ['Apache-2.0'] // list of SPDX license IDs pluginInfo.longDescription = "This gem is a Logstash Dynamodb filter plugin required to be installed as part of IBM Security Guardium, Guardium Universal connector configuration. This gem is not a stand-alone program." -pluginInfo.authors = ['IBM'] -pluginInfo.email = [''] -pluginInfo.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html" -pluginInfo.pluginType = "filter" -pluginInfo.pluginClass = "DynamodbGuardiumPluginFilter" -pluginInfo.pluginName = "dynamodb_guardium_plugin_filter" // must match the @LogstashPlugin annotation in the main plugin class +pluginInfo.authors = ['IBM'] +pluginInfo.email = [''] +pluginInfo.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html" +pluginInfo.pluginType = "filter" +pluginInfo.pluginClass = "DynamodbGuardiumPluginFilter" +pluginInfo.pluginName = "dynamodb_guardium_plugin_filter" // must match the @LogstashPlugin annotation in the main plugin class // =========================================================================== -sourceCompatibility = 1.8 -targetCompatibility = 1.8 +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 -def jacocoVersion = '0.8.4' +def jacocoVersion = '0.8.11' // minimumCoverage can be set by Travis ENV def minimumCoverageStr = System.getenv("MINIMUM_COVERAGE") ?: "50.0%" @@ -33,31 +57,20 @@ if (minimumCoverageStr.endsWith("%")) { def minimumCoverage = Float.valueOf(minimumCoverageStr) / 100 -buildscript { - repositories { - maven { - url "https://plugins.gradle.org/m2/" - } - mavenCentral() - jcenter() - } - - dependencies { - classpath 'com.github.jengelman.gradle.plugins:shadow:4.0.4' - classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.0.0-RC2" - classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' - } -} - -def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); -def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) repositories { - mavenCentral() + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } } -tasks.register("vendor"){ +tasks.register("vendor") { dependsOn shadowJar doLast { String vendorPathPrefix = "vendor/jar-dependencies" @@ -65,14 +78,13 @@ tasks.register("vendor"){ File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") projectJarFile.mkdirs() Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) - validatePluginJar(projectJarFile, project.group) } } apply plugin: 'com.github.johnrengelman.shadow' shadowJar { - classifier = null + archiveClassifier = null } dependencies { @@ -80,9 +92,9 @@ dependencies { implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: versions.dependencies.log4jCore implementation 'org.apache.commons:commons-lang3:' + versions.dependencies.commonsLang implementation 'com.google.code.gson:gson:' + versions.dependencies.gson + implementation group: 'commons-beanutils', name: 'commons-beanutils', version: versions.dependencies.commonsBeanutils implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "build/libs/logstash-core-*.*.*.jar") implementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "common-*.*.*.jar") - testImplementation 'junit:junit:' + versions.dependencies.junit testImplementation 'org.jruby:jruby-complete:' + versions.dependencies.jrubyComplete @@ -104,6 +116,16 @@ tasks.withType(JavaCompile) { options.encoding = 'UTF-8' } +test { + jvmArgs = [ + '--add-opens=java.base/java.lang=ALL-UNNAMED', + '--add-opens=java.base/java.util=ALL-UNNAMED', + '--add-opens=java.base/java.text=ALL-UNNAMED', + '--add-opens=java.base/java.lang.reflect=ALL-UNNAMED', + '--add-opens=java.base/sun.util.resources=ALL-UNNAMED', + '--add-opens=java.base/sun.util.cldr=ALL-UNNAMED' + ] +} tasks.register("generateRubySupportFiles") { @@ -123,7 +145,7 @@ tasks.register("removeObsoleteJars") { } } -tasks.register("gem"){ +tasks.register("gem") { dependsOn = [downloadAndInstallJRuby, removeObsoleteJars, vendor, generateRubySupportFiles] doLast { buildGem(projectDir, buildDir, pluginInfo.pluginFullName() + ".gemspec") @@ -140,18 +162,17 @@ apply plugin: "org.barfuin.gradle.jacocolog" jacoco { toolVersion = "${jacocoVersion}" - reportsDir = file("$buildDir/reports/jacoco") } jacocoTestReport { // You will see "Report -> file://...." at the end of a JaCoCo build // If no output, run this first: ./gradlew test reports { - html.enabled true - xml.enabled true - csv.enabled true - html.destination file("${buildDir}/reports/jacoco") - csv.destination file("${buildDir}/reports/jacoco/all.csv") + html.required = true + xml.required = true + csv.required = true + html.outputLocation = file("${buildDir}/reports/jacoco") + csv.outputLocation = file("${buildDir}/reports/jacoco/all.csv") } executionData.from fileTree(dir: "${buildDir}/jacoco/", includes: [ '**/*.exec' diff --git a/filter-plugin/logstash-filter-dynamodb-guardium/gradle/wrapper/gradle-wrapper.properties b/filter-plugin/logstash-filter-dynamodb-guardium/gradle/wrapper/gradle-wrapper.properties index 60c76b340..ba9ccfe4c 100644 --- a/filter-plugin/logstash-filter-dynamodb-guardium/gradle/wrapper/gradle-wrapper.properties +++ b/filter-plugin/logstash-filter-dynamodb-guardium/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists \ No newline at end of file diff --git a/filter-plugin/logstash-filter-dynamodb-guardium/logstash-filter-dynamodb_guardium_plugin_filter.zip b/filter-plugin/logstash-filter-dynamodb-guardium/logstash-filter-dynamodb_guardium_plugin_filter.zip index 75dedf3ff..ac36e67d2 100644 Binary files a/filter-plugin/logstash-filter-dynamodb-guardium/logstash-filter-dynamodb_guardium_plugin_filter.zip and b/filter-plugin/logstash-filter-dynamodb-guardium/logstash-filter-dynamodb_guardium_plugin_filter.zip differ diff --git a/filter-plugin/logstash-filter-dynamodb-guardium/mainREADME.md b/filter-plugin/logstash-filter-dynamodb-guardium/mainREADME.md new file mode 100644 index 000000000..aec44d408 --- /dev/null +++ b/filter-plugin/logstash-filter-dynamodb-guardium/mainREADME.md @@ -0,0 +1,10 @@ +# DynamoDB Universal Connector + +## Follow this link to set up and use DynamoDB Universal Connector over CloudWatch Logstash Plugin + +[DynamoDBOverCloudwatch](./README.md) + +## Follow this link to set up and use DynamoDB Universal Connector over CloudWatch Connect + +[DynamoDBOverConnectCloudwatch](../../docs/KafkaBasedUCs/DynamoDBCloudwatchKafkaConnect.md) + diff --git a/filter-plugin/logstash-filter-dynamodb-guardium/src/main/java/com/ibm/guardium/dynamodb/Constants.java b/filter-plugin/logstash-filter-dynamodb-guardium/src/main/java/com/ibm/guardium/dynamodb/Constants.java index 1926599eb..769137d8b 100644 --- a/filter-plugin/logstash-filter-dynamodb-guardium/src/main/java/com/ibm/guardium/dynamodb/Constants.java +++ b/filter-plugin/logstash-filter-dynamodb-guardium/src/main/java/com/ibm/guardium/dynamodb/Constants.java @@ -6,7 +6,7 @@ public interface Constants { - public static final String NOT_AVAILABLE = "NA"; + public static final String NOT_AVAILABLE = "N.A."; public static final String DATA_PROTOCOL_STRING = "AMAZON DYNAMODB"; public static final String UNKNOWN_STRING = ""; public static final String SERVER_TYPE_STRING = "DYNAMODB"; diff --git a/filter-plugin/logstash-filter-dynamodb-guardium/src/main/java/com/ibm/guardium/dynamodb/DynamodbGuardiumPluginFilter.java b/filter-plugin/logstash-filter-dynamodb-guardium/src/main/java/com/ibm/guardium/dynamodb/DynamodbGuardiumPluginFilter.java index 6e899e2f8..50800798b 100644 --- a/filter-plugin/logstash-filter-dynamodb-guardium/src/main/java/com/ibm/guardium/dynamodb/DynamodbGuardiumPluginFilter.java +++ b/filter-plugin/logstash-filter-dynamodb-guardium/src/main/java/com/ibm/guardium/dynamodb/DynamodbGuardiumPluginFilter.java @@ -15,7 +15,15 @@ import com.google.gson.stream.JsonReader; import com.ibm.guardium.universalconnector.commons.GuardConstants; import com.ibm.guardium.universalconnector.commons.Util; -import com.ibm.guardium.universalconnector.commons.structures.*; +import com.ibm.guardium.universalconnector.commons.structures.Accessor; +import com.ibm.guardium.universalconnector.commons.structures.Construct; +import com.ibm.guardium.universalconnector.commons.structures.Data; +import com.ibm.guardium.universalconnector.commons.structures.ExceptionRecord; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import com.ibm.guardium.universalconnector.commons.structures.Sentence; +import com.ibm.guardium.universalconnector.commons.structures.SentenceObject; +import com.ibm.guardium.universalconnector.commons.structures.SessionLocator; +import com.ibm.guardium.universalconnector.commons.structures.Time; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/filter-plugin/logstash-filter-dynamodb-guardium/src/main/java/com/ibm/guardium/dynamodb/Parser.java b/filter-plugin/logstash-filter-dynamodb-guardium/src/main/java/com/ibm/guardium/dynamodb/Parser.java index e6e784046..9336efd19 100644 --- a/filter-plugin/logstash-filter-dynamodb-guardium/src/main/java/com/ibm/guardium/dynamodb/Parser.java +++ b/filter-plugin/logstash-filter-dynamodb-guardium/src/main/java/com/ibm/guardium/dynamodb/Parser.java @@ -23,7 +23,15 @@ import com.google.gson.JsonPrimitive; import com.google.gson.JsonParser; import com.ibm.guardium.universalconnector.commons.Util; -import com.ibm.guardium.universalconnector.commons.structures.*; +import com.ibm.guardium.universalconnector.commons.structures.Accessor; +import com.ibm.guardium.universalconnector.commons.structures.Construct; +import com.ibm.guardium.universalconnector.commons.structures.Data; +import com.ibm.guardium.universalconnector.commons.structures.ExceptionRecord; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import com.ibm.guardium.universalconnector.commons.structures.Sentence; +import com.ibm.guardium.universalconnector.commons.structures.SentenceObject; +import com.ibm.guardium.universalconnector.commons.structures.SessionLocator; +import com.ibm.guardium.universalconnector.commons.structures.Time; import com.ibm.guardium.universalconnector.commons.structures.Sentence; import com.ibm.guardium.universalconnector.commons.structures.SentenceObject; diff --git a/filter-plugin/logstash-filter-dynamodb-guardium/src/test/java/com/ibm/guardium/dynamodb/ParserTest.java b/filter-plugin/logstash-filter-dynamodb-guardium/src/test/java/com/ibm/guardium/dynamodb/ParserTest.java index b8df3be65..2457cfc6b 100644 --- a/filter-plugin/logstash-filter-dynamodb-guardium/src/test/java/com/ibm/guardium/dynamodb/ParserTest.java +++ b/filter-plugin/logstash-filter-dynamodb-guardium/src/test/java/com/ibm/guardium/dynamodb/ParserTest.java @@ -8,7 +8,15 @@ import com.google.gson.JsonObject; import com.google.gson.JsonParser; -import com.ibm.guardium.universalconnector.commons.structures.*; +import com.ibm.guardium.universalconnector.commons.structures.Accessor; +import com.ibm.guardium.universalconnector.commons.structures.Construct; +import com.ibm.guardium.universalconnector.commons.structures.Data; +import com.ibm.guardium.universalconnector.commons.structures.ExceptionRecord; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import com.ibm.guardium.universalconnector.commons.structures.Sentence; +import com.ibm.guardium.universalconnector.commons.structures.SentenceObject; +import com.ibm.guardium.universalconnector.commons.structures.SessionLocator; +import com.ibm.guardium.universalconnector.commons.structures.Time; import org.junit.Assert; import org.junit.Test; diff --git a/filter-plugin/logstash-filter-elasticsearch-guardium/README.md b/filter-plugin/logstash-filter-elasticsearch-guardium/README.md index 4bd8c5752..653d07992 100644 --- a/filter-plugin/logstash-filter-elasticsearch-guardium/README.md +++ b/filter-plugin/logstash-filter-elasticsearch-guardium/README.md @@ -233,7 +233,8 @@ https://www.elastic.co/guide/en/beats/filebeat/current/directory-layout.html For example:- filebeat.inputs: - - type: log + - type: filestream + - id: enabled: true paths: - /var/log/elasticsearch/_audit.json diff --git a/filter-plugin/logstash-filter-elasticsearch-guardium/build.gradle b/filter-plugin/logstash-filter-elasticsearch-guardium/build.gradle index e0a272e68..8ef1b39c9 100644 --- a/filter-plugin/logstash-filter-elasticsearch-guardium/build.gradle +++ b/filter-plugin/logstash-filter-elasticsearch-guardium/build.gradle @@ -22,45 +22,56 @@ pluginInfo.pluginName = "elasticsearch_guardium_filter" // must match the @ // =========================================================================== -sourceCompatibility = 1.8 -targetCompatibility = 1.8 +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 buildscript { repositories { + mavenLocal() maven { - url "https://plugins.gradle.org/m2/" + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } } - mavenCentral() - jcenter() } dependencies { - classpath 'com.github.jengelman.gradle.plugins:shadow:4.0.4' + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' } } repositories { - mavenCentral() + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } } apply plugin: 'com.github.johnrengelman.shadow' shadowJar { - classifier = null + archiveClassifier = null transform(com.github.jengelman.gradle.plugins.shadow.transformers.Log4j2PluginsCacheFileTransformer) } dependencies { - compile group: 'commons-validator', name: 'commons-validator', version: '1.7' - compile group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.17.1' - compile 'org.apache.commons:commons-lang3:3.7' - compile 'com.google.code.gson:gson:2.8.9' - compile fileTree(dir: LOGSTASH_CORE_PATH, include: "build/libs/logstash-core-*.*.*.jar") - compile fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "guardium-universalconnector-commons-master-*.*.*.jar") - testCompile 'junit:junit:4.12' - testCompile 'org.jruby:jruby-complete:9.2.7.0' + implementation group: 'commons-validator', name: 'commons-validator', version: '1.7' + implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.17.1' + implementation group: 'commons-beanutils', name: 'commons-beanutils', version: versions.dependencies.commonsBeanutils + implementation 'org.apache.commons:commons-lang3:3.7' + implementation 'com.google.code.gson:gson:2.8.9' + implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "build/libs/logstash-core-*.*.*.jar") + implementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "guardium-universalconnector-commons-master-*.*.*.jar") + testImplementation 'junit:junit:4.12' + testImplementation 'org.jruby:jruby-complete:9.2.7.0' - testCompile fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "guardium-universalconnector-commons-master-*.*.*.jar") + testImplementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "guardium-universalconnector-commons-master-*.*.*.jar") } clean { @@ -78,6 +89,17 @@ tasks.withType(JavaCompile) { options.encoding = 'UTF-8' } +test { + jvmArgs = [ + '--add-opens=java.base/java.lang=ALL-UNNAMED', + '--add-opens=java.base/java.util=ALL-UNNAMED', + '--add-opens=java.base/java.text=ALL-UNNAMED', + '--add-opens=java.base/java.lang.reflect=ALL-UNNAMED', + '--add-opens=java.base/sun.util.resources=ALL-UNNAMED', + '--add-opens=java.base/sun.util.cldr=ALL-UNNAMED' + ] +} + tasks.register("vendor"){ dependsOn shadowJar doLast { @@ -86,7 +108,6 @@ tasks.register("vendor"){ File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") projectJarFile.mkdirs() Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) - validatePluginJar(projectJarFile, project.group) } } @@ -115,8 +136,8 @@ tasks.register("gem"){ } jacocoTestReport { reports { - xml.enabled true - html.enabled true + xml.required = true + html.required = true } afterEvaluate { // (optional) : to exclude classes / packages from coverage diff --git a/filter-plugin/logstash-filter-elasticsearch-guardium/gradle/wrapper/gradle-wrapper.properties b/filter-plugin/logstash-filter-elasticsearch-guardium/gradle/wrapper/gradle-wrapper.properties index bb8b2fc26..81aa1c044 100644 --- a/filter-plugin/logstash-filter-elasticsearch-guardium/gradle/wrapper/gradle-wrapper.properties +++ b/filter-plugin/logstash-filter-elasticsearch-guardium/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-6.5.1-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/filter-plugin/logstash-filter-generic-guardium/GenericDBPackage/GenericDB/genericCloudwatch.conf b/filter-plugin/logstash-filter-generic-guardium/GenericDBPackage/GenericDB/genericCloudwatch.conf index 88e168de2..220d7de0b 100644 --- a/filter-plugin/logstash-filter-generic-guardium/GenericDBPackage/GenericDB/genericCloudwatch.conf +++ b/filter-plugin/logstash-filter-generic-guardium/GenericDBPackage/GenericDB/genericCloudwatch.conf @@ -56,7 +56,7 @@ filter { mutate {add_field => { "addAppUserName" => "AWSService" } } mutate { add_field => {"addServerType" => "Postgre"} } mutate { add_field => {"addCommProtocol" => "AwsApiCall"} } - mutate { add_field => {"addDbProtocol" => "Postgre AWS Native Audit" } } + mutate { add_field => {"addDbProtocol" => "Postgre AWS" } } mutate { add_field => {"addLanguage" => "PGRS" } } mutate { add_field => {"addDataType" => "TEXT" } } diff --git a/filter-plugin/logstash-filter-generic-guardium/build.gradle b/filter-plugin/logstash-filter-generic-guardium/build.gradle index e63a2ae9b..141613f2c 100644 --- a/filter-plugin/logstash-filter-generic-guardium/build.gradle +++ b/filter-plugin/logstash-filter-generic-guardium/build.gradle @@ -2,6 +2,29 @@ import java.nio.file.Files import static java.nio.file.StandardCopyOption.REPLACE_EXISTING apply plugin: 'java' + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + + dependencies { + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' + classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' + } +} + +def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" // =========================================================================== @@ -20,29 +43,19 @@ pluginInfo.pluginClass = "GenericGuardiumFilter" pluginInfo.pluginName = "generic_guardium_filter" // must match the @LogstashPlugin annotation in the main plugin class // =========================================================================== -sourceCompatibility = 1.8 -targetCompatibility = 1.8 - -buildscript { - repositories { - maven { - url "https://plugins.gradle.org/m2/" - } - mavenCentral() - jcenter() - } - - dependencies { - classpath 'com.github.jengelman.gradle.plugins:shadow:4.0.4' - classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' - } -} +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 -def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); -def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) repositories { - mavenCentral() + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } } @@ -54,14 +67,13 @@ tasks.register("vendor"){ File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") projectJarFile.mkdirs() Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) - validatePluginJar(projectJarFile, project.group) } } apply plugin: 'com.github.johnrengelman.shadow' shadowJar { - classifier = null + archiveClassifier = null } dependencies { @@ -69,6 +81,7 @@ dependencies { implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: versions.dependencies.log4jCore implementation 'org.apache.commons:commons-lang3:' + versions.dependencies.commonsLang implementation 'com.google.code.gson:gson:' + versions.dependencies.gson + implementation group: 'commons-beanutils', name: 'commons-beanutils', version: versions.dependencies.commonsBeanutils implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "build/libs/logstash-core-*.*.*.jar") implementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "common-*.*.*.jar") @@ -92,6 +105,17 @@ tasks.withType(JavaCompile) { options.encoding = 'UTF-8' } +test { + jvmArgs = [ + '--add-opens=java.base/java.lang=ALL-UNNAMED', + '--add-opens=java.base/java.util=ALL-UNNAMED', + '--add-opens=java.base/java.text=ALL-UNNAMED', + '--add-opens=java.base/java.lang.reflect=ALL-UNNAMED', + '--add-opens=java.base/sun.util.resources=ALL-UNNAMED', + '--add-opens=java.base/sun.util.cldr=ALL-UNNAMED' + ] +} + tasks.register("generateRubySupportFiles") { doLast { diff --git a/filter-plugin/logstash-filter-generic-guardium/filter-test-generator-postgres.conf b/filter-plugin/logstash-filter-generic-guardium/filter-test-generator-postgres.conf index 844058d49..9ed6e3f5d 100644 --- a/filter-plugin/logstash-filter-generic-guardium/filter-test-generator-postgres.conf +++ b/filter-plugin/logstash-filter-generic-guardium/filter-test-generator-postgres.conf @@ -53,7 +53,7 @@ filter { mutate {add_field => { "addAppUserName" => "AWSService" } } mutate { add_field => {"addServerType" => "Postgre"} } mutate { add_field => {"addCommProtocol" => "AwsApiCall"} } - mutate { add_field => {"addDbProtocol" => "Postgre AWS Native Audit" } } + mutate { add_field => {"addDbProtocol" => "Postgre AWS" } } mutate { add_field => {"addLanguage" => "PGRS" } } mutate { add_field => {"addDataType" => "TEXT" } } diff --git a/filter-plugin/logstash-filter-generic-guardium/gradle/wrapper/gradle-wrapper.properties b/filter-plugin/logstash-filter-generic-guardium/gradle/wrapper/gradle-wrapper.properties index aa991fcea..81aa1c044 100644 --- a/filter-plugin/logstash-filter-generic-guardium/gradle/wrapper/gradle-wrapper.properties +++ b/filter-plugin/logstash-filter-generic-guardium/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/filter-plugin/logstash-filter-generic-guardium/src/main/java/com/ibm/guardium/generic/GenericGuardiumFilter.java b/filter-plugin/logstash-filter-generic-guardium/src/main/java/com/ibm/guardium/generic/GenericGuardiumFilter.java index b0e65266b..5d94f5520 100644 --- a/filter-plugin/logstash-filter-generic-guardium/src/main/java/com/ibm/guardium/generic/GenericGuardiumFilter.java +++ b/filter-plugin/logstash-filter-generic-guardium/src/main/java/com/ibm/guardium/generic/GenericGuardiumFilter.java @@ -14,7 +14,15 @@ import com.google.gson.*; import com.ibm.guardium.universalconnector.commons.GuardConstants; import com.ibm.guardium.universalconnector.commons.Util; -import com.ibm.guardium.universalconnector.commons.structures.*; +import com.ibm.guardium.universalconnector.commons.structures.Accessor; +import com.ibm.guardium.universalconnector.commons.structures.Construct; +import com.ibm.guardium.universalconnector.commons.structures.Data; +import com.ibm.guardium.universalconnector.commons.structures.ExceptionRecord; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import com.ibm.guardium.universalconnector.commons.structures.Sentence; +import com.ibm.guardium.universalconnector.commons.structures.SentenceObject; +import com.ibm.guardium.universalconnector.commons.structures.SessionLocator; +import com.ibm.guardium.universalconnector.commons.structures.Time; import org.apache.commons.validator.routines.InetAddressValidator; import org.apache.logging.log4j.LogManager; diff --git a/filter-plugin/logstash-filter-generic-guardium/src/test/java/com/ibm/guardium/generic/ParserTest.java b/filter-plugin/logstash-filter-generic-guardium/src/test/java/com/ibm/guardium/generic/ParserTest.java index af798995a..e7cefc85f 100644 --- a/filter-plugin/logstash-filter-generic-guardium/src/test/java/com/ibm/guardium/generic/ParserTest.java +++ b/filter-plugin/logstash-filter-generic-guardium/src/test/java/com/ibm/guardium/generic/ParserTest.java @@ -11,7 +11,15 @@ import com.google.gson.JsonParser; import com.ibm.guardium.generic.Constants; import com.ibm.guardium.generic.Parser; -import com.ibm.guardium.universalconnector.commons.structures.*; +import com.ibm.guardium.universalconnector.commons.structures.Accessor; +import com.ibm.guardium.universalconnector.commons.structures.Construct; +import com.ibm.guardium.universalconnector.commons.structures.Data; +import com.ibm.guardium.universalconnector.commons.structures.ExceptionRecord; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import com.ibm.guardium.universalconnector.commons.structures.Sentence; +import com.ibm.guardium.universalconnector.commons.structures.SentenceObject; +import com.ibm.guardium.universalconnector.commons.structures.SessionLocator; +import com.ibm.guardium.universalconnector.commons.structures.Time; import org.junit.Assert; import org.junit.Test; diff --git a/filter-plugin/logstash-filter-hdfs-guardium/README.md b/filter-plugin/logstash-filter-hdfs-guardium/README.md index 7c15ea076..92f9331b3 100644 --- a/filter-plugin/logstash-filter-hdfs-guardium/README.md +++ b/filter-plugin/logstash-filter-hdfs-guardium/README.md @@ -37,7 +37,8 @@ https://www.elastic.co/guide/en/beats/filebeat/current/directory-layout.html ``` filebeat.inputs: - - type: log + - type: filestream + - id: enabled: true paths: - /var/log/hadoop-hdfs/hdfs-audit.log* diff --git a/filter-plugin/logstash-filter-hdfs-guardium/build.gradle b/filter-plugin/logstash-filter-hdfs-guardium/build.gradle index 1b18b3c36..2f3a667fe 100644 --- a/filter-plugin/logstash-filter-hdfs-guardium/build.gradle +++ b/filter-plugin/logstash-filter-hdfs-guardium/build.gradle @@ -2,6 +2,30 @@ import java.nio.file.Files import static java.nio.file.StandardCopyOption.REPLACE_EXISTING apply plugin: 'java' + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + + dependencies { + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' + classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.1.0" + classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' + } +} + +def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" // =========================================================================== @@ -20,10 +44,10 @@ pluginInfo.pluginClass = "HdfsGuardiumFilter" pluginInfo.pluginName = "hdfs_guardium_filter" // must match the @LogstashPlugin annotation in the main plugin class // =========================================================================== -sourceCompatibility = 1.8 -targetCompatibility = 1.8 +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 -def jacocoVersion = '0.8.4' +def jacocoVersion = '0.8.11' // minimumCoverage can be set by Travis ENV def minimumCoverageStr = System.getenv("MINIMUM_COVERAGE") ?: "50.0%" if (minimumCoverageStr.endsWith("%")) { @@ -31,27 +55,16 @@ if (minimumCoverageStr.endsWith("%")) { } def minimumCoverage = Float.valueOf(minimumCoverageStr) / 100 -buildscript { - repositories { - maven { - url "https://plugins.gradle.org/m2/" - } - mavenCentral() - jcenter() - } - - dependencies { - classpath 'com.github.jengelman.gradle.plugins:shadow:4.0.4' - classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.0.0-RC2" - classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' - } -} - -def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); -def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) repositories { - mavenCentral() + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } } tasks.register("vendor"){ @@ -62,14 +75,13 @@ tasks.register("vendor"){ File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") projectJarFile.mkdirs() Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) - validatePluginJar(projectJarFile, project.group) } } apply plugin: 'com.github.johnrengelman.shadow' shadowJar { - classifier = null + archiveClassifier = null } dependencies { @@ -77,6 +89,7 @@ dependencies { implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: versions.dependencies.log4jCore implementation 'org.apache.commons:commons-lang3:' + versions.dependencies.commonsLang implementation 'com.google.code.gson:gson:' + versions.dependencies.gson + implementation group: 'commons-beanutils', name: 'commons-beanutils', version: versions.dependencies.commonsBeanutils implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "build/libs/logstash-core-*.*.*.jar") implementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "common-*.*.*.jar") @@ -100,6 +113,17 @@ tasks.withType(JavaCompile) { options.encoding = 'UTF-8' } +test { + jvmArgs = [ + '--add-opens=java.base/java.lang=ALL-UNNAMED', + '--add-opens=java.base/java.util=ALL-UNNAMED', + '--add-opens=java.base/java.text=ALL-UNNAMED', + '--add-opens=java.base/java.lang.reflect=ALL-UNNAMED', + '--add-opens=java.base/sun.util.resources=ALL-UNNAMED', + '--add-opens=java.base/sun.util.cldr=ALL-UNNAMED' + ] +} + tasks.register("generateRubySupportFiles") { @@ -142,17 +166,16 @@ apply plugin: "org.barfuin.gradle.jacocolog" // ------------------------------------ jacoco { toolVersion = "${jacocoVersion}" - reportsDir = file("$buildDir/reports/jacoco") } jacocoTestReport { // You will see "Report -> file://...." at the end of a JaCoCo build // If no output, run this first: ./gradlew test reports { - html.enabled true - xml.enabled true - csv.enabled true - html.destination file("${buildDir}/reports/jacoco") - csv.destination file("${buildDir}/reports/jacoco/all.csv") + html.required = true + xml.required = true + csv.required = true + html.outputLocation = file("${buildDir}/reports/jacoco") + csv.outputLocation = file("${buildDir}/reports/jacoco/all.csv") } executionData.from fileTree(dir: "${buildDir}/jacoco/", includes: [ '**/*.exec' diff --git a/filter-plugin/logstash-filter-hdfs-guardium/gradle/wrapper/gradle-wrapper.properties b/filter-plugin/logstash-filter-hdfs-guardium/gradle/wrapper/gradle-wrapper.properties index aa991fcea..81aa1c044 100644 --- a/filter-plugin/logstash-filter-hdfs-guardium/gradle/wrapper/gradle-wrapper.properties +++ b/filter-plugin/logstash-filter-hdfs-guardium/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/filter-plugin/logstash-filter-hdfs-guardium/src/main/java/com/ibm/guardium/hdfs/HdfsGuardiumFilter.java b/filter-plugin/logstash-filter-hdfs-guardium/src/main/java/com/ibm/guardium/hdfs/HdfsGuardiumFilter.java index ba749718b..776cbca6d 100644 --- a/filter-plugin/logstash-filter-hdfs-guardium/src/main/java/com/ibm/guardium/hdfs/HdfsGuardiumFilter.java +++ b/filter-plugin/logstash-filter-hdfs-guardium/src/main/java/com/ibm/guardium/hdfs/HdfsGuardiumFilter.java @@ -16,7 +16,15 @@ import com.google.gson.*; import com.ibm.guardium.universalconnector.commons.GuardConstants; import com.ibm.guardium.universalconnector.commons.Util; -import com.ibm.guardium.universalconnector.commons.structures.*; +import com.ibm.guardium.universalconnector.commons.structures.Accessor; +import com.ibm.guardium.universalconnector.commons.structures.Construct; +import com.ibm.guardium.universalconnector.commons.structures.Data; +import com.ibm.guardium.universalconnector.commons.structures.ExceptionRecord; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import com.ibm.guardium.universalconnector.commons.structures.Sentence; +import com.ibm.guardium.universalconnector.commons.structures.SentenceObject; +import com.ibm.guardium.universalconnector.commons.structures.SessionLocator; +import com.ibm.guardium.universalconnector.commons.structures.Time; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -47,7 +55,7 @@ public class HdfsGuardiumFilter implements Filter { private String id; public static final String HDFS_AUDIT_MARK_STRING = "FSNamesystem.audit"; - public static final String DATA_PROTOCOL_STRING = "HDFS native audit"; + public static final String DATA_PROTOCOL_STRING = "HDFS"; public static final String UNKNOWN_STRING = ""; public static final String SERVER_TYPE_STRING = "HDFS"; public static final String APP_NAME_STRING = "HADOOP CLIENT PROGRAM"; diff --git a/filter-plugin/logstash-filter-intersystems-iris-guardium/build.gradle b/filter-plugin/logstash-filter-intersystems-iris-guardium/build.gradle index e2ab442c5..1cbfc60d3 100644 --- a/filter-plugin/logstash-filter-intersystems-iris-guardium/build.gradle +++ b/filter-plugin/logstash-filter-intersystems-iris-guardium/build.gradle @@ -2,6 +2,29 @@ import java.nio.file.Files import static java.nio.file.StandardCopyOption.REPLACE_EXISTING apply plugin: 'java' + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + + dependencies { + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' + classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' + } +} + +def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + apply plugin: 'jacoco' apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" // =========================================================================== @@ -20,35 +43,25 @@ pluginInfo.pluginClass = "IntersystemsIRISGuardiumFilter" pluginInfo.pluginName = "intersystems_iris_guardium_filter" // must match the @LogstashPlugin annotation in the main plugin class // =========================================================================== -sourceCompatibility = 1.8 -targetCompatibility = 1.8 - -buildscript { - repositories { - maven { - url "https://plugins.gradle.org/m2/" - } - mavenCentral() - jcenter() - } - - dependencies { - classpath 'com.github.jengelman.gradle.plugins:shadow:4.0.4' - classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' - } -} +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 -def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); -def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) repositories { - mavenCentral() + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } } apply plugin: 'com.github.johnrengelman.shadow' shadowJar { - classifier = null + archiveClassifier = null transform(com.github.jengelman.gradle.plugins.shadow.transformers.Log4j2PluginsCacheFileTransformer) } @@ -57,7 +70,8 @@ dependencies { implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: versions.dependencies.log4jCore implementation 'org.apache.commons:commons-lang3:' + versions.dependencies.commonsLang implementation 'com.google.code.gson:gson:' + versions.dependencies.gson - implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "build/libs/logstash-core-*.*.*.jar") + implementation group: 'commons-beanutils', name: 'commons-beanutils', version: versions.dependencies.commonsBeanutils + implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "build/libs/logstash-core.jar") implementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "common-*.*.*.jar") implementation group: 'org.parboiled', name: 'parboiled-java', version: versions.dependencies.parboiledJava implementation group: 'org.json', name: 'json', version: versions.dependencies.json @@ -83,6 +97,17 @@ tasks.withType(JavaCompile) { options.encoding = 'UTF-8' } +test { + jvmArgs = [ + '--add-opens=java.base/java.lang=ALL-UNNAMED', + '--add-opens=java.base/java.util=ALL-UNNAMED', + '--add-opens=java.base/java.text=ALL-UNNAMED', + '--add-opens=java.base/java.lang.reflect=ALL-UNNAMED', + '--add-opens=java.base/sun.util.resources=ALL-UNNAMED', + '--add-opens=java.base/sun.util.cldr=ALL-UNNAMED' + ] +} + tasks.register("vendor"){ dependsOn shadowJar doLast { @@ -91,7 +116,6 @@ tasks.register("vendor"){ File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") projectJarFile.mkdirs() Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) - validatePluginJar(projectJarFile, project.group) } } @@ -120,8 +144,8 @@ tasks.register("gem"){ } jacocoTestReport { reports { - xml.enabled true - html.enabled true + xml.required = true + html.required = true } afterEvaluate { // (optional) : to exclude classes / packages from coverage diff --git a/filter-plugin/logstash-filter-intersystems-iris-guardium/gradle/wrapper/gradle-wrapper.properties b/filter-plugin/logstash-filter-intersystems-iris-guardium/gradle/wrapper/gradle-wrapper.properties index aa991fcea..81aa1c044 100644 --- a/filter-plugin/logstash-filter-intersystems-iris-guardium/gradle/wrapper/gradle-wrapper.properties +++ b/filter-plugin/logstash-filter-intersystems-iris-guardium/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/filter-plugin/logstash-filter-mariadb-aws-guardium/README.md b/filter-plugin/logstash-filter-mariadb-aws-guardium/README.md index b63f537c6..fb43cd086 100644 --- a/filter-plugin/logstash-filter-mariadb-aws-guardium/README.md +++ b/filter-plugin/logstash-filter-mariadb-aws-guardium/README.md @@ -63,6 +63,7 @@ To add `MARIADB_AUDIT_PLUGIN` which will enable Server Audit Logs. - Select the created Option groups and then click **Add options**. - Set the **option name** to MARIADB_AUDIT_PLUGIN, and then keep option setting parameters with the default values - Change the `SERVER_AUDIT_EXCL_USERS` value to rdsadmin + - Set the value for `SERVER_AUDIT_EVENTS` to `QUERY, CONNECT` in order to see query and connection logs. - To enable the option immediately, choose **Yes** for **Apply Immediately**. (By default, **No** is selected instead.) Keep this default selection if you want the option enabled for each associated database instance during its next maintenance window. - Click **Add option**. @@ -147,7 +148,8 @@ The Guardium universal connector is the Guardium entry point for native audit lo - Client HostName : Not available with audit logs when we connect to the MariaDB instance through SQL standard and third party tools. - serverIP : This field is populated with 0.0.0.0, as this information is not embedded in the messages pulled from AWS Cloudwatch. - clientPort and serverPort : Not available with audit logs - - For system generated LOGIN_FAILED logs, the Dbuser value not available,so we set it as "NA". + - For system generated LOGIN_FAILED logs, the Dbuser value not available,so we set it as "N.A.". + - Large SQL statements are truncated by AWS by default which can cause a GuardUCInvalidRecordException as the event is no longer valid. ## 7. Configuring the AWS MariaDB Guardium Logstash filters in Guardium Data Security Center diff --git a/filter-plugin/logstash-filter-mariadb-aws-guardium/build.gradle b/filter-plugin/logstash-filter-mariadb-aws-guardium/build.gradle index a0530ae6b..a0d29c6e7 100644 --- a/filter-plugin/logstash-filter-mariadb-aws-guardium/build.gradle +++ b/filter-plugin/logstash-filter-mariadb-aws-guardium/build.gradle @@ -2,6 +2,30 @@ import java.nio.file.Files import static java.nio.file.StandardCopyOption.REPLACE_EXISTING apply plugin: 'java' + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + + dependencies { + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' + classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.1.0" + classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' + } +} + +def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" @@ -22,10 +46,10 @@ pluginInfo.pluginName = "awsmariadb_guardium_filter" // must match the @Log // =========================================================================== -sourceCompatibility = 1.8 -targetCompatibility = 1.8 +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 -def jacocoVersion = '0.8.4' +def jacocoVersion = '0.8.11' // minimumCoverage can be set by Travis ENV def minimumCoverageStr = System.getenv("MINIMUM_COVERAGE") ?: "50.0%" if (minimumCoverageStr.endsWith("%")) { @@ -33,28 +57,17 @@ if (minimumCoverageStr.endsWith("%")) { } def minimumCoverage = Float.valueOf(minimumCoverageStr) / 100 -buildscript { - repositories { - maven { - url "https://plugins.gradle.org/m2/" - } - mavenCentral() - jcenter() - } - - dependencies { - classpath 'com.github.jengelman.gradle.plugins:shadow:4.0.4' - classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.0.0-RC2" - classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' - } -} - -def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); -def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) repositories { - mavenCentral() + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } } tasks.register("vendor"){ @@ -65,7 +78,6 @@ tasks.register("vendor"){ File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") projectJarFile.mkdirs() Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) - validatePluginJar(projectJarFile, project.group) } } @@ -74,7 +86,7 @@ apply plugin: "com.github.johnrengelman.shadow" apply plugin: 'jacoco' shadowJar { - classifier = null + archiveClassifier = null } dependencies { @@ -82,6 +94,7 @@ dependencies { implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: versions.dependencies.log4jCore implementation 'org.apache.commons:commons-lang3:' + versions.dependencies.commonsLang implementation 'com.google.code.gson:gson:' + versions.dependencies.gson + implementation group: 'commons-beanutils', name: 'commons-beanutils', version: versions.dependencies.commonsBeanutils implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "build/libs/logstash-core-*.*.*.jar") implementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "common-*.*.*.jar") @@ -111,6 +124,17 @@ tasks.withType(JavaCompile) { options.encoding = 'UTF-8' } +test { + jvmArgs = [ + '--add-opens=java.base/java.lang=ALL-UNNAMED', + '--add-opens=java.base/java.util=ALL-UNNAMED', + '--add-opens=java.base/java.text=ALL-UNNAMED', + '--add-opens=java.base/java.lang.reflect=ALL-UNNAMED', + '--add-opens=java.base/sun.util.resources=ALL-UNNAMED', + '--add-opens=java.base/sun.util.cldr=ALL-UNNAMED' + ] +} + tasks.register("generateRubySupportFiles") { @@ -153,17 +177,16 @@ apply plugin: "org.barfuin.gradle.jacocolog" // ------------------------------------ jacoco { toolVersion = "${jacocoVersion}" - reportsDir = file("$buildDir/reports/jacoco") } jacocoTestReport { // You will see "Report -> file://...." at the end of a JaCoCo build // If no output, run this first: ./gradlew test reports { - html.enabled true - xml.enabled true - csv.enabled true - html.destination file("${buildDir}/reports/jacoco") - csv.destination file("${buildDir}/reports/jacoco/all.csv") + html.required = true + xml.required = true + csv.required = true + html.outputLocation = file("${buildDir}/reports/jacoco") + csv.outputLocation = file("${buildDir}/reports/jacoco/all.csv") } executionData.from fileTree(dir: "${buildDir}/jacoco/", includes: [ '**/*.exec' diff --git a/filter-plugin/logstash-filter-mariadb-aws-guardium/gradle/wrapper/gradle-wrapper.properties b/filter-plugin/logstash-filter-mariadb-aws-guardium/gradle/wrapper/gradle-wrapper.properties index aa991fcea..81aa1c044 100644 --- a/filter-plugin/logstash-filter-mariadb-aws-guardium/gradle/wrapper/gradle-wrapper.properties +++ b/filter-plugin/logstash-filter-mariadb-aws-guardium/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/filter-plugin/logstash-filter-mariadb-aws-guardium/mainREADME.md b/filter-plugin/logstash-filter-mariadb-aws-guardium/mainREADME.md new file mode 100644 index 000000000..93aa184a6 --- /dev/null +++ b/filter-plugin/logstash-filter-mariadb-aws-guardium/mainREADME.md @@ -0,0 +1,10 @@ +# MariaDB Universal Connector + +## Follow this link to set up and use MariaDB Universal Connector over CloudWatch Logstash Plugin + +[MariaDBOverCloudwatch](./README.md) + +## Follow this link to set up and use MariaDB Universal Connector over CloudWatch Connect + +[MariaDBOverConnectCloudwatch](../../docs/KafkaBasedUCs/MariaDBCloudwatchKafkaConnect.md) + diff --git a/filter-plugin/logstash-filter-mariadb-guardium/README.md b/filter-plugin/logstash-filter-mariadb-guardium/README.md index dd755efac..c82e6f865 100644 --- a/filter-plugin/logstash-filter-mariadb-guardium/README.md +++ b/filter-plugin/logstash-filter-mariadb-guardium/README.md @@ -64,7 +64,8 @@ https://www.elastic.co/guide/en/beats/filebeat/current/directory-layout.html • Locate "filebeat.inputs" in the filebeat.yml file, then add the following parameters. ``` filebeat.inputs: - - type: log + - type: filestream + - id: enabled: true paths: -/var/lib/mysql/*.log diff --git a/filter-plugin/logstash-filter-mariadb-guardium/build.gradle b/filter-plugin/logstash-filter-mariadb-guardium/build.gradle index c868c1dfd..82a5ee3b8 100644 --- a/filter-plugin/logstash-filter-mariadb-guardium/build.gradle +++ b/filter-plugin/logstash-filter-mariadb-guardium/build.gradle @@ -2,6 +2,30 @@ import java.nio.file.Files import static java.nio.file.StandardCopyOption.REPLACE_EXISTING apply plugin: 'java' + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + + dependencies { + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' + classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.1.0" + classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' + } +} + +def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" // =========================================================================== // plugin info @@ -19,10 +43,10 @@ pluginInfo.pluginClass = "MariadbGuardiumFilter" pluginInfo.pluginName = "mariadb_guardium_filter" // must match the @LogstashPlugin annotation in the main plugin class // =========================================================================== -sourceCompatibility = 1.8 -targetCompatibility = 1.8 +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 -def jacocoVersion = '0.8.4' +def jacocoVersion = '0.8.11' // minimumCoverage can be set by Travis ENV def minimumCoverageStr = System.getenv("MINIMUM_COVERAGE") ?: "50.0%" if (minimumCoverageStr.endsWith("%")) { @@ -30,27 +54,16 @@ if (minimumCoverageStr.endsWith("%")) { } def minimumCoverage = Float.valueOf(minimumCoverageStr) / 100 -buildscript { - repositories { - maven { - url "https://plugins.gradle.org/m2/" - } - mavenCentral() - jcenter() - } - - dependencies { - classpath 'com.github.jengelman.gradle.plugins:shadow:4.0.4' - classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.0.0-RC2" - classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' - } -} - -def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); -def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) repositories { - mavenCentral() + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } } tasks.register("vendor"){ @@ -61,14 +74,13 @@ tasks.register("vendor"){ File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") projectJarFile.mkdirs() Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) - validatePluginJar(projectJarFile, project.group) } } apply plugin: 'com.github.johnrengelman.shadow' shadowJar { - classifier = null + archiveClassifier = null } dependencies { @@ -76,6 +88,7 @@ dependencies { implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: versions.dependencies.log4jCore implementation 'org.apache.commons:commons-lang3:' + versions.dependencies.commonsLang implementation 'com.google.code.gson:gson:' + versions.dependencies.gson + implementation group: 'commons-beanutils', name: 'commons-beanutils', version: versions.dependencies.commonsBeanutils implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "build/libs/logstash-core-*.*.*.jar") implementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "common-*.*.*.jar") @@ -100,6 +113,17 @@ tasks.withType(JavaCompile) { options.encoding = 'UTF-8' } +test { + jvmArgs = [ + '--add-opens=java.base/java.lang=ALL-UNNAMED', + '--add-opens=java.base/java.util=ALL-UNNAMED', + '--add-opens=java.base/java.text=ALL-UNNAMED', + '--add-opens=java.base/java.lang.reflect=ALL-UNNAMED', + '--add-opens=java.base/sun.util.resources=ALL-UNNAMED', + '--add-opens=java.base/sun.util.cldr=ALL-UNNAMED' + ] +} + tasks.register("generateRubySupportFiles") { @@ -142,17 +166,16 @@ apply plugin: "org.barfuin.gradle.jacocolog" // ------------------------------------ jacoco { toolVersion = "${jacocoVersion}" - reportsDir = file("$buildDir/reports/jacoco") } jacocoTestReport { // You will see "Report -> file://...." at the end of a JaCoCo build // If no output, run this first: ./gradlew test reports { - html.enabled true - xml.enabled true - csv.enabled true - html.destination file("${buildDir}/reports/jacoco") - csv.destination file("${buildDir}/reports/jacoco/all.csv") + html.required = true + xml.required = true + csv.required = true + html.outputLocation = file("${buildDir}/reports/jacoco") + csv.outputLocation = file("${buildDir}/reports/jacoco/all.csv") } executionData.from fileTree(dir: "${buildDir}/jacoco/", includes: [ '**/*.exec' diff --git a/filter-plugin/logstash-filter-mariadb-guardium/gradle/wrapper/gradle-wrapper.properties b/filter-plugin/logstash-filter-mariadb-guardium/gradle/wrapper/gradle-wrapper.properties index aa991fcea..81aa1c044 100644 --- a/filter-plugin/logstash-filter-mariadb-guardium/gradle/wrapper/gradle-wrapper.properties +++ b/filter-plugin/logstash-filter-mariadb-guardium/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/filter-plugin/logstash-filter-milvus-guardium/.gitignore b/filter-plugin/logstash-filter-milvus-guardium/.gitignore new file mode 100644 index 000000000..ce45dbb01 --- /dev/null +++ b/filter-plugin/logstash-filter-milvus-guardium/.gitignore @@ -0,0 +1,24 @@ +build +bin +*.idea +*.iml +lib/ +vendor/ +.bundle/ +build/ +out/ +.idea +.gradle +.vscode +.classpath +.project +*.code-workspace +*.DS_Store +*.iml +*.class +*.ipr +*.iws +*.gemspec +Gemfile* +settings.gradle +gradle.properties diff --git a/filter-plugin/logstash-filter-milvus-guardium/CHANGELOG.md b/filter-plugin/logstash-filter-milvus-guardium/CHANGELOG.md new file mode 100644 index 000000000..143dd1602 --- /dev/null +++ b/filter-plugin/logstash-filter-milvus-guardium/CHANGELOG.md @@ -0,0 +1,10 @@ + +# Changelog +Notable changes will be documented in this file. + + + +## [] + +### Added +- Initial release, in parallel to Guardium . diff --git a/filter-plugin/logstash-filter-milvus-guardium/LICENSE b/filter-plugin/logstash-filter-milvus-guardium/LICENSE new file mode 100644 index 000000000..261eeb9e9 --- /dev/null +++ b/filter-plugin/logstash-filter-milvus-guardium/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/filter-plugin/logstash-filter-milvus-guardium/MilvusOverFilebeatPackage/milvusOverFilebeat.conf b/filter-plugin/logstash-filter-milvus-guardium/MilvusOverFilebeatPackage/milvusOverFilebeat.conf new file mode 100644 index 000000000..3fc43ca89 --- /dev/null +++ b/filter-plugin/logstash-filter-milvus-guardium/MilvusOverFilebeatPackage/milvusOverFilebeat.conf @@ -0,0 +1,12 @@ +input { + beats { + port => 5044 + type => "filebeat" + } +} + +filter { + if [type] == "filebeat" and "milvus" in [tags] { + milvus_guardium_filter{} + } +} diff --git a/filter-plugin/logstash-filter-milvus-guardium/README.md b/filter-plugin/logstash-filter-milvus-guardium/README.md new file mode 100644 index 000000000..ca6edbc6c --- /dev/null +++ b/filter-plugin/logstash-filter-milvus-guardium/README.md @@ -0,0 +1,120 @@ +# Zilliz Milvus - Guardium Logstash filter plug-in + +### Meet Milvus + +* Tested versions: 2.4.4 or later +* Environment: Milvus Standalone (Docker Linux), Milvus Distributed (Milvus Operator) +* Supported inputs: Filebeat (push) +* Supported Guardium versions: Guardium Data Protection 12.2 and later + +This is a [Logstash](https://github.com/elastic/logstash) filter plug-in for the universal connector that is featured in IBM Security Guardium. It parses events and messages from the Zilliz Milvus access log into a Guardium Record. + +The plug-in is free and open-source (Apache 2.0). It can be used as a starting point to develop additional filter plug-ins for Guardium universal connector. + +### Limitations + +1. Milvus access logs do not include the server IP address. +2. Milvus access logs do not specify the source program but do provide the SDK version. + +## Configuring access logs for Milvus + +### Before you begin + +Install Milvus. For more information, see [Milvus](https://milvus.io/docs). + +### Procedure + +1. Configure access logs for **Milvus**. + + In the ``milvus.yaml`` file, find the ``proxy | accessLog`` section and configure the following parameters: + + - Set ``accessLog | enable`` to ``true``. + - In the ``localPath`` parameter, enter the directory where the access log file is located. + - In the ``filename`` parameter, enter the name of your access log file. +

+ ``` + proxy: + accessLog: + enable: true + localPath: /tmp/milvus_access + filename: access.log + ``` + For more information, see [Configure Access logs](https://milvus.io/docs/configure_access_logs.md). + +2. The Milvus filter requires IBM Log Event Extended Format (LEEF) for the access log entry. For more information, see [LEEF overview](https://www.ibm.com/docs/en/dsm?topic=leef-overview). +3. Update the ``formatters`` section in your Milvus configuration file to use LEEF. + + ``` + formatters: + base: + format:"LEEF:1.0|Zilliz|Milvus|1.0|$method_name-$method_status|devTime=$time_now\tdevTimeFormat=yyyy/MM/dd HH:mm:ss.SSS xxx\tuserName=$user_name\tuserAddress=$user_addr\tdatabaseName=$database_name\tcollectionName=$collection_name\tpartitionName=$partition_name\tqueryExpression=$method_expr\terrorCode=$error_code\terrorMessage=$error_msg\ttraceId=$trace_Id\tresponseSize=$response_size\ttimeCost=$time_cost\ttimeStart=$time_start\ttimeEnd=$time_end\tsdkVersion=$sdk_version\tmethodName=$method_name\tmethodStatus=$method_status" + ``` + +## Installing and configuring Filebeat + +Guardium uses the Filebeat input plugin to ingest access logs from Milvus. For more information, see [Filebeat](https://www.elastic.co/docs/reference/beats/filebeat). + + +### Procedure +1. Install Filebeat on your system. For more information, see [Install Filebeat](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-installation-configuration.html#installation). +2. Configure Filebeat to use Logstash for additional data processing by updating the ``filebeat.yml`` configuration file located in the Filebeat installation directory. For more information about locating the installation directory, see [Directory layout](https://www.elastic.co/guide/en/beats/filebeat/current/directory-layout.html). +3. In the `filebeat.yml` file, navigate to the `filebeat.inputs` section and add the following parameters. Make sure to add the ``milvus`` tag to identify the Milvus events from other data. + ``` + filebeat.inputs: + - type: filestream + - id: + enabled: true + paths: + - + fields: + service: milvus + fields_under_root: true + tags: ["milvus"] + ``` + +4. Configure the output section in the ``filebeat.yml`` file by completing the following steps. + + a. Disable Elasticsearch output by commenting it out. + + b. Enable Logstash output by uncommenting the Logstash section. For more information, see [Configure the Logstash output](https://www.elastic.co/guide/en/beats/filebeat/current/logstash-output.html#logstash-output). + + For example: + + ``` + output.logstash: + hosts: [":"] + ``` + + Note: The ``hosts`` specifies the Logstash server and the ``port`` specifies where Logstash is configured to listen for incoming Beats connections. You can set any port number except ``5044``, ``5141``, and ``5000`` as these ports are currently reserved in Guardium v11.3 and v11.4. + +5. Navigate to the ``processors`` section and add the following attribute to get the server's time zone. For more information, see [Add the local time zone](https://www.elastic.co/guide/en/beats/filebeat/current/add-locale.html). + + In the following example, the processor is enabled with the default settings. + ``` + processors: + - add_locale: ~ + ``` + +6. Start FileBeat. For more information, see [Start filebeat](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-installation-configuration.html#start). + +## Configuring Milvus filters in Guardium + +The Guardium universal connector is the Guardium entry point for native access logs. The Guardium universal connector identifies and parses the received events, and converts them to a standard Guardium format. The output of the Guardium universal connector is forwarded to the Guardium sniffer on the collector for policy and auditing enforcements. + +### Before you begin +* Configure the policies you need. For more information, see [Policies](/docs/#policies). +* You must have permissions for the S-Tap Management role. By default, the admin user is assigned the S-Tap Management role. +* Download the [logstash-filter-milvus-guardium](https://github.com/IBM/universal-connectors/releases/download/v1.7.0/logstash-filter-milvus_guardium_filter.zip) plug-in. + +### Procedure +1. On the collector, go to **Setup** > **Tools and Views** > **Configure Universal Connector**. +2. Enable the universal connector if it is disabled. +3. Click **Upload File** and select the offline [logstash-filter-milvus-guardium](https://github.com/IBM/universal-connectors/releases/download/v1.7.0/logstash-filter-milvus_guardium_filter.zip) plug-in. After it is uploaded, click **OK**. +4. Click **Upload File** and select the ``key.json`` file. After it is uploaded, click **OK**. +5. Click the **Plus** sign to open the Connector Configuration dialog box. +6. In the **Connector name** field, enter a name. +7. Update the input section to add the details from the [``milvusOverFilebeat.conf``](milvusOverFilebeat.conf) file's ``input`` section, omitting the keyword ``input{`` at the beginning and its corresponding ``}`` at the end. +8. Update the filter section to add the details from the [``milvusOverFilebeat.conf``](milvusOverFilebeat.conf) file's ``filter`` section, omitting the keyword ``filter{`` at the beginning and its corresponding ``}`` at the end. +9. Make sure that the ``type`` fields in the ``input`` and ``filter`` configuration sections align. This field must be unique for each connector added to the system. +10. Click **Save**. Guardium validates the new connector and displays it in the Configure Universal Connector page. +11. After the offline plug-in is installed and the configuration is uploaded and saved in the Guardium machine, restart the universal connector by using the **Disable/Enable** button. diff --git a/filter-plugin/logstash-filter-milvus-guardium/VERSION b/filter-plugin/logstash-filter-milvus-guardium/VERSION new file mode 100644 index 000000000..9f8e9b69a --- /dev/null +++ b/filter-plugin/logstash-filter-milvus-guardium/VERSION @@ -0,0 +1 @@ +1.0 \ No newline at end of file diff --git a/filter-plugin/logstash-filter-milvus-guardium/build.gradle b/filter-plugin/logstash-filter-milvus-guardium/build.gradle new file mode 100644 index 000000000..a31450464 --- /dev/null +++ b/filter-plugin/logstash-filter-milvus-guardium/build.gradle @@ -0,0 +1,215 @@ +import java.nio.file.Files +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING + +apply plugin: 'java' + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + + dependencies { + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' + classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.1.0" + classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' + } + + ext { + snakeYamlVersion = '2.2' + } +} + +def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + +apply plugin: 'jacoco' +apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" +apply plugin: "com.github.johnrengelman.shadow" +apply plugin: "eclipse" + + +// =========================================================================== +// plugin info +// =========================================================================== +group "com.ibm.guardium.milvus" // must match the package of the main plugin class +version "${file("VERSION").text.trim()}" // read from required VERSION file +description = "Milvus-Guardium filter plugin" +pluginInfo.licenses = ['Apache-2.0'] // list of SPDX license IDs +pluginInfo.longDescription = "This gem is a Logstash Milvus filter plugin required to be installed as part of IBM Security Guardium, Guardium Universal connector configuration. This gem is not a stand-alone program." +pluginInfo.authors = ['IBM', '', ''] +pluginInfo.email = [''] +pluginInfo.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html" +pluginInfo.pluginType = "filter" +pluginInfo.pluginClass = "MilvusGuardiumFilter" +pluginInfo.pluginName = "milvus_guardium_filter" // must match the @LogstashPlugin annotation in the main plugin class +// =========================================================================== + +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 + +def jacocoVersion = '0.8.11' +// minimumCoverage can be set by Travis ENV +def minimumCoverageStr = System.getenv("MINIMUM_COVERAGE") ?: "50.0%" +if (minimumCoverageStr.endsWith("%")) { + minimumCoverageStr = minimumCoverageStr.substring(0, minimumCoverageStr.length() - 1) +} +def minimumCoverage = Float.valueOf(minimumCoverageStr) / 100 + + + +repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } +} + +tasks.register("vendor"){ + dependsOn shadowJar + doLast { + String vendorPathPrefix = "vendor/jar-dependencies" + String projectGroupPath = project.group.replaceAll('\\.', '/') + File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") + projectJarFile.mkdirs() + Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) + } +} + +apply plugin: 'com.github.johnrengelman.shadow' + +shadowJar { + archiveClassifier = null +} + + +dependencies { + implementation 'com.google.code.gson:gson:' + versions.dependencies.gson + implementation 'org.apache.commons:commons-lang3:' + versions.dependencies.commonsLang + implementation 'commons-validator:commons-validator:' + versions.dependencies.commonsValidator + implementation 'commons-beanutils:commons-beanutils:' + versions.dependencies.commonsBeanutils + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.17.2' + + testImplementation group: 'org.mockito', name: 'mockito-all', version: versions.dependencies.mockitoAll + testImplementation 'org.junit.jupiter:junit-jupiter:' + versions.dependencies.junitJupiter + testImplementation 'org.jruby:jruby-complete:' + versions.dependencies.jrubyComplete + implementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "common-*.*.*.jar") + implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "build/libs/logstash-core.jar") + implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: versions.dependencies.log4jCore + implementation group: 'org.json', name: 'json', version: versions.dependencies.json + implementation group: 'org.parboiled', name: 'parboiled-java', version: versions.dependencies.parboiledJava + implementation group: 'org.glassfish', name: 'javax.json', version: versions.dependencies.javaxJson + testImplementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "common-*.*.*.jar") +} + +clean { + delete "${projectDir}/Gemfile" + delete "${projectDir}/" + pluginInfo.pluginFullName() + ".gemspec" + delete "${projectDir}/lib/" + delete "${projectDir}/vendor/" + new FileNameFinder().getFileNames(projectDir.toString(), pluginInfo.pluginFullName() + "-*.*.*.gem").each { filename -> + delete filename + } +} + +tasks.withType(JavaCompile) { + options.encoding = 'UTF-8' +} +test { + useJUnitPlatform() +} +tasks.register("generateRubySupportFiles") { + doLast { + generateRubySupportFilesForPlugin(project.description, project.group, version) + } +} + +tasks.register("removeObsoleteJars") { + doLast { + new FileNameFinder().getFileNames( + projectDir.toString(), + "vendor/**/" + pluginInfo.pluginFullName() + "*.jar", + "vendor/**/" + pluginInfo.pluginFullName() + "-" + version + ".jar").each { f -> + delete f + } + } +} + +tasks.register("gem"){ + dependsOn = [downloadAndInstallJRuby, removeObsoleteJars, vendor, generateRubySupportFiles] + doLast { + buildGem(projectDir, buildDir, pluginInfo.pluginFullName() + ".gemspec") + } +} + +tasks.register("copyDependencyLibs", Copy){ + into "dependenciesLib" + from configurations.compileClasspath + from configurations.runtimeClasspath + from configurations.testCompileClasspath + from configurations.testRuntimeClasspath +} + +apply plugin: 'jacoco' +//apply plugin: 'org.barfuin.gradle.jacocolog' version '2.0.0' +apply plugin: "org.barfuin.gradle.jacocolog" +// ------------------------------------ +// JaCoCo is a code coverage tool +// ------------------------------------ +jacoco { + toolVersion = "${jacocoVersion}" +} +jacocoTestReport { + // You will see "Report -> file://...." at the end of a JaCoCo build + // If no output, run this first: ./gradlew test + reports { + html.required = true + xml.required = true + csv.required = true + html.outputLocation = file("${buildDir}/reports/jacoco") + csv.outputLocation = file("${buildDir}/reports/jacoco/all.csv") + } + executionData.from fileTree(dir: "${buildDir}/jacoco/", includes: [ + '**/*.exec' + ]) + afterEvaluate { + // objective is to test TicketingService class + classDirectories.setFrom(files(classDirectories.files.collect { + fileTree(dir: it, exclude: []) + })) + } + doLast { + println "Report -> file://${buildDir}/reports/jacoco/index.html" + } +} +test.finalizedBy jacocoTestReport +jacocoTestCoverageVerification { + violationRules { + rule { + limit { + minimum = minimumCoverage + } + } + } + executionData.from fileTree(dir: "${buildDir}/jacoco/", includes: [ + '**/*.exec' + ]) + afterEvaluate { + // objective is to test TicketingService class + classDirectories.setFrom(files(classDirectories.files.collect { + fileTree(dir: it, exclude: []) + })) + } +} +project.tasks.check.dependsOn(jacocoTestCoverageVerification, jacocoTestReport) \ No newline at end of file diff --git a/filter-plugin/logstash-filter-milvus-guardium/gradle/wrapper/gradle-wrapper.jar b/filter-plugin/logstash-filter-milvus-guardium/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 000000000..7454180f2 Binary files /dev/null and b/filter-plugin/logstash-filter-milvus-guardium/gradle/wrapper/gradle-wrapper.jar differ diff --git a/filter-plugin/logstash-filter-milvus-guardium/gradle/wrapper/gradle-wrapper.properties b/filter-plugin/logstash-filter-milvus-guardium/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 000000000..81aa1c044 --- /dev/null +++ b/filter-plugin/logstash-filter-milvus-guardium/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,5 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/filter-plugin/logstash-filter-milvus-guardium/gradlew b/filter-plugin/logstash-filter-milvus-guardium/gradlew new file mode 100755 index 000000000..744e882ed --- /dev/null +++ b/filter-plugin/logstash-filter-milvus-guardium/gradlew @@ -0,0 +1,185 @@ +#!/usr/bin/env sh + +# +# Copyright 2015 the original author or authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MSYS* | MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin or MSYS, switch paths to Windows format before running java +if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=`expr $i + 1` + done + case $i in + 0) set -- ;; + 1) set -- "$args0" ;; + 2) set -- "$args0" "$args1" ;; + 3) set -- "$args0" "$args1" "$args2" ;; + 4) set -- "$args0" "$args1" "$args2" "$args3" ;; + 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=`save "$@"` + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +exec "$JAVACMD" "$@" diff --git a/filter-plugin/logstash-filter-milvus-guardium/gradlew.bat b/filter-plugin/logstash-filter-milvus-guardium/gradlew.bat new file mode 100644 index 000000000..107acd32c --- /dev/null +++ b/filter-plugin/logstash-filter-milvus-guardium/gradlew.bat @@ -0,0 +1,89 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto execute + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/filter-plugin/logstash-filter-milvus-guardium/milvusOverFilebeat.conf b/filter-plugin/logstash-filter-milvus-guardium/milvusOverFilebeat.conf new file mode 100644 index 000000000..4dd1e7d71 --- /dev/null +++ b/filter-plugin/logstash-filter-milvus-guardium/milvusOverFilebeat.conf @@ -0,0 +1,21 @@ +input { + beats { + port => 5044 + type => "filebeat" + ssl_enabled => false + ssl_certificate => "${SSL_DIR}/tls.crt" + ssl_key => "${SSL_DIR}/tls.key" + } +} + +filter { + if [type] == "filebeat" and "milvus" in [tags] { + mutate { + add_field => { "server_hostname" => "%{[host][name]}" } + } + + milvus_guardium_filter{} + + mutate { remove_field => ["serverHostname","@version","@timestamp","type","sequence","message","host","tags","input","log","ecs","agent","@metadata","service"] } + } +} \ No newline at end of file diff --git a/filter-plugin/logstash-filter-milvus-guardium/src/main/java/com/ibm/guardium/milvus/Constants.java b/filter-plugin/logstash-filter-milvus-guardium/src/main/java/com/ibm/guardium/milvus/Constants.java new file mode 100644 index 000000000..3cf07cb0d --- /dev/null +++ b/filter-plugin/logstash-filter-milvus-guardium/src/main/java/com/ibm/guardium/milvus/Constants.java @@ -0,0 +1,18 @@ +package com.ibm.guardium.milvus; + +public class Constants { + public static final String DATA_TYPE = "TEXT"; + public static final String LANGUAGE = "Milvus"; + static final String MESSAGE = "message"; + static final String INVALID_MSG = "EVENT_IS_INVALID"; + static final String EMPTY = ""; + static final String SQL_ERROR = "SQL_ERROR"; + static final String LOGIN_FAILED = "LOGIN_FAILED"; + static final String ERROR_MSG = "exception_desc"; + static final String EVENT_ID = "event_id"; + static final String USER_ADDR = "client"; + static final String DB_PROTOCOL = "MILVUS"; + static final String METHOD_STATUS = "method_status"; + static final String SERVER_HOSTNAME = "serverHostname"; + +} diff --git a/filter-plugin/logstash-filter-milvus-guardium/src/main/java/com/ibm/guardium/milvus/MilvusGuardiumFilter.java b/filter-plugin/logstash-filter-milvus-guardium/src/main/java/com/ibm/guardium/milvus/MilvusGuardiumFilter.java new file mode 100644 index 000000000..4d9bfceb2 --- /dev/null +++ b/filter-plugin/logstash-filter-milvus-guardium/src/main/java/com/ibm/guardium/milvus/MilvusGuardiumFilter.java @@ -0,0 +1,114 @@ +/* +Copyright IBM Corp. 2021, 2023 All rights reserved. +SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.guardium.milvus; + +import co.elastic.logstash.api.*; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.ibm.guardium.universalconnector.commons.GuardConstants; +import com.ibm.guardium.universalconnector.commons.custom_parsing.ParserFactory; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; + +import static com.ibm.guardium.milvus.Constants.*; +import static com.ibm.guardium.milvus.Constants.SERVER_HOSTNAME; + +@LogstashPlugin(name = "milvus_guardium_filter") +public class MilvusGuardiumFilter implements Filter { + + private static Logger logger = LogManager.getLogger(MilvusGuardiumFilter.class); + + private String id; + private String filter; + private Parser parser; + public static final PluginConfigSpec SOURCE_CONFIG = PluginConfigSpec.stringSetting("source", "message"); + + public MilvusGuardiumFilter(String id, String filter, Configuration config, Context context) { + this(id, config, context); + this.filter = filter; + } + + public MilvusGuardiumFilter(String id, Configuration config, Context context) { + this.id = id; + this.parser = new Parser(ParserFactory.ParserType.leef); + } + + @Override + public Collection> configSchema() { + return Collections.singletonList(SOURCE_CONFIG); + } + + /** + * Returns the id + * + * @return id + */ + @Override + public String getId() { + return this.id; + } + + /** + * Filters the received events by skipping the invalid ones and normalizing them + * by parsing the provided payloads into Guardium Generic Records. + * + * @param events A list of received events + * @param filterMatchListener The listener for this plugin + * @return A list of normalized events + */ + public Collection filter(Collection events, FilterMatchListener filterMatchListener) { + ArrayList skippedEvents = new ArrayList<>(); + for (Event e : events) { + if (logger.isDebugEnabled()) { + logger.debug("Event Now: {}", e.getData()); + } + if (!(e.getField(MESSAGE) instanceof String) + || (filter != null && !String.valueOf(e.getField(MESSAGE)).contains(filter))) { + e.tag(INVALID_MSG); + skippedEvents.add(e); + } else { + + // Get the message content + String messageContent = e.getField(MESSAGE).toString(); + + // Get the server hostname if available + String serverHostname = ""; + if (e.getField(SERVER_HOSTNAME) != null) { + serverHostname = e.getField(SERVER_HOSTNAME).toString(); + logger.debug("Found serverHostname in event: {}", serverHostname); + } + + // Parse the record + Record record = this.parser.parseRecord(messageContent); + + if(record == null) { + e.tag(INVALID_MSG); + skippedEvents.add(e); + logger.debug("Invalid event skipped: {}", e); + continue; + } + // Set the server hostname in the accessor if available + if (record.getAccessor() != null) { + record.getAccessor().setServerHostName(serverHostname); + logger.debug("Set serverHostName in accessor: {}", serverHostname); + } + + final Gson gson = new GsonBuilder().disableHtmlEscaping().serializeNulls().create(); + e.setField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME, gson.toJson(record)); + filterMatchListener.filterMatched(e); + } + } + events.removeAll(skippedEvents); + return events; + + } + +} \ No newline at end of file diff --git a/filter-plugin/logstash-filter-milvus-guardium/src/main/java/com/ibm/guardium/milvus/Parser.java b/filter-plugin/logstash-filter-milvus-guardium/src/main/java/com/ibm/guardium/milvus/Parser.java new file mode 100644 index 000000000..b7a412fbe --- /dev/null +++ b/filter-plugin/logstash-filter-milvus-guardium/src/main/java/com/ibm/guardium/milvus/Parser.java @@ -0,0 +1,310 @@ +/* +Copyright IBM Corp. 2021, 2023 All rights reserved. +SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.guardium.milvus; + +import com.ibm.guardium.universalconnector.commons.custom_parsing.CustomParser; +import com.ibm.guardium.universalconnector.commons.custom_parsing.ParserFactory; +import com.ibm.guardium.universalconnector.commons.structures.Accessor; +import com.ibm.guardium.universalconnector.commons.structures.Construct; +import com.ibm.guardium.universalconnector.commons.structures.Data; +import com.ibm.guardium.universalconnector.commons.structures.ExceptionRecord; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import com.ibm.guardium.universalconnector.commons.structures.Sentence; +import com.ibm.guardium.universalconnector.commons.structures.SentenceObject; +import com.ibm.guardium.universalconnector.commons.structures.SessionLocator; +import com.ibm.guardium.universalconnector.commons.structures.Time; +import com.ibm.guardium.universalconnector.commons.structures.Record; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.regex.*; + +import static com.ibm.guardium.milvus.Constants.*; +import static com.ibm.guardium.universalconnector.commons.custom_parsing.PropertyConstant.*; + +/** + * Parser Class will perform operation on parsing events and messages from the + * Milvus audit logs into a Guardium record instance Guardium records include + * the accessor, the sessionLocator, data, and exceptions. If there are no + * errors, the data contains details about the query "construct" + * + * @className @Parser + */ +public class Parser extends CustomParser { + private static final Logger logger = LogManager.getLogger(Parser.class); + private static final String TIME_FORMAT = "yyyy/MM/dd HH:mm:ss.SSS xxx"; + private static final String CLIENT_REGEX = "^([a-zA-Z0-9.-]+)-([0-9a-fA-F:.]+):(\\d+)$"; + + private Client client; + + public Parser(ParserFactory.ParserType parserType) { + super(parserType); + } + + @Override + public Record parseRecord(String payload) { + if (!isValid(payload)) + return null; + + client = getClient(getValue(payload, USER_ADDR)); // DB_PROTOCOL ??? + return extractRecord(payload); + } + + @Override + public String getConfigFileContent() { + return "{\n" + + " \"db_name\": \"databaseName\",\n" + + " \"db_user\": \"userName\",\n" + + " \"db_protocol\": \"MILVUS GRPC\",\n" + + " \"app_user_name\": \"userName\",\n" + + " \"client\": \"userAddress\",\n" + + " \"exception_type_id\": \"errorCode\",\n" + + " \"exception_desc\": \"errorMessage\",\n" + + " \"collection_name\": \"collectionName\",\n" + + " \"partition\": \"partitionName\",\n" + + " \"query_expression\": \"queryExpression\",\n" + + " \"trace_id\": \"traceId\",\n" + + " \"response_size\": \"responseSize\",\n" + + " \"time_cost\": \"timeCost\",\n" + + " \"time_start\": \"timeStart\",\n" + + " \"time_end\": \"timeEnd\",\n" + + " \"sdk_version\": \"sdkVersion\",\n" + + " \"timestamp\": \"devTime\",\n" + + " \"method_name\": \"methodName\",\n" + + " \"method_status\": \"methodStatus\",\n" + + " \"server_port\": \"{0}\",\n" + + " \"server_type\": \"{Milvus}\",\n" + + " \"session_id\": \"\",\n" + + " \"event_id\": \"$eventid$\",\n" + + " \"sniffer_parser\": \"FREE_TEXT\"\n" + + "}"; + + } + + @Override + protected String getExceptionTypeId(String payload) { + String value = getValue(payload, EXCEPTION_TYPE_ID); + if (value == null || value.equals("0")) + return DEFAULT_STRING; + + // Check if this is a login failure + String eventId = getValue(payload, EVENT_ID); + String methodStatus = getValue(payload, METHOD_STATUS); + + // Identify login failures by checking multiple indicators + boolean isLoginFailure = (eventId != null && eventId.contains("Connect-GrpcUnauthenticated")) + || (methodStatus != null && methodStatus.equals("GrpcUnauthenticated")); + + return isLoginFailure ? LOGIN_FAILED : SQL_ERROR; + } + + @Override + protected String getExceptionDescription(String payload) { + String value = getValue(payload, ERROR_MSG); + return value != null ? value : DEFAULT_STRING; + } + + @Override + protected String getSqlString(String payload) { + String value = getValue(payload, EVENT_ID); + return value != null ? value : DEFAULT_STRING; // Set the SQL command that caused the exception + } + + @Override + protected Time getTimestamp(String payload) { + String value = getValue(payload, TIMESTAMP); + if (value != null) { + try { + DateTimeFormatter formatter = DateTimeFormatter.ofPattern(TIME_FORMAT); + + OffsetDateTime dateTime = OffsetDateTime.parse(value, formatter); + long millis = dateTime.toInstant().toEpochMilli(); + int minOffsetFromGMT = dateTime.getOffset().getTotalSeconds() / 60; + return new Time(millis, minOffsetFromGMT, 0); + } catch (DateTimeParseException e) { + logger.error("Time {} is invalid.", value, e); + } + } + return new Time(0L, 0, 0); + } + @Override + protected String getDbProtocol(String payload) { + return client.protocol; + } + + @Override + protected Integer getClientPort(String payload) { + return client.port; + } + @Override + protected String getClientIpv6(String payload) { + return client.ip; + } + @Override + protected String getClientIp(String payload) { + return client.ip; + } + + Client getClient(String clientInfo) { + if (clientInfo == null) + return new Client(); + + Pattern pattern = Pattern.compile(CLIENT_REGEX); + Matcher matcher = pattern.matcher(clientInfo); + if (matcher.matches()) { + return new Client(matcher.group(1), matcher.group(2), matcher.group(3)); + } + + return new Client(); + } + + @Override + protected Data getData(String payload, String sqlString) { + Data data = new Data(); + + try { + // Convert parsed data to JSON format with correct separators + sqlString = getMlvsGrpcMessage(payload); + } catch (Exception e) { + logger.error("Error in parsing LEEF log: ",e.getMessage()); + } + + // Set the final MLVS JSON string + data.setOriginalSqlCommand(sqlString); + + return data; + } + + @Override + protected String getLanguage(String payload) { + return LANGUAGE; + } + + @Override + protected String getDataType(String payload) { + return DATA_TYPE; + } + + @Override + protected Record extractRecord(String payload) { + Record record = new Record(); + + record.setSessionId(getSessionId(payload)); + record.setDbName(getDbName(payload)); + record.setAppUserName(getAppUserName(payload)); + String sqlString = getSqlString(payload); + record.setException(getException(payload, sqlString)); + record.setAccessor(getAccessor(payload)); + record.setSessionLocator(getSessionLocator(payload)); + record.setTime(getTimestamp(payload)); + + record.setData(getData(payload, sqlString)); + + return record; + } + + @Override + protected String getServiceName(String payload) { + return getDbName(payload); + } + + private String getMlvsGrpcMessage(String payload) throws Exception { + + StringBuilder sb = new StringBuilder(); + + sb.append("__MLVS { "); + sb.append("\"identifier\":\"").append(getTraceId(payload)).append("\",\n"); + sb.append("\"collection_name\":=\'").append(getCollectionName(payload)).append("\',\n");//object + sb.append("\"action\":\"").append(getMethodName(payload)).append("\",\n"); //verb + sb.append("\"query_expression\":\"").append(getQueryExpression(payload)).append("\",\n"); + sb.append("\"dev_time\":\"").append(this.getTimestamp(payload).toString()).append("\",\n"); + sb.append("\"partition_name\":\"").append(getPartitionName(payload)).append("\",\n"); + sb.append("\"trace_id\":\"").append(getTraceId(payload)).append("\",\n"); + sb.append("\"responseSize\":\"").append(getResponseSize(payload)).append("\",\n"); + sb.append("\"time_cost\":\"").append(getTimeCost(payload)).append("\",\n"); + sb.append("\"time_start\":\"").append(getTimeStart(payload)).append("\",\n"); + sb.append("\"time_end\":\"").append(getTimeEnd(payload)).append("\",\n"); + sb.append("\"sdk_version\":\"").append(getSDKVersion(payload)).append("\"\n"); + sb.append("}"); + + return sb.toString(); + } + + private String getSDKVersion(String payload) { + String value = this.getValue(payload,"sdk_version"); + return value != null ? value : ""; + } + + private String getTimeEnd(String payload) { + String value = this.getValue(payload,"time_end"); + return value != null ? value : ""; + } + + private String getTimeStart(String payload) { + String value = this.getValue(payload,"time_start"); + return value != null ? value : ""; + } + + private String getTimeCost(String payload) { + String value = this.getValue(payload,"time_cost"); + return value != null ? value : ""; + } + + private String getResponseSize(String payload) { + String value = this.getValue(payload,"response_size"); + return value != null ? value : ""; + } + + private String getPartitionName(String payload) { + String value = this.getValue(payload,"partition_name"); + return value != null ? value : ""; + } + + private String getQueryExpression(String payload) { + String value = this.getValue(payload,"method_expr"); + return value != null ? value : ""; + } + + private String getMethodName(String payload) { + String value = this.getValue(payload, "method_name"); + return value != null ? value : ""; + } + + private String getTraceId(String payload) { + String value = this.getValue(payload, "trace_id"); + return value != null ? value : ""; + } + + private String getCollectionName(String payload) { + String value = this.getValue(payload, "collection_name"); + return value != null ? value : ""; + } + + + static class Client { + private String protocol = Constants.DB_PROTOCOL; + private String ip = EMPTY; + private int port = 0; + + Client(String protocol, String ip, String port) { + // Always use the constant instead of the extracted protocol + this.protocol = Constants.DB_PROTOCOL; + this.ip = ip; + try { + this.port = Integer.parseInt(port); + } catch (NumberFormatException e) { + this.port = -1; + } + } + + Client() { + } + } +} \ No newline at end of file diff --git a/filter-plugin/logstash-filter-milvus-guardium/src/test/java/com/ibm/guardium/milvus/MilvusGuardiumFilterTest.java b/filter-plugin/logstash-filter-milvus-guardium/src/test/java/com/ibm/guardium/milvus/MilvusGuardiumFilterTest.java new file mode 100644 index 000000000..a1185cbda --- /dev/null +++ b/filter-plugin/logstash-filter-milvus-guardium/src/test/java/com/ibm/guardium/milvus/MilvusGuardiumFilterTest.java @@ -0,0 +1,72 @@ +package com.ibm.guardium.milvus; + +import co.elastic.logstash.api.Event; +import co.elastic.logstash.api.FilterMatchListener; +import co.elastic.logstash.api.Configuration; +import co.elastic.logstash.api.Context; +import org.logstash.plugins.ConfigurationImpl; +import org.logstash.plugins.ContextImpl; +import org.junit.jupiter.api.Test; + +import java.util.Collection; +import java.util.Collections; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.jupiter.api.Assertions.assertNotNull; + +class MilvusGuardiumFilterTest { + + FilterMatchListener matchListener = new TestMatchListener(); + + String id = "1"; + Configuration config = new ConfigurationImpl(Collections.singletonMap("source", "")); + Context context = new ContextImpl(null, null); + MilvusGuardiumFilter filter = new MilvusGuardiumFilter(id, config, context); + + @Test + void test() { + String payload = "LEEF:1.0|Zilliz|Milvus|1.0|DropAlias-GrpcPermissionDenied|devTime=2025/02/06 21:38:53.557 +00:00\tdevTimeFormat=yyyy/MM/dd HH:mm:ss.SSS xxx\tuserName=zilliz\tuserAddress=tcp-172.17.0.1:47286\tdatabaseName=default\tqueryExpression=Unknown\terrorCode=65535\terrorMessage=rpc error: code = PermissionDenied desc = PrivilegeDropAlias: permission deny to zilliz in the `default` database"; + Event event = new org.logstash.Event(); + event.setField("message", payload); + Collection actualResponse = filter.filter(Collections.singletonList(event), matchListener); + + assertNotNull(actualResponse.toArray(new Event[0])[0].getField("GuardRecord")); + } + + @Test + void test_filebeat(){ + String payload = "{ \"@timestamp\" : \"2025-03-24T15:23:41.631Z\", " + + "\"@version\" : \"1\", " + + "\"type\" : \"filebeat\", " + + "\"service\" : \"milvus\", " + + "\"host\" : \"{name=lima-rancher-desktop}\", " + + "\"log\" : \"{offset=28947, file={path=/tmp/milvus_access/access.log}}\", " + + "\"GuardRecord\" : \"{\\\"connectorName\\\":\\\"filebeat\\\",\\\"connectorId\\\":\\\"36\\\",\\\"sessionId\\\":\\\"\\\",\\\"dbName\\\":\\\"Unknown\\\",\\\"appUserName\\\":\\\"wronguser\\\",\\\"time\\\":{\\\"timestamp\\\":1742829821046,\\\"minOffsetFromGMT\\\":0,\\\"minDst\\\":0},\\\"sessionLocator\\\":{\\\"clientIp\\\":\\\"0.0.0.0\\\",\\\"clientPort\\\":0,\\\"serverIp\\\":\\\"0.0.0.0\\\",\\\"serverPort\\\":0,\\\"isIpv6\\\":false,\\\"clientIpv6\\\":\\\"0000:0000:0000:0000:0000:ffff:0000:0000\\\",\\\"serverIpv6\\\":\\\"0000:0000:0000:0000:0000:ffff:0000:0000\\\"},\\\"accessor\\\":{\\\"dbUser\\\":\\\"wronguser\\\",\\\"serverType\\\":\\\"Milvus\\\",\\\"serverOs\\\":\\\"\\\",\\\"clientOs\\\":\\\"\\\",\\\"clientHostName\\\":\\\"\\\",\\\"serverHostName\\\":\\\"\\\",\\\"commProtocol\\\":\\\"\\\",\\\"dbProtocol\\\":\\\"\\\",\\\"dbProtocolVersion\\\":\\\"\\\",\\\"osUser\\\":\\\"\\\",\\\"sourceProgram\\\":\\\"\\\",\\\"client_mac\\\":\\\"\\\",\\\"serverDescription\\\":\\\"\\\",\\\"serviceName\\\":\\\"\\\",\\\"language\\\":\\\"Milvus\\\",\\\"dataType\\\":\\\"TEXT\\\"},\\\"data\\\":null,\\\"exception\\\":{\\\"exceptionTypeId\\\":\\\"SQL_ERROR\\\",\\\"description\\\":\\\"rpc error: code = Unauthenticated desc = auth check failure, please check username and password are correct\\\",\\\"sqlString\\\":\\\"Connect-GrpcUnauthenticated\\\"},\\\"recordsAffected\\\":null}\", " + + "\"tags\" : \"ConvertedList{delegate=[milvus, beats_input_codec_plain_applied]}\", " + + "\"message\" : \"LEEF:1.0|Zilliz|Milvus|1.0|Connect-GrpcUnauthenticated|devTime=2025/03/24 15:23:41.046 +00:00 devTimeFormat=yyyy/MM/dd HH:mm:ss.SSS xxx userName=wronguser userAddress=tcp-[::1]:55046 databaseName=Unknown queryExpression=Unknown errorCode=65535 errorMessage=rpc error: code = Unauthenticated desc = auth check failure, please check username and password are correct\", " + + "\"ecs\" : \"{version=8.0.0}\", " + + "\"agent\" : \"{id=ff90b92a-0855-4352-a153-473b387006b1, type=filebeat, ephemeral_id=dc200cee-a9ce-43e7-b263-57236012a14a, name=lima-rancher-desktop, version=8.16.1}\", " + + "\"input\" : \"{type=log}\" }"; + + Event event = new org.logstash.Event(); + event.setField("message", payload); + Collection actualResponse = filter.filter(Collections.singletonList(event), matchListener); + + assertNotNull(actualResponse.toArray(new Event[0])[0].getField("GuardRecord")); + + } + + class TestMatchListener implements FilterMatchListener { + private AtomicInteger matchCount = new AtomicInteger(0); + + public int getMatchCount() { + return matchCount.get(); + } + + @Override + public void filterMatched(co.elastic.logstash.api.Event arg0) { + matchCount.incrementAndGet(); + + } + } +} diff --git a/filter-plugin/logstash-filter-milvus-guardium/src/test/java/com/ibm/guardium/milvus/ParserTest.java b/filter-plugin/logstash-filter-milvus-guardium/src/test/java/com/ibm/guardium/milvus/ParserTest.java new file mode 100644 index 000000000..e43a9e1c3 --- /dev/null +++ b/filter-plugin/logstash-filter-milvus-guardium/src/test/java/com/ibm/guardium/milvus/ParserTest.java @@ -0,0 +1,158 @@ +package com.ibm.guardium.milvus; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.ibm.guardium.universalconnector.commons.custom_parsing.ParserFactory; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import org.junit.jupiter.api.Test; + +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +class ParserTest { + Parser parser = new Parser(ParserFactory.ParserType.leef); + + @Test + void test1() { + String payload = "LEEF:1.0|Zilliz|Milvus|1.0|Connect-Successful|devTime=2025/04/02 23:14:21.017 +00:00\tdevTimeFormat=yyyy/MM/dd HH:mm:ss.SSS xxx\tuserName=Unknown\tuserAddress=tcp-172.17.0.1:35050\tdatabaseName=Unknown\tcollectionName=Unknown\tpartitionName=Unknown\tqueryExpression=Unknown\terrorCode=0\terrorMessage=\ttraceId=cf03d862397e1fc9339f717865ae31f0\tresponseSize=105\ttimeCost=2.737959ms\ttimeStart=2025/04/02 23:14:21.014 +00:00\ttimeEnd=2025/04/02 23:14:21.017 +00:00\tsdkVersion=Python-2.4.3\tmethodName=Connect\tmethodStatus=Successful"; + Record record = parser.parseRecord(payload); + assertNotNull(record); + + assertEquals("Unknown", record.getDbName()); + assertEquals(35050, record.getSessionLocator().getClientPort()); + assertEquals("172.17.0.1", record.getSessionLocator().getClientIp()); + assertEquals("MILVUS", record.getAccessor().getDbProtocol()); + assertEquals("Unknown", record.getAccessor().getDbUser()); + assertEquals("Milvus", record.getAccessor().getServerType()); + assertEquals("Time{timstamp=1743635661017, minOffsetFromGMT=0, minDst=0}", record.getTime().toString()); + try { + String expected = new String(Files.readAllBytes(Paths.get("src/test/resources/milvusGRPCMessage_test1.txt"))); + assertEquals(expected, record.getData().getOriginalSqlCommand()); + } catch (IOException e) { + throw new RuntimeException(e); + } catch (Exception e) { + throw new RuntimeException(e); + } + assertNull(record.getException()); + } + + @Test + void test2() { + String payload = "LEEF:1.0|Zilliz|Milvus|1.0|Connect-GrpcUnauthenticated|devTime=2025/02/06 21:20:05.353 +00:00\tdevTimeFormat=yyyy/MM/dd HH:mm:ss.SSS xxx\tuserName=Unknown\tuserAddress=tcp-172.17.0.1:46936\tdatabaseName=TestDB\tqueryExpression=Unknown\terrorCode=65535\terrorMessage=rpc error: code = Unauthenticated desc = auth check failure, please check api key is correct"; + Record record = parser.parseRecord(payload); + assertNotNull(record); + + assertEquals("TestDB", record.getDbName()); + assertEquals(46936, record.getSessionLocator().getClientPort()); + assertEquals("172.17.0.1", record.getSessionLocator().getClientIp()); + assertEquals("MILVUS", record.getAccessor().getDbProtocol()); + assertEquals("Unknown", record.getAccessor().getDbUser()); + assertEquals("Milvus", record.getAccessor().getServerType()); + assertEquals("Time{timstamp=1738876805353, minOffsetFromGMT=0, minDst=0}", record.getTime().toString()); + assertEquals("LOGIN_FAILED", record.getException().getExceptionTypeId()); + try { + String expected = new String(Files.readAllBytes(Paths.get("src/test/resources/milvusGRPCMessage_test2.txt"))); + assertEquals(expected, record.getData().getOriginalSqlCommand()); + } catch (IOException e) { + throw new RuntimeException(e); + } catch (Exception e) { + throw new RuntimeException(e); + } + assertEquals("rpc error: code = Unauthenticated desc = auth check failure, please check api key is correct", + record.getException().getDescription()); + } + + @Test + void test3() { + String payload = "LEEF:1.0|Zilliz|Milvus|1.0|DropAlias-GrpcPermissionDenied|devTime=2025/02/06 21:38:53.557 +00:00\tdevTimeFormat=yyyy/MM/dd HH:mm:ss.SSS xxx\tuserName=zilliz\tuserAddress=tcp-172.17.0.1:47286\tdatabaseName=default\tqueryExpression=Unknown\terrorCode=65535\terrorMessage=rpc error: code = PermissionDenied desc = PrivilegeDropAlias: permission deny to zilliz in the `default` database"; + Record record = parser.parseRecord(payload); + assertNotNull(record); + + assertEquals("default", record.getDbName()); + assertEquals(47286, record.getSessionLocator().getClientPort()); + assertEquals("172.17.0.1", record.getSessionLocator().getClientIp()); + assertEquals("MILVUS", record.getAccessor().getDbProtocol()); + assertEquals("zilliz", record.getAccessor().getDbUser()); + assertEquals("Milvus", record.getAccessor().getServerType()); + assertEquals("Time{timstamp=1738877933557, minOffsetFromGMT=0, minDst=0}", record.getTime().toString()); + assertEquals("SQL_ERROR", record.getException().getExceptionTypeId()); + assertEquals( + "rpc error: code = PermissionDenied desc = PrivilegeDropAlias: permission deny to zilliz in the `default` database", + record.getException().getDescription()); + } + + @Test + void testMoreLogs() { + String filePath = "src/test/resources/logs.txt"; + try (BufferedReader reader = new BufferedReader(new FileReader(filePath))) { + String payload; + while ((payload = reader.readLine()) != null) { + Record record = parser.parseRecord(payload); + + assertNotEquals("", record.getDbName()); + assertNotEquals(-1, record.getSessionLocator().getClientPort()); + assertNotEquals("0.0.0.0", record.getSessionLocator().getClientIp()); + assertNotEquals("", record.getAccessor().getDbProtocol()); + assertNotEquals("", record.getAccessor().getDbUser()); + assertEquals("Milvus", record.getAccessor().getServerType()); + assertNotEquals("", record.getTime().toString()); + if (!payload.contains("errorCode=0")) { + // Check that exception type is either SQL_ERROR or LOGIN_FAILED + String exceptionTypeId = record.getException().getExceptionTypeId(); + assertTrue(exceptionTypeId.equals("SQL_ERROR") || exceptionTypeId.equals("LOGIN_FAILED"), + "Exception type should be either SQL_ERROR or LOGIN_FAILED, but was: " + exceptionTypeId); + + // For login failures, verify it's correctly classified + if (payload.contains("Connect-GrpcUnauthenticated") && + payload.contains("auth check failure")) { + assertEquals("LOGIN_FAILED", exceptionTypeId); + } + + assertNotEquals("", record.getException().getSqlString()); + assertNotEquals("", record.getException().getDescription()); + } else { + assertNull(record.getException()); + assertNotEquals("", record.getData().getOriginalSqlCommand()); + } + } + } catch (IOException e) { + System.err.println("Error reading file: " + e.getMessage()); + } + } + + @Test + void testLoginFailure() { + // Test case with api key error + String payload1 = "LEEF:1.0|Zilliz|Milvus|1.0|Connect-GrpcUnauthenticated|devTime=2025/09/16 17:04:39.489 +00:00\tdevTimeFormat=yyyy/MM/dd HH:mm:ss.SSS xxx\tuserName=Unknown\tuserAddress=tcp-127.0.0.1:47050\tdatabaseName=Unknown\tcollectionName=Unknown\tpartitionName=Unknown\tqueryExpression=Unknown\terrorCode=65535\terrorMessage=rpc error: code = Unauthenticated desc = auth check failure, please check api key is correct\ttraceId=72267fad5d0a8f303744ae9949f8763f\tresponseSize=Unknown\ttimeCost=28.475187ms\ttimeStart=2025/09/16 17:04:39.460 +00:00\ttimeEnd=2025/09/16 17:04:39.489 +00:00\tsdkVersion=Python-2.4.3\tmethodName=Connect\tmethodStatus=GrpcUnauthenticated"; + Record record1 = parser.parseRecord(payload1); + assertNotNull(record1); + assertEquals("LOGIN_FAILED", record1.getException().getExceptionTypeId()); + + // Test case with username/password error + String payload2 = "LEEF:1.0|Zilliz|Milvus|1.0|Connect-GrpcUnauthenticated|devTime=2025/09/16 17:08:12.241 +00:00\tdevTimeFormat=yyyy/MM/dd HH:mm:ss.SSS xxx\tuserName=k8s\tuserAddress=tcp-127.0.0.1:44614\tdatabaseName=Unknown\tcollectionName=Unknown\tpartitionName=Unknown\tqueryExpression=Unknown\terrorCode=65535\terrorMessage=rpc error: code = Unauthenticated desc = auth check failure, please check username and password are correct\ttraceId=ec1684550ed89dbf55ae62364bf39304\tresponseSize=Unknown\ttimeCost=308.881µs\ttimeStart=2025/09/16 17:08:12.241 +00:00\ttimeEnd=2025/09/16 17:08:12.241 +00:00\tsdkVersion=Python-2.4.3\tmethodName=Connect\tmethodStatus=GrpcUnauthenticated"; + Record record2 = parser.parseRecord(payload2); + assertNotNull(record2); + assertEquals("LOGIN_FAILED", record2.getException().getExceptionTypeId()); + + // Test case with successful login (should not be classified as an error) + String payload3 = "LEEF:1.0|Zilliz|Milvus|1.0|Connect-Successful|devTime=2025/09/16 17:07:02.880 +00:00\tdevTimeFormat=yyyy/MM/dd HH:mm:ss.SSS xxx\tuserName=k8s\tuserAddress=tcp-127.0.0.1:44614\tdatabaseName=Unknown\tcollectionName=Unknown\tpartitionName=Unknown\tqueryExpression=Unknown\terrorCode=0\terrorMessage=\ttraceId=88fdf6f03b4427a1db1aa703c463cf65\tresponseSize=107\ttimeCost=5.415566ms\ttimeStart=2025/09/16 17:07:02.873 +00:00\ttimeEnd=2025/09/16 17:07:02.879 +00:00\tsdkVersion=Python-2.4.3\tmethodName=Connect\tmethodStatus=Successful"; + Record record3 = parser.parseRecord(payload3); + assertNotNull(record3); + assertNull(record3.getException()); + } + + private void assertJsonEquals(String expectedJson, String actualJson) throws Exception { + ObjectMapper objectMapper = new ObjectMapper(); + Map expectedMap = objectMapper.readValue(expectedJson, Map.class); + Map actualMap = objectMapper.readValue(actualJson, Map.class); + + assertEquals(expectedMap, actualMap, "JSON content does not match!"); + } +} + +// Made with Bob diff --git a/filter-plugin/logstash-filter-milvus-guardium/src/test/resources/logs.txt b/filter-plugin/logstash-filter-milvus-guardium/src/test/resources/logs.txt new file mode 100644 index 000000000..d6e5f17dd --- /dev/null +++ b/filter-plugin/logstash-filter-milvus-guardium/src/test/resources/logs.txt @@ -0,0 +1,16 @@ +LEEF:1.0|Zilliz|Milvus|1.0|Connect-GrpcUnauthenticated|devTime=2025/02/06 21:51:23.769 +00:00 devTimeFormat=yyyy/MM/dd HH:mm:ss.SSS xxx userName=Unknown userAddress=tcp-172.17.0.1:50768 databaseName=Unknown queryExpression=Unknown errorCode=65535 errorMessage=rpc error: code = Unauthenticated desc = auth check failure, please check api key is correct +LEEF:1.0|Zilliz|Milvus|1.0|Connect-Successful|devTime=2025/02/06 21:51:28.189 +00:00 devTimeFormat=yyyy/MM/dd HH:mm:ss.SSS xxx userName=zilliz userAddress=tcp-172.17.0.1:50780 databaseName=Unknown queryExpression=Unknown errorCode=0 errorMessage= +LEEF:1.0|Zilliz|Milvus|1.0|DropAlias-GrpcPermissionDenied|devTime=2025/02/06 21:51:32.070 +00:00 devTimeFormat=yyyy/MM/dd HH:mm:ss.SSS xxx userName=zilliz userAddress=tcp-172.17.0.1:50780 databaseName=default queryExpression=Unknown errorCode=65535 errorMessage=rpc error: code = PermissionDenied desc = PrivilegeDropAlias: permission deny to zilliz in the `default` database +LEEF:1.0|Zilliz|Milvus|1.0|DescribeCollection-Successful|devTime=2025/02/06 22:00:38.639 +00:00 devTimeFormat=yyyy/MM/dd HH:mm:ss.SSS xxx userName=zilliz userAddress=tcp-172.17.0.1:50780 databaseName=default queryExpression=Unknown errorCode=0 errorMessage= +LEEF:1.0|Zilliz|Milvus|1.0|DescribeCollection-Successful|devTime=2025/02/06 22:00:38.643 +00:00 devTimeFormat=yyyy/MM/dd HH:mm:ss.SSS xxx userName=zilliz userAddress=tcp-172.17.0.1:50780 databaseName=default queryExpression=Unknown errorCode=0 errorMessage= +LEEF:1.0|Zilliz|Milvus|1.0|ShowPartitions-GrpcPermissionDenied|devTime=2025/02/06 22:00:38.645 +00:00 devTimeFormat=yyyy/MM/dd HH:mm:ss.SSS xxx userName=zilliz userAddress=tcp-172.17.0.1:50780 databaseName=default queryExpression=Unknown errorCode=65535 errorMessage=rpc error: code = PermissionDenied desc = PrivilegeShowPartitions: permission deny to zilliz in the `default` database +LEEF:1.0|Zilliz|Milvus|1.0|Connect-Successful|devTime=2025/02/06 22:00:48.060 +00:00 devTimeFormat=yyyy/MM/dd HH:mm:ss.SSS xxx userName=zilliz userAddress=tcp-172.17.0.1:50780 databaseName=Unknown queryExpression=Unknown errorCode=0 errorMessage= +LEEF:1.0|Zilliz|Milvus|1.0|CreateDatabase-GrpcPermissionDenied|devTime=2025/02/06 22:25:54.968 +00:00 devTimeFormat=yyyy/MM/dd HH:mm:ss.SSS xxx userName=zilliz userAddress=tcp-172.17.0.1:50780 databaseName=nexus_test_db queryExpression=Unknown errorCode=65535 errorMessage=rpc error: code = PermissionDenied desc = PrivilegeCreateDatabase: permission deny to zilliz in the `default` database +LEEF:1.0|Zilliz|Milvus|1.0|ListCredUsers-GrpcPermissionDenied|devTime=2025/02/06 22:26:59.684 +00:00 devTimeFormat=yyyy/MM/dd HH:mm:ss.SSS xxx userName=zilliz userAddress=tcp-172.17.0.1:50780 databaseName=Unknown queryExpression=Unknown errorCode=65535 errorMessage=rpc error: code = PermissionDenied desc = PrivilegeSelectOwnership: permission deny to zilliz in the `default` database +LEEF:1.0|Zilliz|Milvus|1.0|Connect-GrpcUnauthenticated|devTime=2025/02/06 22:39:53.361 +00:00 devTimeFormat=yyyy/MM/dd HH:mm:ss.SSS xxx userName=Unknown userAddress=tcp-172.17.0.1:50780 databaseName=Unknown queryExpression=Unknown errorCode=65535 errorMessage=rpc error: code = Unauthenticated desc = auth check failure, please check api key is correct +LEEF:1.0|Zilliz|Milvus|1.0|Connect-Successful|devTime=2025/02/06 22:42:42.197 +00:00 devTimeFormat=yyyy/MM/dd HH:mm:ss.SSS xxx userName=zilliz userAddress=tcp-172.17.0.1:50780 databaseName=Unknown queryExpression=Unknown errorCode=0 errorMessage= +LEEF:1.0|Zilliz|Milvus|1.0|ListCredUsers-GrpcPermissionDenied|devTime=2025/02/06 22:42:46.117 +00:00 devTimeFormat=yyyy/MM/dd HH:mm:ss.SSS xxx userName=zilliz userAddress=tcp-172.17.0.1:50780 databaseName=Unknown queryExpression=Unknown errorCode=65535 errorMessage=rpc error: code = PermissionDenied desc = PrivilegeSelectOwnership: permission deny to zilliz in the `default` database +LEEF:1.0|Zilliz|Milvus|1.0|Connect-Successful|devTime=2025/02/06 23:02:32.507 +00:00 devTimeFormat=yyyy/MM/dd HH:mm:ss.SSS xxx userName=zilliz userAddress=tcp-172.17.0.1:50780 databaseName=Unknown queryExpression=Unknown errorCode=0 errorMessage= +LEEF:1.0|Zilliz|Milvus|1.0|ListDatabases-Successful|devTime=2025/02/06 22:47:32.540 +00:00 devTimeFormat=yyyy/MM/dd HH:mm:ss.SSS xxx userName=zilliz userAddress=tcp-172.17.0.1:50780 databaseName=Unknown queryExpression=Unknown errorCode=0 errorMessage= +LEEF:1.0|Zilliz|Milvus|1.0|DescribeCollection-Successful|devTime=2025/02/06 22:48:13.190 +00:00 devTimeFormat=yyyy/MM/dd HH:mm:ss.SSS xxx userName=zilliz userAddress=tcp-172.17.0.1:50780 databaseName=default queryExpression=Unknown errorCode=0 errorMessage= +LEEF:1.0|Zilliz|Milvus|1.0|ShowPartitions-GrpcPermissionDenied|devTime=2025/02/06 22:48:13.198 +00:00 devTimeFormat=yyyy/MM/dd HH:mm:ss.SSS xxx userName=zilliz userAddress=tcp-172.17.0.1:50780 databaseName=default queryExpression=Unknown errorCode=65535 errorMessage=rpc error: code = PermissionDenied desc = PrivilegeShowPartitions: permission deny to zilliz in the `default` database diff --git a/filter-plugin/logstash-filter-milvus-guardium/src/test/resources/milvusGRPCMessage_test1.txt b/filter-plugin/logstash-filter-milvus-guardium/src/test/resources/milvusGRPCMessage_test1.txt new file mode 100644 index 000000000..74da08747 --- /dev/null +++ b/filter-plugin/logstash-filter-milvus-guardium/src/test/resources/milvusGRPCMessage_test1.txt @@ -0,0 +1,13 @@ +__MLVS { "identifier":"cf03d862397e1fc9339f717865ae31f0", +"collection_name":='Unknown', +"action":"Connect", +"query_expression":"", +"dev_time":"Time{timstamp=1743635661017, minOffsetFromGMT=0, minDst=0}", +"partition_name":"", +"trace_id":"cf03d862397e1fc9339f717865ae31f0", +"responseSize":"105", +"time_cost":"2.737959ms", +"time_start":"2025/04/02 23:14:21.014 +00:00", +"time_end":"2025/04/02 23:14:21.017 +00:00", +"sdk_version":"Python-2.4.3" +} \ No newline at end of file diff --git a/filter-plugin/logstash-filter-milvus-guardium/src/test/resources/milvusGRPCMessage_test2.txt b/filter-plugin/logstash-filter-milvus-guardium/src/test/resources/milvusGRPCMessage_test2.txt new file mode 100644 index 000000000..ccab02d0b --- /dev/null +++ b/filter-plugin/logstash-filter-milvus-guardium/src/test/resources/milvusGRPCMessage_test2.txt @@ -0,0 +1,13 @@ +__MLVS { "identifier":"", +"collection_name":='', +"action":"", +"query_expression":"", +"dev_time":"Time{timstamp=1738876805353, minOffsetFromGMT=0, minDst=0}", +"partition_name":"", +"trace_id":"", +"responseSize":"", +"time_cost":"", +"time_start":"", +"time_end":"", +"sdk_version":"" +} \ No newline at end of file diff --git a/filter-plugin/logstash-filter-mongodb-guardium/CHANGELOG.md b/filter-plugin/logstash-filter-mongodb-guardium/CHANGELOG.md index f8cb7d383..c1c6b51fb 100644 --- a/filter-plugin/logstash-filter-mongodb-guardium/CHANGELOG.md +++ b/filter-plugin/logstash-filter-mongodb-guardium/CHANGELOG.md @@ -3,6 +3,10 @@ Notable changes will be documented in this file. ## [Unreleased] +## [0.6.13] +### Changed +- GRD-114688: Guardium Implementation for Mongo Atlas. Modify MongoDB Atlas config to include API base URL. + ## [0.6.12] ### Changed - Added support for IBM Cloud Logs. diff --git a/filter-plugin/logstash-filter-mongodb-guardium/IBMCloudMongoDB_README.md b/filter-plugin/logstash-filter-mongodb-guardium/IBMCloudMongoDB_README.md index 2602f2609..038ff1a0f 100644 --- a/filter-plugin/logstash-filter-mongodb-guardium/IBMCloudMongoDB_README.md +++ b/filter-plugin/logstash-filter-mongodb-guardium/IBMCloudMongoDB_README.md @@ -195,7 +195,7 @@ Verify [here](https://ondeck.console.cloud.ibm.com/docs/cloud-logs?topic=cloud-l - Verify all of the information and click ```Create stream```. ## 4. Limitations -1. The analysis is based on IBM Cloud Database for MongoDB 4.4. +1. The analysis is based on IBM Cloud Database for MongoDB 7.0. 2. Logs for SQL errors do not get generated from the data source. 3. IBM Cloud Databases for MongoDB only supports 22 events. See [here](https://cloud.ibm.com/docs/databases-for-mongodb?topic=databases-for-mongodb-auditlogging) for more information. 4. In this example, we used both CLI and UI queries to run the analysis. @@ -211,7 +211,7 @@ Verify [here](https://ondeck.console.cloud.ibm.com/docs/cloud-logs?topic=cloud-l 6. IBM Cloud Platform doesn't retain the source IP addresses of the connection. Instead, an internal IP addresses (local and remote) are shown in audit logs for each connection. 7. The following important fields cannot be mapped with MongoDB logs: - Client HostName -9. For admin user, Failed Login is not supported. +8. For admin DB, Failed Login is not supported for admin user. ## 5. Configuring the IBM Cloud MongoDB filter in Guardium The Guardium universal connector is the Guardium entry point for native audit logs. The Guardium universal connector identifies and parses the received events, and converts them to a standard Guardium format. The output of the Guardium universal connector is forwarded to the Guardium sniffer on the collector, for policy and auditing enforcements. Configure Guardium to read the native audit logs by customizing the MongoDB template. diff --git a/filter-plugin/logstash-filter-mongodb-guardium/MongoDBOverSyslogPackage/MongoDB/filter.conf b/filter-plugin/logstash-filter-mongodb-guardium/MongoDBOverSyslogPackage/MongoDB/filter.conf index d5926a4b0..be40a2b1b 100644 --- a/filter-plugin/logstash-filter-mongodb-guardium/MongoDBOverSyslogPackage/MongoDB/filter.conf +++ b/filter-plugin/logstash-filter-mongodb-guardium/MongoDBOverSyslogPackage/MongoDB/filter.conf @@ -6,10 +6,11 @@ if [type] == "syslog-mongodb" { } date { - match => [ "timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ] + match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ] } mutate { rename => { "host" => "server_ip" } } + mutate { gsub => ["message", "mongod\[\d+\]:", "mongod:"] } # send to filter mongodb_guardium_filter {} diff --git a/filter-plugin/logstash-filter-mongodb-guardium/MongoDBOverSyslogPackage/mongodbSyslog.conf b/filter-plugin/logstash-filter-mongodb-guardium/MongoDBOverSyslogPackage/mongodbSyslog.conf new file mode 100644 index 000000000..0a81387b9 --- /dev/null +++ b/filter-plugin/logstash-filter-mongodb-guardium/MongoDBOverSyslogPackage/mongodbSyslog.conf @@ -0,0 +1,55 @@ +#/* +#Copyright 2020-2021 IBM Inc. All rights reserved +#SPDX-License-Identifier: Apache-2.0 +#*/ + +input { + tcp { + port => 5001 + type => "syslog-mongodb" + dns_reverse_lookup_enabled => false + ssl_enable => true + # ssl_certificate_authorities => SSL_CERT_AUTH + ssl_cert => "/service/certs/external/tls-syslog.crt" + ssl_key => "/service/certs/external/tls-syslog.key" + ssl_verify => true + } +} + + + +filter { +if [type] == "syslog-mongodb" { + # break apart the message and prepare for what filter expects + grok { + match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:server_hostname} %{SYSLOGPROG:source_program}(?:[%{POSINT:syslog_pid}])?: %{GREEDYDATA:syslog_message}" } + } + + date { + match => [ "timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ] + } + + mutate { rename => { "host" => "server_ip" } } + mutate { gsub => ["message", "mongod\\\[\\\d+\\\]:", "mongod:"] } + + # send to filter + mongodb_guardium_filter {} + + # keep original event fields, for debugging + if "_mongoguardium_json_parse_error" not in [tags] { + mutate { remove_field => [ + "message", "syslog_timestamp", "source_program", "program", + "syslog_pid", "syslog_message", + "server_hostname", "client_hostname", "host", + "ecs", "log", "agent", "input"] + } + } +} + + +# uncomment to test events/sec +# metrics { +# meter => "events" +# add_tag => "metric" +# } +} diff --git a/filter-plugin/logstash-filter-mongodb-guardium/MongodbOverFilebeatPackage/mongodbFilebeat.conf b/filter-plugin/logstash-filter-mongodb-guardium/MongodbOverFilebeatPackage/mongodbFilebeat.conf new file mode 100644 index 000000000..26473213d --- /dev/null +++ b/filter-plugin/logstash-filter-mongodb-guardium/MongodbOverFilebeatPackage/mongodbFilebeat.conf @@ -0,0 +1,36 @@ +#/* +#Copyright 2020-2021 IBM Inc. All rights reserved +#SPDX-License-Identifier: Apache-2.0 +#*/ + +input { + beats { + port => + type => "filebeat" + # For SSL over Filebeat, uncomment the following lines after generating an SSL key and a certificate authority (CA) using GuardAPI (see documentation), copy the public certificate authority (CA) to your data source and adjust Filebeat configuration: + #ssl => true + #ssl_certificate => "${SSL_DIR}/cert.pem" + #ssl_key => "${SSL_DIR}/key.pem" + } +} +filter{ + if [type] == "filebeat" and "guc_filter_param_datasource_tag" in [tags] { + mutate { add_field => { "source_program" => "mongod" } } + mutate { add_field => { "server_hostname" => "%{[host][name]}" } } + mutate { add_field => { "server_ip" => "%{[host][ip][0]}" } } + mutate { replace => { "message" => "%{source_program}: %{message}" } } + + mongodb_guardium_filter {} + + if "_mongoguardium_json_parse_error" not in [tags] { + mutate { remove_field => ["message","syslog_timestamp","source_program","program","syslog_pid","syslog_message","server_hostname","client_hostname","host","ecs","log","agent","input"] } + } + } + +# uncomment to test events/sec +# metrics { +# meter => "events" +# add_tag => "metric" +# } +} + diff --git a/filter-plugin/logstash-filter-mongodb-guardium/MongodbOverMongoAtlasPackage/mongodbAtlas.conf b/filter-plugin/logstash-filter-mongodb-guardium/MongodbOverMongoAtlasPackage/mongodbAtlas.conf new file mode 100644 index 000000000..5fcaa8436 --- /dev/null +++ b/filter-plugin/logstash-filter-mongodb-guardium/MongodbOverMongoAtlasPackage/mongodbAtlas.conf @@ -0,0 +1,37 @@ +input { + mongo_atlas_input{ + interval => 300 + public-key => "" + private-key => "" + group-id => "" # Example: 61f8b9021d9dcc4b97fbfcf1 + hostname => "" # Example: cluster1-shard-00-02.i2jq9.mongodb.net + # For MongoDB Atlas deployments on custom domains or private cloud installations (other than cloud.mongodb.com), + # uncomment the following line and specify your MongoDB Atlas API base URL. + # mongo-api-url => "" # Example: https://cloud.mongodb.com/api/atlas/v1.0/groups/ + type => "mongodbatlas" + } +} + +filter { + if [type] == "mongodbatlas" { + mutate { add_field => { "source_program" => "mongod" } } + mutate { add_field => { "client_hostname" => "%{[agent][hostname]}" } } + mutate { add_field => { "server_hostname" => "%{hostname}" } } + mutate { add_field => { "server_ip" => "%{[host][ip][0]}" } } + mutate { replace => { "message" => "%{source_program}: %{message}" } } + + mongodb_guardium_filter {} + + # keep original event fields, for debugging + if "_mongoguardium_json_parse_error" not in [tags] { + mutate { remove_field => [ + "message", "syslog_timestamp", + "source_program", "program", + "syslog_pid", "syslog_message", + "server_hostname", "client_hostname", "host", + "ecs", "log", "agent", "input"] + } + } + } +} + diff --git a/filter-plugin/logstash-filter-mongodb-guardium/README.md b/filter-plugin/logstash-filter-mongodb-guardium/README.md index 580c3c76a..ae103842e 100644 --- a/filter-plugin/logstash-filter-mongodb-guardium/README.md +++ b/filter-plugin/logstash-filter-mongodb-guardium/README.md @@ -60,14 +60,14 @@ The filter plug-in also supports sending errors. For this, MongoDB access contro * The "type" fields should match in the input and the filter configuration sections. This field should be unique for every individual connector added. ## Example -### Filebeat input +### Sample Audit Log A typical original log file looks like: ``` { "atype" : "authCheck", "ts" : { "$date" : "2020-02-16T03:21:58.185-0500" }, "local" : { "ip" : "127.0.30.1", "port" : 0 }, "remote" : { "ip" : "127.0.20.1", "port" : 0 }, "users" : [], "roles" : [], "param" : { "command" : "find", "ns" : "config.transactions", "args" : { "find" : "transactions", "filter" : { "lastWriteDate" : { "$lt" : { "$date" : "2020-02-16T02:51:58.185-0500" } } }, "projection" : { "_id" : 1 }, "sort" : { "_id" : 1 }, "$db" : "config" } }, "result" : 0 } ``` -The Filebeat version of the same file looks like: +### The Filebeat version of the same Sample Audit Log looks like: ``` { "@version" => "1", @@ -105,46 +105,6 @@ The Filebeat version of the same file looks like: } ``` -## Filter result -The filter tweaks the event by adding a _GuardRecord_ field to it with a JSON representation of a Guardium record object. As the filter takes the responsiblity of breaking the database command into its atomic parts, it details the construct object with the parsed command structure: - { - - "sequence" => 0, - "GuardRecord" => "{"sessionId":"mV20eHvvRha2ELTeqJxQJg\u003d\u003d","dbName":"admin","appUserName":"","time":{"timstamp":1591883051070,"minOffsetFromGMT":-240,"minDst":0},"sessionLocator":{"clientIp":"9.148.202.94","clientPort":60185,"serverIp":"9.70.147.59","serverPort":27017,"isIpv6":false,"clientIpv6":"","serverIpv6":""},"accessor":{"dbUser":"realAdmin ","serverType":"MongoDB","serverOs":"","clientOs":"","clientHostName":"","serverHostName":"","commProtocol":"","dbProtocol":"MongoDB native audit","dbProtocolVersion":"","osUser":"","sourceProgram":"","client_mac":"","serverDescription":"","serviceName":"admin","language":"FREE_TEXT","dataType":"CONSTRUCT"},"data":{"construct":{"sentences":[{"verb":"find","objects":[{"name":"USERS","type":"collection","fields":[],"schema":""}],"descendants":[],"fields":[]}],"fullSql":"{\"atype\":\"authCheck\",\"ts\":{\"$date\":\"2020-06-11T09:44:11.070-0400\"},\"local\":{\"ip\":\"9.70.147.59\",\"port\":27017},\"remote\":{\"ip\":\"9.148.202.94\",\"port\":60185},\"users\":[{\"user\":\"realAdmin\",\"db\":\"admin\"}],\"roles\":[{\"role\":\"readWriteAnyDatabase\",\"db\":\"admin\"},{\"role\":\"userAdminAnyDatabase\",\"db\":\"admin\"}],\"param\":{\"command\":\"find\",\"ns\":\"admin.USERS\",\"args\":{\"find\":\"USERS\",\"filter\":{},\"lsid\":{\"id\":{\"$binary\":\"mV20eHvvRha2ELTeqJxQJg\u003d\u003d\",\"$type\":\"04\"}},\"$db\":\"admin\",\"$readPreference\":{\"mode\":\"primaryPreferred\"}}},\"result\":0}","redactedSensitiveDataSql":"{\"atype\":\"authCheck\",\"ts\":{\"$date\":\"2020-06-11T09:44:11.070-0400\"},\"local\":{\"ip\":\"9.70.147.59\",\"port\":27017},\"remote\":{\"ip\":\"9.148.202.94\",\"port\":60185},\"users\":[{\"user\":\"realAdmin\",\"db\":\"admin\"}],\"roles\":[{\"role\":\"readWriteAnyDatabase\",\"db\":\"admin\"},{\"role\":\"userAdminAnyDatabase\",\"db\":\"admin\"}],\"param\":{\"command\":\"find\",\"ns\":\"admin.USERS\",\"args\":{\"filter\":{},\"lsid\":{\"id\":{\"$binary\":\"?\",\"$type\":\"?\"}},\"$readPreference\":{\"mode\":\"?\"},\"find\":\"USERS\",\"$db\":\"admin\"}},\"result\":0}"},"originalSqlCommand":""},"exception":null}", - "@version" => "1", - "@timestamp" => 2020-02-25T12:32:16.314Z, - "type" => "syslog", - "timestamp" => "2020-01-26T10:47:41.225-0500" - } - -This Guardium record, which is added to Logstash event after the filter, is examined and handled by Guardium universal connector (in an output stage) and inserted into Guardium. - -If the event message is not related to MongoDB, the event is tagged with "_mongoguardium_skip_not_mongodb" (not removed from the pipeline). If it is an event from MongoDB but JSON parsing fails, the event is tagged with "_mongoguardium_json_parse_error" but not removed (this may happen if the syslog message is too long and was truncated). These tags can be useful for debugging purposes. - - -To build and create an updated GEM of this filter plug-in which can be installed onto Logstash: -1. Build Logstash from the repository source. -2. Create or edit _gradle.properties_ and add the LOGSTASH_CORE_PATH variable with the path to the logstash-core folder. For example: - - ```LOGSTASH_CORE_PATH=/Users/taldan/logstash76/logstash-core``` - -3. Run ```$ ./gradlew.unix gem --info``` to create the GEM. - - **Note**: Ensure that JRuby is already installed. - -## Install -To install this plug-in on your local developer machine with Logstash installed, issue this command: - - $ ~/Downloads/logstash-7.5.2/bin/logstash-plugin install ./logstash-filter-mongodb_guardium_filter-?.?.?.gem - -Notes: -* Replace "?" with this plug-in version. -* The logstash-plugin may not handle relative paths well. It is recommended that you install the GEM from a simple path, as in the above example. - -To test the filter using your local Logstash installation, run this command: - - $ logstash -f ./filter-test-generator.conf --config.reload.automatic - ## Configuring audit logs on MongoDB and forwarding to Guardium via Filebeat First, configure the MongoDB native audit logs so that they can be parsed by Guardium. Then, configure Filebeat to forward the audit logs to the Guardium universal connector. This implementation supports Linux and Windows database servers. @@ -155,7 +115,7 @@ First, configure the MongoDB native audit logs so that they can be parsed by Gua - Filebeat must be installed on your database server. For more information on installation, see [https://www.elastic.co/guide/en/beats/filebeat/current/setup-repositories.html\#\_yum](https://www.elastic.co/guide/en/beats/filebeat/current/setup-repositories.html#_yum). The recommended Filebeat version is 7.5.0 and higher. - Native audit configuration is performed by the database admin. - Filebeat cannot handle messages over approximately 1 GB. Make sure the MongoDB does not save files larger than this limit \(by using `logRotate`\). File messages that exceed the limit are dropped. -- You can configure multiple collectors simultaneously by using GIM \([Configuring the GIM client to handle Filebeat and Syslog on MongoDB](https://github.com/IBM/universal-connectors/blob/main/docs/general%20topics/GIM.md). If you configure collectors manually, you need to configure them individually. +- You can configure multiple collectors simultaneously by using GIM [Configuring the GIM client to handle Filebeat and Syslog on MongoDB](https://github.com/IBM/universal-connectors/blob/main/docs/general%20topics/GIM.md). If you configure collectors manually, you need to configure them individually. - For more information about MongoDB native audit, see [https://docs.mongodb.com/manual/core/auditing/](https://docs.mongodb.com/manual/core/auditing/). ### Procedure @@ -164,62 +124,56 @@ First, configure the MongoDB native audit logs so that they can be parsed by Gua a. Configure the AuditLog section in the mongod.conf file. - - `destination`: file - - `format`: JSON - - `path`: /var/log/mongodb/.json, for example /var/log/mongodb/auditLog.json + - destination: file + - format: JSON + - path: /var/log/mongodb/.json for example /var/log/mongodb/auditLog.json b. Add the following field to audit the `auditAuthorizationSuccess` messages: - - - setParameter: {auditAuthorizationSuccess: **true**} - +``` + setParameter: {auditAuthorizationSuccess: true} +``` c. Add or uncomment the security section and edit the following parameter: - - authorization: **enabled** + ``` + authorization: enabled + ``` d. `filter`: For the Guardium universal connector MongoDB filter to handle events properly, a few conditions must exist: - - MongoDB access control must be set. \(Messages without users are removed.\) - - - `authCheck` and `authenticate events` are not filtered out from the MongoDB audit log messages. Verify that the filter section contains at least the following commands: - - - - '{ atype: { $in: ["authCheck", "authenticate"] }' - - - To narrow down the events, you can tweak the filter. - - - To audit only the delete actions made in MongoDB, for example, add the following suffix to the filter section: - - - '{ atype: { $in: ["authCheck", "authenticate"] } ' - "param.command": { $in: [" - delete"] } }' - - - - Auditing all commands can lead to excessive records. To prevent performance issues, make sure you have `authCheck` and `authenticate` log types, and any other commands you want to see. The filter parameters are an allowed list. They define what you see in the logs, not what is filtered from the logs. For more information about the MongoDB filter, see [Configuring Audit Filters](https://docs.mongodb.com/manual/tutorial/configure-audit-filters/) and [Configuring Filebeat](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-installation-configuration.html#installation). - - **Note:** The spaces in the configuration file are important, and must be located in the file as presented here. - - After configuration, the file has these lines: - - - ... - auditLog: - destination: file - format: JSON - path: /var/lib/mongo/auditLog.json - filter: '{ atype: { $in: ["authCheck", "authenticate"] } , "param.command": { $in: ["delete"] } }' - setParameter: {auditAuthorizationSuccess: true} - ... - security: - authorization: enabled - + * MongoDB access control must be set. (Messages without users are removed.) + * `authCheck` and `authenticate` events are not filtered out from the MongoDB audit log messages. + Verify that the filter section contains at least the following commands: + ``` + '{ atype: { $in: ["authCheck", "authenticate"] }' + ``` + To narrow down the events, you can tweak the filter. + For example, To audit only the delete actions made in MongoDB, add the following suffix to the filter section: + ``` + '{ atype: { $in: ["authCheck", "authenticate"] } ' + "param.command": { $in: [" + delete"] } }' + ``` + * Auditing all commands can lead to excessive records. To prevent performance issues, make sure you have `authCheck` and `authenticate` log types, and any other commands you want to see. The filter parameters are an allowed list. They define what you see in the logs, not what is filtered from the logs. For more information about the MongoDB filter, see [Configuring Audit Filters](https://docs.mongodb.com/manual/tutorial/configure-audit-filters/) and [Configuring Filebeat](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-installation-configuration.html#installation). + + **Note:** The spaces in the configuration file are important, and must be located in the file as presented here. + + e. After configuration, the file has these lines: +``` + ... + auditLog: + destination: file + format: JSON + path: /var/lib/mongo/auditLog.json + filter: '{"$or": [{ atype: { $ne: ["authCheck"] }, "param.command": { $in: [ "find", "insert", "delete", "update", "findandmodify", "create", "drop", "mapReduce", "applyOps", "eval", "resetError","renameCollection","adminCommand"] } },{ atype: "authCheck", "param.command": { $in: ["aggregate"]}},{atype:"authenticate", result:{ $ne: 0 }}]}' + setParameter: {auditAuthorizationSuccess: true} + ... + security: + authorization: enabled +``` **Important:** The MongoDB needs to be restarted for the configuration changes to take effect. + 2. Configure the Filebeat data shipper to forward the audit logs to the Guardium universal connector. In the file filebeat.yml, usually located in /etc/filebeat/filebeat.yml, modify the Filebeat inputs section. a. Select a template from the Universal Connector page and enter your desired port in the port line, beginning at port 5001. \(Use a new port for each new future connection.\) Save the configuration. @@ -228,18 +182,19 @@ First, configure the MongoDB native audit logs so that they can be parsed by Gua filebeat.inputs - - type: log - enabled: **true** + - type: filestream + - id: + enabled: true paths: - - **/var/log/mongodb/auditLog.json** + - /var/log/mongodb/auditLog.json #- c:\programdata\elasticsearch\logs\* tags: ["mongodb"] c. If you send multiple, different data sources from the same server on the same port: -- Attach a different tag to each input log. Then, use the tags when you configure the connector -- Use the ```tags``` parameter from the following code while configuring the connector: + - Attach a different tag to each input log. Then, use the tags when you configure the connector + - Use the ```tags``` parameter from the following code while configuring the connector: # ============================== Filebeat inputs =============================== @@ -247,78 +202,34 @@ First, configure the MongoDB native audit logs so that they can be parsed by Gua # Each -is an input. Most options can be set at the input level, so # you can use different inputs for various configurations. # Below are the input specific configurations. - -type: log + -type: filestream + - id: # Change to true to enable this input configuration. enabled: true # Paths that should be crawled and fetched. Glob based paths. - paths:-/var/lib/mongo/auditLog.json + paths: /var/lib/mongo/auditLog.json tags: ["mongodb"] d. In the Outputs section: -- Make sure that Elasticsearch output is commented out. - - Add or uncomment the Logstash output and edit the following parameters: - - Add all the Guardium Universal Connector IPs and ports: + - Make sure that Elasticsearch output is commented out. + - Add or uncomment the Logstash output and edit the following parameters: + - Add all the Guardium Universal Connector IPs and ports: - - hosts: **hosts: \[“:”,”:,”:”...\]** + hosts: [“:”,”:,”:”...] - - Use the same port you selected when configuring the Universal Connector. - - Enable load balancing: + - Use the same port you selected when configuring the Universal Connector. + - Enable load balancing: - loadbalance: **true** + loadbalance: true - For more information on Elastic's Filebeat load-balancing, see: [https://www.elastic.co/guide/en/beats/filebeat/current/load-balancing.html](https://www.elastic.co/guide/en/beats/filebeat/7.17/load-balancing.html) - More optional parameters are described in the Elastic official documentation: [https://www.elastic.co/guide/en/beats/filebeat/current/logstash-output.html](https://www.elastic.co/guide/en/beats/filebeat/current/logstash-output.html) - A typical original log file looks like: - - ``` - { "atype" : "authCheck", "ts" : { "$date" : "2020-02-16T03:21:58.185-0500" }, "local" : { "ip" : "127.0.30.1", "port" : 0 }, "remote" : { "ip" : "127.0.20.1", "port" : 0 }, "users" : [], "roles" : [], "param" : { "command" : "find", "ns" : "config.transactions", "args" : { "find" : "transactions", "filter" : { "lastWriteDate" : { "$lt" : { "$date" : "2020-02-16T02:51:58.185-0500" } } }, "projection" : { "_id" : 1 }, "sort" : { "_id" : 1 }, "$db" : "config" } }, "result" : 0 } - ``` - - The Filebeat version of the same file looks like: - - - { - "@version" => "1", - "input" => { "type" => "log"}, - "tags" => [[0] "beats_input_codec_plain_applied"], - "@timestamp" => 2020-06-11T13:46:20.663Z, - "log" => {"offset" => 1997890,"file" => { "path" =>"C:\\Users\\Name\\Desktop\\p1.log" }}, - "ecs" => {"version" => "1.4.0"}, - "type" => "filebeat", - "agent" => { - "ephemeral_id" => - "b7d849f9-dfa9-4d27-be8c-20061b1facdf", - "id" => - "a54b2184-0bb5-4683-a039-7e1c70f1a57c", - "version" => "7.6.2", - "type" => "filebeat", - "hostname" => "" - }, - "message" =>"{ \"atype\" : \"authCheck\", \"ts\" : { \"$date\" : \"2020-02-16T03:21:58.185-0500\" }, \"local\" : { \"ip\" : \"127.0.30.1\", \"port\" : 0 }, \"remote\" : { \"ip\" : \"127.0.20.1\", \"port\" : 0 }, \"users\" : [], \"roles\" : [], \"param\" : { \"command\" : \"find\", \"ns\" : \"config.transactions\", \"args\" : { \"find\" : \"transactions\", \"filter\" : { \"lastWriteDate\" : { \"$lt\" : { \"$date\" : \"2020-02-16T02:51:58.185-0500\" } } }, \"projection\" : { \"_id\" : 1 }, \"sort\" : { \"_id\" : 1 }, \"$db\" : \"config\" } }, \"result\" : 0 }", - "host" => { - "architecture" => - "x86_64", - "id" => "d4e2c297-47bf-443a-8af8-e921715ed047", - "os" => { - "version" => "10.0", - "kernel" => "10.0.18362.836 (WinBuild.160101.0800)", - "build" => "18363.836", - "name" => "Windows 10 Enterprise", - "platform" => "windows", - "family" => "windows" - }, - "name" => "", - "hostname" => "" - } - } - 3. Restart Filebeat to effect these changes. @@ -333,9 +244,130 @@ First, configure the MongoDB native audit logs so that they can be parsed by Gua #### For details on configuring Filebeat connection over SSL, refer [Configuring Filebeat to push logs to Guardium](https://github.com/IBM/universal-connectors/blob/main/input-plugin/logstash-input-beats/README.md#configuring-filebeat-to-push-logs-to-guardium). -### What to do next +## Configuring Syslog to push logs to Guardium + +### Syslogs configuration: +To make the Logstash able to process the data collected by syslogs, configure available +syslog utility. The example is based on rsyslog utility available in many +versions of the Linux distributions. To check the service is active and running, execute the below +command: + +```text +systemctl status rsyslog +``` -Enable the universal connector on your collector. [Enabling the Guardium universal connector on collectors](https://www.ibm.com/docs/en/SSMPHH_11.4.0/com.ibm.guardium.doc.stap/guc/cfg_guc_input_filters.html) +#### Rsyslog installation guide: +* [Ubuntu](https://www.rsyslog.com/ubuntu-repository) +* [RHEL](https://www.rsyslog.com/rhelcentos-rpms) + +1. Generate Certificate Authority (CA): + * **Guardium Data Protection**
+ To obtain the Certificate Authority content on the Collector, run the following API command: + ```text + grdapi generate_ssl_key_universal_connector + ``` + This API command will display the content of the public Certificate Authority. Copy this certificate authority content to your database source and save it as a file named 'ca.pem' . + + * **Guardium Data Security Center - SaaS**
+ Refer to the instructions provided [here](https://www.ibm.com/docs/en/gdsc/saas?topic=connector-connecting-data-source-by-using-universal#plugin_connection_configuration__title__15) to obtain the Certificate Authority + and connection details for Guardium Insights-SaaS. +2. Create a file with name `mongo_syslog.conf` in the /etc/rsyslog.d/ directory with the content below in the +snippet and change the values of target and port, + ```text + global(DefaultNetstreamDriverCAFile="/path/to/ca_file/ca.pem") + # The template for message formatting + $template UcMessageFormat,"%HOSTNAME%,,%msg%" + + module(load="imfile") + ruleset(name="imfile_to_gdp") { + action(type="omfwd" + protocol="tcp" + StreamDriver="gtls" + StreamDriverMode="1" + StreamDriverAuthMode="x509/certvalid" + template="UcMessageFormat" + target="" + port="") + } + + input( + type="imfile" + file="/path/to/logs/directory/auditLog.json" + # Keep the value of tag below as same as here, + tag="syslog" + ruleset="imfile_to_gdp" + ) + ``` + This configuration reads the logs from the MongoDB log directory path and sends +the syslog messages to the provided host (target_host) at the provided port (target_port).

+ + **NOTE**: For further configuration requirements that are specific to Guardium Insights - SaaS +environment, please follow the instructions provided [here](https://github.com/IBM/universal-connectors/blob/main/docs/Guardium%20Insights/SaaS_1.0/UC_Configuration_GI.md#tcp-input-plug-in-configuration-for-connection-with-syslog). +

+ +3. Include this file in the main rsyslog configurations file. + 1. Open the file `/etc/rsyslog.conf`. + 2. Append the below line at the end. + ```text + $IncludeConfig /etc/rsyslog.d/mongo_syslog.conf + ``` +4. Restart the rsyslog utility. + ```text + systemctl restart rsyslog + ``` + +## Configuring the MongoDB filters in Guardium +The Guardium universal connector is the Guardium entry point for native audit logs. The universal connector identifies and parses received events, and then converts them to a standard Guardium format. The output of the universal connector is forwarded to the Guardium sniffer on the collector, for policy and auditing enforcements. Configure Guardium to read the native audit logs by customizing the MongoDB template. + +**Important** + +• Starting with Guardium Data Protection version 12.1, you can configuring the Universal Connectors in 2 ways. You can either use the legacy flow or the new flow. + +• To configure Universal Connector by using the new flow, see [Managing universal connector configuration](https://www.ibm.com/docs/en/gdp/12.x?topic=connector-managing-universal-configuration) on the Guardium Universal Connector page. + +• To configure the Universal Connector by using the legacy flow, use the procedure in this topic. + +### Limitations +* The filter supports events sent through Syslog or Filebeat. It relies on the "mongod:" or "mongos:" prefixes in +the event message for the JSON portion of the audit to be parsed. +* Field **server_hostname** (required) - Server hostname is expected (extracted from the nested field "name" +inside the host object of the Filebeat message). +* Field **server_ip** - States the IP address of the MongoDB server, if it is available to the +filter plug-in. The filter will use this IP address instead of localhost IP addresses +that are reported by MongoDB, if actions were performed directly on the database server. +* The client "Source program" is not available in messages sent by MongoDB. This is because +this data is sent only in the first audit log message upon database connection - and the +filter plug-in doesn't aggregate data from different messages. + + +### Before You Begin +* Configure the policies you require. See [policies](https://github.com/IBM/universal-connectors/blob/main/docs/Guardium%20Data%20Protection/uc_policies_gdp.md) for more information. +* You must have permission for the S-Tap Management role. The admin user includes this role by default. + + +### Configuration +1. On the collector, go to ```Setup``` > ```Tools and Views``` > ```Configure Universal Connector```. +2. Enable the universal connector if it is disabled. +3. Click the plus sign to open the Connector Configuration dialog box. +4. Type a name in the ```Connector name``` field. +5. Update the input section, + 1. To collect data over Filebeat, add the details from [mongoDBFilebeat.conf](./MongodbOverFilebeatPackage/mongodbFilebeat.conf) + file input section, omitting the keyword "input{" at the beginning and its corresponding "}" + at the end. + 2. To collect data over Syslogs, add the details from [mongoDBSyslog.conf](./MongoDBOverSyslogPackage/mongodbSyslog.conf) file input section, + omitting the keyword "input{" at the beginning and its corresponding "}" at the end. + 3. To collect data over Mongo Atlas API, add the details from [mongoAtlas.conf](./MongodbOverMongoAtlasPackage/mongodbAtlas.conf) file input section, + omitting the keyword "input{" at the beginning and its corresponding "}" at the end. +6. Update the filter section, + 1. To filter the data collected from the Filebeat, add the details from the + [mongoDBFilebeat.conf](./MongodbOverFilebeatPackage/mongodbFilebeat.conf) file filter section, omitting the keyword + "filter{" at the beginning and its corresponding "}" at the end. + 2. To filter the data collected from the Syslogs, add the details from the + [mongoDBSyslog.conf](MongoDBOverSyslogPackage/mongodbSyslog.conf) file filter section, + omitting the keyword "filter{" at the beginning and its corresponding "}" at the end. + 3. To filter the data collected from the Mongo Atlas API, add the details from the [mongoAtlas.conf](./MongodbOverMongoAtlasPackage/mongodbAtlas.conf) file filter section, omitting the keyword "filter{" at the beginning and its corresponding "}" at the end. +7. The "type" fields should match in the input and the filter configuration sections. This field should be unique for every individual connector added. This is no longer required starting v12p20 and v12.1. +8. Click ```Save```. Guardium validates the new connector, and enables the universal connector if it was disabled. After it is validated, it appears in the Configure Universal Connector page. ## Configuring the MongoDB filters in Guardium Data Security Center diff --git a/filter-plugin/logstash-filter-mongodb-guardium/VERSION b/filter-plugin/logstash-filter-mongodb-guardium/VERSION index e9acb99e6..4655c9e99 100644 --- a/filter-plugin/logstash-filter-mongodb-guardium/VERSION +++ b/filter-plugin/logstash-filter-mongodb-guardium/VERSION @@ -1 +1 @@ -0.6.12 \ No newline at end of file +0.6.13 \ No newline at end of file diff --git a/filter-plugin/logstash-filter-mongodb-guardium/build.gradle b/filter-plugin/logstash-filter-mongodb-guardium/build.gradle index 70f682988..d414e4399 100644 --- a/filter-plugin/logstash-filter-mongodb-guardium/build.gradle +++ b/filter-plugin/logstash-filter-mongodb-guardium/build.gradle @@ -2,6 +2,30 @@ import java.nio.file.Files import static java.nio.file.StandardCopyOption.REPLACE_EXISTING apply plugin: 'java' + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + + dependencies { + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' + classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.1.0" + classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' + } +} + +def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" // =========================================================================== @@ -20,10 +44,10 @@ pluginInfo.pluginClass = "MongodbGuardiumFilter" pluginInfo.pluginName = "mongodb_guardium_filter" // must match the @LogstashPlugin annotation in the main plugin class // =========================================================================== -sourceCompatibility = 1.8 -targetCompatibility = 1.8 +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 -def jacocoVersion = '0.8.4' +def jacocoVersion = '0.8.11' // minimumCoverage can be set by Travis ENV def minimumCoverageStr = System.getenv("MINIMUM_COVERAGE") ?: "50.0%" if (minimumCoverageStr.endsWith("%")) { @@ -31,27 +55,16 @@ if (minimumCoverageStr.endsWith("%")) { } def minimumCoverage = Float.valueOf(minimumCoverageStr) / 100 -buildscript { - repositories { - maven { - url "https://plugins.gradle.org/m2/" - } - mavenCentral() - jcenter() - } - - dependencies { - classpath 'com.github.jengelman.gradle.plugins:shadow:4.0.4' - classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.0.0-RC2" - classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' - } -} - -def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); -def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) repositories { - mavenCentral() + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } } tasks.register("vendor"){ @@ -62,14 +75,13 @@ tasks.register("vendor"){ File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") projectJarFile.mkdirs() Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) - validatePluginJar(projectJarFile, project.group) } } apply plugin: 'com.github.johnrengelman.shadow' shadowJar { - classifier = null + archiveClassifier = null } dependencies { @@ -77,7 +89,8 @@ dependencies { implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: versions.dependencies.log4jCore implementation 'org.apache.commons:commons-lang3:' + versions.dependencies.commonsLang implementation 'com.google.code.gson:gson:' + versions.dependencies.gson - implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "build/libs/logstash-core-*.*.*.jar") + implementation group: 'commons-beanutils', name: 'commons-beanutils', version: versions.dependencies.commonsBeanutils + implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "build/libs/logstash-core.jar") implementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "common-*.*.*.jar") testImplementation 'junit:junit:' + versions.dependencies.junit @@ -100,6 +113,17 @@ tasks.withType(JavaCompile) { options.encoding = 'UTF-8' } +test { + jvmArgs = [ + '--add-opens=java.base/java.lang=ALL-UNNAMED', + '--add-opens=java.base/java.util=ALL-UNNAMED', + '--add-opens=java.base/java.text=ALL-UNNAMED', + '--add-opens=java.base/java.lang.reflect=ALL-UNNAMED', + '--add-opens=java.base/sun.util.resources=ALL-UNNAMED', + '--add-opens=java.base/sun.util.cldr=ALL-UNNAMED' + ] +} + tasks.register("generateRubySupportFiles") { @@ -142,17 +166,16 @@ apply plugin: "org.barfuin.gradle.jacocolog" // ------------------------------------ jacoco { toolVersion = "${jacocoVersion}" - reportsDir = file("$buildDir/reports/jacoco") } jacocoTestReport { // You will see "Report -> file://...." at the end of a JaCoCo build // If no output, run this first: ./gradlew test reports { - html.enabled true - xml.enabled true - csv.enabled true - html.destination file("${buildDir}/reports/jacoco") - csv.destination file("${buildDir}/reports/jacoco/all.csv") + html.required = true + xml.required = true + csv.required = true + html.outputLocation = file("${buildDir}/reports/jacoco") + csv.outputLocation = file("${buildDir}/reports/jacoco/all.csv") } executionData.from fileTree(dir: "${buildDir}/jacoco/", includes: [ '**/*.exec' diff --git a/filter-plugin/logstash-filter-mongodb-guardium/gradle/wrapper/gradle-wrapper.properties b/filter-plugin/logstash-filter-mongodb-guardium/gradle/wrapper/gradle-wrapper.properties index aa991fcea..81aa1c044 100644 --- a/filter-plugin/logstash-filter-mongodb-guardium/gradle/wrapper/gradle-wrapper.properties +++ b/filter-plugin/logstash-filter-mongodb-guardium/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/filter-plugin/logstash-filter-mongodb-guardium/src/main/java/com/ibm/guardium/mongodb/MongodbGuardiumFilter.java b/filter-plugin/logstash-filter-mongodb-guardium/src/main/java/com/ibm/guardium/mongodb/MongodbGuardiumFilter.java index 269d42e79..06fae2065 100644 --- a/filter-plugin/logstash-filter-mongodb-guardium/src/main/java/com/ibm/guardium/mongodb/MongodbGuardiumFilter.java +++ b/filter-plugin/logstash-filter-mongodb-guardium/src/main/java/com/ibm/guardium/mongodb/MongodbGuardiumFilter.java @@ -15,7 +15,15 @@ import com.ibm.guardium.mongodb.parsersbytype.BaseParser; import com.ibm.guardium.universalconnector.commons.GuardConstants; import com.ibm.guardium.universalconnector.commons.Util; -import com.ibm.guardium.universalconnector.commons.structures.*; +import com.ibm.guardium.universalconnector.commons.structures.Accessor; +import com.ibm.guardium.universalconnector.commons.structures.Construct; +import com.ibm.guardium.universalconnector.commons.structures.Data; +import com.ibm.guardium.universalconnector.commons.structures.ExceptionRecord; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import com.ibm.guardium.universalconnector.commons.structures.Sentence; +import com.ibm.guardium.universalconnector.commons.structures.SentenceObject; +import com.ibm.guardium.universalconnector.commons.structures.SessionLocator; +import com.ibm.guardium.universalconnector.commons.structures.Time; import org.apache.commons.validator.routines.InetAddressValidator; import org.apache.logging.log4j.LogManager; diff --git a/filter-plugin/logstash-filter-mongodb-guardium/src/main/java/com/ibm/guardium/mongodb/Parser.java b/filter-plugin/logstash-filter-mongodb-guardium/src/main/java/com/ibm/guardium/mongodb/Parser.java index aae5c5aed..00efd627f 100644 --- a/filter-plugin/logstash-filter-mongodb-guardium/src/main/java/com/ibm/guardium/mongodb/Parser.java +++ b/filter-plugin/logstash-filter-mongodb-guardium/src/main/java/com/ibm/guardium/mongodb/Parser.java @@ -19,7 +19,15 @@ import com.google.gson.JsonObject; import com.google.gson.JsonPrimitive; import com.ibm.guardium.universalconnector.commons.Util; -import com.ibm.guardium.universalconnector.commons.structures.*; +import com.ibm.guardium.universalconnector.commons.structures.Accessor; +import com.ibm.guardium.universalconnector.commons.structures.Construct; +import com.ibm.guardium.universalconnector.commons.structures.Data; +import com.ibm.guardium.universalconnector.commons.structures.ExceptionRecord; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import com.ibm.guardium.universalconnector.commons.structures.Sentence; +import com.ibm.guardium.universalconnector.commons.structures.SentenceObject; +import com.ibm.guardium.universalconnector.commons.structures.SessionLocator; +import com.ibm.guardium.universalconnector.commons.structures.Time; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -27,7 +35,7 @@ public class Parser { private static Logger log = LogManager.getLogger(Parser.class); - public static final String DATA_PROTOCOL_STRING = "MongoDB native audit"; + public static final String DATA_PROTOCOL_STRING = "MongoDB"; public static final String UNKOWN_STRING = ""; public static final String SERVER_TYPE_STRING = "MongoDB"; private static final String MASK_STRING = "?"; diff --git a/filter-plugin/logstash-filter-mongodb-guardium/src/main/java/com/ibm/guardium/mongodb/parsersbytype/BaseParser.java b/filter-plugin/logstash-filter-mongodb-guardium/src/main/java/com/ibm/guardium/mongodb/parsersbytype/BaseParser.java index 9b2b5dab5..40b3d89ef 100644 --- a/filter-plugin/logstash-filter-mongodb-guardium/src/main/java/com/ibm/guardium/mongodb/parsersbytype/BaseParser.java +++ b/filter-plugin/logstash-filter-mongodb-guardium/src/main/java/com/ibm/guardium/mongodb/parsersbytype/BaseParser.java @@ -3,7 +3,15 @@ import com.google.gson.*; import com.ibm.guardium.mongodb.Parser; import com.ibm.guardium.universalconnector.commons.Util; -import com.ibm.guardium.universalconnector.commons.structures.*; +import com.ibm.guardium.universalconnector.commons.structures.Accessor; +import com.ibm.guardium.universalconnector.commons.structures.Construct; +import com.ibm.guardium.universalconnector.commons.structures.Data; +import com.ibm.guardium.universalconnector.commons.structures.ExceptionRecord; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import com.ibm.guardium.universalconnector.commons.structures.Sentence; +import com.ibm.guardium.universalconnector.commons.structures.SentenceObject; +import com.ibm.guardium.universalconnector.commons.structures.SessionLocator; +import com.ibm.guardium.universalconnector.commons.structures.Time; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -21,7 +29,7 @@ public abstract class BaseParser { private static Logger log = LogManager.getLogger(BaseParser.class); - public static final String DATA_PROTOCOL_STRING = "MongoDB native audit"; + public static final String DATA_PROTOCOL_STRING = "MongoDB"; public static final String UNKOWN_STRING = ""; public static final String SERVER_TYPE_STRING = "MongoDB"; private static final String MASK_STRING = "?"; diff --git a/filter-plugin/logstash-filter-mongodb-guardium/src/main/java/com/ibm/guardium/mongodb/parsersbytype/LoginParser.java b/filter-plugin/logstash-filter-mongodb-guardium/src/main/java/com/ibm/guardium/mongodb/parsersbytype/LoginParser.java index 2c5171e13..d7acdaeb8 100644 --- a/filter-plugin/logstash-filter-mongodb-guardium/src/main/java/com/ibm/guardium/mongodb/parsersbytype/LoginParser.java +++ b/filter-plugin/logstash-filter-mongodb-guardium/src/main/java/com/ibm/guardium/mongodb/parsersbytype/LoginParser.java @@ -97,7 +97,7 @@ private Accessor getAccessor(JsonObject data) { String dbUser = getInitialUser(data).get("user").getAsString(); accesor.setDbUser(dbUser); accesor.setServerType("MongoDB"); - accesor.setDbProtocol("MongoDB native audit"); + accesor.setDbProtocol("MongoDB"); String serviceName = getInitialUser(data).get("db").getAsString(); accesor.setServiceName(serviceName); diff --git a/filter-plugin/logstash-filter-mongodb-guardium/src/test/java/com/ibm/guardium/mongodb/ParserTest.java b/filter-plugin/logstash-filter-mongodb-guardium/src/test/java/com/ibm/guardium/mongodb/ParserTest.java index 9282805f6..c8e675ffb 100644 --- a/filter-plugin/logstash-filter-mongodb-guardium/src/test/java/com/ibm/guardium/mongodb/ParserTest.java +++ b/filter-plugin/logstash-filter-mongodb-guardium/src/test/java/com/ibm/guardium/mongodb/ParserTest.java @@ -9,7 +9,15 @@ import com.google.gson.JsonArray; import com.google.gson.JsonObject; import com.google.gson.JsonParser; -import com.ibm.guardium.universalconnector.commons.structures.*; +import com.ibm.guardium.universalconnector.commons.structures.Accessor; +import com.ibm.guardium.universalconnector.commons.structures.Construct; +import com.ibm.guardium.universalconnector.commons.structures.Data; +import com.ibm.guardium.universalconnector.commons.structures.ExceptionRecord; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import com.ibm.guardium.universalconnector.commons.structures.Sentence; +import com.ibm.guardium.universalconnector.commons.structures.SentenceObject; +import com.ibm.guardium.universalconnector.commons.structures.SessionLocator; +import com.ibm.guardium.universalconnector.commons.structures.Time; import org.junit.Assert; import org.junit.Test; diff --git a/filter-plugin/logstash-filter-mongodb-guardium/src/test/java/com/ibm/guardium/mongodb/parsers/LoginParserTest.java b/filter-plugin/logstash-filter-mongodb-guardium/src/test/java/com/ibm/guardium/mongodb/parsers/LoginParserTest.java index ed4015318..ea4d8b225 100644 --- a/filter-plugin/logstash-filter-mongodb-guardium/src/test/java/com/ibm/guardium/mongodb/parsers/LoginParserTest.java +++ b/filter-plugin/logstash-filter-mongodb-guardium/src/test/java/com/ibm/guardium/mongodb/parsers/LoginParserTest.java @@ -5,7 +5,15 @@ import com.google.gson.JsonParser; import com.ibm.guardium.mongodb.parsersbytype.BaseParser; import com.ibm.guardium.mongodb.parsersbytype.LoginParser; -import com.ibm.guardium.universalconnector.commons.structures.*; +import com.ibm.guardium.universalconnector.commons.structures.Accessor; +import com.ibm.guardium.universalconnector.commons.structures.Construct; +import com.ibm.guardium.universalconnector.commons.structures.Data; +import com.ibm.guardium.universalconnector.commons.structures.ExceptionRecord; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import com.ibm.guardium.universalconnector.commons.structures.Sentence; +import com.ibm.guardium.universalconnector.commons.structures.SentenceObject; +import com.ibm.guardium.universalconnector.commons.structures.SessionLocator; +import com.ibm.guardium.universalconnector.commons.structures.Time; import org.junit.Assert; import org.junit.Test; @@ -64,7 +72,7 @@ public void tesParseRecord() { Assert.assertEquals(accessor.getDbUser(), dbUser); Assert.assertEquals(accessor.getServiceName(), dbName); Assert.assertEquals(accessor.getServerType(), "MongoDB"); - Assert.assertEquals(accessor.getDbProtocol(), "MongoDB native audit"); + Assert.assertEquals(accessor.getDbProtocol(), "MongoDB"); Assert.assertEquals(accessor.getLanguage(), "FREE_TEXT"); Assert.assertEquals(accessor.getDataType(), "CONSTRUCT"); diff --git a/filter-plugin/logstash-filter-mongodb-guardium/src/test/java/com/ibm/guardium/mongodb/parsers/ParserTest.java b/filter-plugin/logstash-filter-mongodb-guardium/src/test/java/com/ibm/guardium/mongodb/parsers/ParserTest.java index c5c4c780c..7285068a4 100644 --- a/filter-plugin/logstash-filter-mongodb-guardium/src/test/java/com/ibm/guardium/mongodb/parsers/ParserTest.java +++ b/filter-plugin/logstash-filter-mongodb-guardium/src/test/java/com/ibm/guardium/mongodb/parsers/ParserTest.java @@ -11,7 +11,15 @@ import com.google.gson.JsonParser; import com.ibm.guardium.mongodb.Parser; import com.ibm.guardium.mongodb.parsersbytype.AuthCheckParser; -import com.ibm.guardium.universalconnector.commons.structures.*; +import com.ibm.guardium.universalconnector.commons.structures.Accessor; +import com.ibm.guardium.universalconnector.commons.structures.Construct; +import com.ibm.guardium.universalconnector.commons.structures.Data; +import com.ibm.guardium.universalconnector.commons.structures.ExceptionRecord; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import com.ibm.guardium.universalconnector.commons.structures.Sentence; +import com.ibm.guardium.universalconnector.commons.structures.SentenceObject; +import com.ibm.guardium.universalconnector.commons.structures.SessionLocator; +import com.ibm.guardium.universalconnector.commons.structures.Time; import org.junit.Assert; import org.junit.Test; diff --git a/filter-plugin/logstash-filter-mssql-guardium/CHANGELOG.md b/filter-plugin/logstash-filter-mssql-guardium/CHANGELOG.md new file mode 100644 index 000000000..3e68f10ae --- /dev/null +++ b/filter-plugin/logstash-filter-mssql-guardium/CHANGELOG.md @@ -0,0 +1,4 @@ +## 0.0.1 +Ticket:https://ibm-datasecurity.atlassian.net/browse/GRD-114698 +Release: Q12026 +Description: Change server type from "MSSQL" to "MS SQL SERVER" \ No newline at end of file diff --git a/filter-plugin/logstash-filter-mssql-guardium/MssqlAWSOverJdbcPackage/MssqlOverJdbcPackage/filter.conf b/filter-plugin/logstash-filter-mssql-guardium/MssqlAWSOverJdbcPackage/MssqlOverJdbcPackage/filter.conf index 083e644c6..d6882f726 100644 --- a/filter-plugin/logstash-filter-mssql-guardium/MssqlAWSOverJdbcPackage/MssqlOverJdbcPackage/filter.conf +++ b/filter-plugin/logstash-filter-mssql-guardium/MssqlAWSOverJdbcPackage/MssqlOverJdbcPackage/filter.conf @@ -7,19 +7,22 @@ filter{ target => "mssqlEventTime" } - if [session_id]{ - mutate { add_field => { "[GuardRecord][sessionId]" => "%{session_id}" }} + mutate { add_field => { "[GuardRecord][sessionId]" => "" }} + + if [database_name]{ + mutate { add_field => { "[GuardRecord][dbName]" => "%{account_id}:%{database_name}" }} + } else{ - mutate { add_field => { "[GuardRecord][sessionId]" => "NA" }} + mutate { add_field => { "[GuardRecord][dbName]" => "N.A." }} } - + if [database_name]{ - mutate { add_field => { "[GuardRecord][dbName]" => "%{account_id}:%{database_name}" }} + mutate { add_field => { "[GuardRecord][accessor][serviceName]" => "%{account_id}:%{database_name}" }} } else{ - mutate { add_field => { "[GuardRecord][dbName]" => "NA" }} + mutate { add_field => { "[GuardRecord][accessor][serviceName]" => "N.A." }} } mutate { add_field => { "[GuardRecord][appUserName]" => "AWSService" }} @@ -44,11 +47,11 @@ filter{ mutate { add_field => { "[GuardRecord][accessor][dbUser]" => "%{server_principal_name}" }} } else{ - mutate { add_field => { "[GuardRecord][accessor][dbUser]" => "NA" }} + mutate { add_field => { "[GuardRecord][accessor][dbUser]" => "N.A." }} } mutate { add_field => { "[GuardRecord][accessor][dataType]" => "TEXT" }} mutate { add_field => { "[GuardRecord][accessor][osUser]" => "" }} - mutate { add_field => { "[GuardRecord][accessor][serverType]" => "MSSQL" }} + mutate { add_field => { "[GuardRecord][accessor][serverType]" => "MS SQL SERVER" }} mutate { add_field => { "[GuardRecord][accessor][commProtocol]" => "AwsApiCall" }} mutate { add_field => { "[GuardRecord][accessor][dbProtocol]" => "MS SQL SERVER" }} mutate { add_field => { "[GuardRecord][accessor][language]" => "MSSQL" }} # MSSQL is a Guardium internal code for MSSQL @@ -68,13 +71,12 @@ filter{ mutate { add_field => { "[GuardRecord][accessor][dbProtocolVersion]" => "" }} mutate { add_field => { "[GuardRecord][accessor][clientMac]" => ""} } mutate { add_field => { "[GuardRecord][accessor][serverDescription]" => "" } } - mutate { add_field => { "[GuardRecord][accessor][serviceName]" => ""} } if [application_name]{ mutate { add_field => { "[GuardRecord][accessor][sourceProgram]" => "%{application_name}"} } } else{ - mutate { add_field => { "[GuardRecord][accessor][sourceProgram]" => "NA"} } + mutate { add_field => { "[GuardRecord][accessor][sourceProgram]" => ""} } } mutate {add_field => { "isSuccess" => "%{succeeded}" }} @@ -89,7 +91,7 @@ filter{ else { mutate { add_field => { "[GuardRecord][exception][exceptionTypeId]" => "LOGIN_FAILED" }} mutate { add_field => { "[GuardRecord][exception][description]" => "%{statement}" }} - mutate { add_field => { "[GuardRecord][exception][sqlString]" => "NA" }} + mutate { add_field => { "[GuardRecord][exception][sqlString]" => "N.A." }} ruby { code => 'event.set("[GuardRecord][data]", nil)' } diff --git a/filter-plugin/logstash-filter-mssql-guardium/MssqlOnPremOverJdbcPackage/MssqlOverJdbcPackage/filter.conf b/filter-plugin/logstash-filter-mssql-guardium/MssqlOnPremOverJdbcPackage/MssqlOverJdbcPackage/filter.conf index b6fb64501..3bbead0d2 100644 --- a/filter-plugin/logstash-filter-mssql-guardium/MssqlOnPremOverJdbcPackage/MssqlOverJdbcPackage/filter.conf +++ b/filter-plugin/logstash-filter-mssql-guardium/MssqlOnPremOverJdbcPackage/MssqlOverJdbcPackage/filter.conf @@ -9,17 +9,18 @@ if [type]=="MSSQL" { target => "mssqlEventTime" } - if [session_id]{ - mutate { add_field => { "[GuardRecord][sessionId]" => "%{session_id}" }} + mutate { add_field => { "[GuardRecord][sessionId]" => "" }} + if [database_name]{ + mutate { add_field => { "[GuardRecord][dbName]" => "%{database_name}" }} } else{ - mutate { add_field => { "[GuardRecord][sessionId]" => "NA" }} + mutate { add_field => { "[GuardRecord][dbName]" => "N.A." }} } if [database_name]{ - mutate { add_field => { "[GuardRecord][dbName]" => "%{database_name}" }} + mutate { add_field => { "[GuardRecord][accessor][serviceName]" => "%{database_name}" }} } else{ - mutate { add_field => { "[GuardRecord][dbName]" => "NA" }} + mutate { add_field => { "[GuardRecord][accessor][serviceName]" => "N.A." }} } mutate { add_field => { "[GuardRecord][appUserName]" => "AWSService" }} json_encode { @@ -92,11 +93,11 @@ if [type]=="MSSQL" { mutate { add_field => { "[GuardRecord][accessor][dbUser]" => "%{server_principal_name}" }} } else{ - mutate { add_field => { "[GuardRecord][accessor][dbUser]" => "NA" }} + mutate { add_field => { "[GuardRecord][accessor][dbUser]" => "N.A." }} } mutate { add_field => { "[GuardRecord][accessor][dataType]" => "TEXT" }} mutate { add_field => { "[GuardRecord][accessor][osUser]" => "" }} - mutate { add_field => { "[GuardRecord][accessor][serverType]" => "MSSQL" }} + mutate { add_field => { "[GuardRecord][accessor][serverType]" => "MS SQL SERVER" }} mutate { add_field => { "[GuardRecord][accessor][commProtocol]" => "AwsApiCall" }} mutate { add_field => { "[GuardRecord][accessor][dbProtocol]" => "MS SQL SERVER" }} mutate { add_field => { "[GuardRecord][accessor][language]" => "MSSQL" }} # MSSQL is a Guardium internal code for MSSQL @@ -114,12 +115,12 @@ if [type]=="MSSQL" { mutate { add_field => { "[GuardRecord][accessor][dbProtocolVersion]" => "" }} mutate { add_field => { "[GuardRecord][accessor][clientMac]" => ""} } mutate { add_field => { "[GuardRecord][accessor][serverDescription]" => "" } } - mutate { add_field => { "[GuardRecord][accessor][serviceName]" => ""} } + if [application_name]{ mutate { add_field => { "[GuardRecord][accessor][sourceProgram]" => "%{application_name}"} } } else{ - mutate { add_field => { "[GuardRecord][accessor][sourceProgram]" => "NA"} } + mutate { add_field => { "[GuardRecord][accessor][sourceProgram]" => ""} } } json_encode { source => "[GuardRecord][accessor][dataType]" @@ -206,7 +207,7 @@ if [type]=="MSSQL" { else { mutate { add_field => { "[GuardRecord][exception][exceptionTypeId]" => "LOGIN_FAILED" }} mutate { add_field => { "[GuardRecord][exception][description]" => "%{statement}" }} - mutate { add_field => { "[GuardRecord][exception][sqlString]" => "NA" }} + mutate { add_field => { "[GuardRecord][exception][sqlString]" => "N.A." }} ruby { code => 'event.set("[GuardRecord][data]", nil)' } json_encode { source => "[GuardRecord][exception][exceptionTypeId]" diff --git a/filter-plugin/logstash-filter-mssql-guardium/MssqlOverJdbcPackage/awsMSSQL.conf b/filter-plugin/logstash-filter-mssql-guardium/MssqlOverJdbcPackage/awsMSSQL.conf index a3c627f21..e9b2222b7 100644 --- a/filter-plugin/logstash-filter-mssql-guardium/MssqlOverJdbcPackage/awsMSSQL.conf +++ b/filter-plugin/logstash-filter-mssql-guardium/MssqlOverJdbcPackage/awsMSSQL.conf @@ -37,7 +37,7 @@ filter{ mutate { add_field => { "[GuardRecord][sessionId]" => "%{session_id}" }} } else{ - mutate { add_field => { "[GuardRecord][sessionId]" => "NA" }} + mutate { add_field => { "[GuardRecord][sessionId]" => "" }} } if [database_name]{ @@ -45,7 +45,14 @@ filter{ } else{ - mutate { add_field => { "[GuardRecord][dbName]" => "NA" }} + mutate { add_field => { "[GuardRecord][dbName]" => "N.A." }} + } + + if [database_name]{ + mutate { add_field => { "[GuardRecord][accessor][serviceName]" => "%{database_name}" }} + } + else{ + mutate { add_field => { "[GuardRecord][accessor][serviceName]" => "N.A." }} } mutate { add_field => { "[GuardRecord][appUserName]" => "AWSService" }} @@ -70,11 +77,11 @@ filter{ mutate { add_field => { "[GuardRecord][accessor][dbUser]" => "%{server_principal_name}" }} } else{ - mutate { add_field => { "[GuardRecord][accessor][dbUser]" => "NA" }} + mutate { add_field => { "[GuardRecord][accessor][dbUser]" => "N.A." }} } mutate { add_field => { "[GuardRecord][accessor][dataType]" => "TEXT" }} mutate { add_field => { "[GuardRecord][accessor][osUser]" => "" }} - mutate { add_field => { "[GuardRecord][accessor][serverType]" => "MSSQL" }} + mutate { add_field => { "[GuardRecord][accessor][serverType]" => "MS SQL SERVER" }} mutate { add_field => { "[GuardRecord][accessor][commProtocol]" => "AwsApiCall" }} mutate { add_field => { "[GuardRecord][accessor][dbProtocol]" => "MS SQL SERVER" }} mutate { add_field => { "[GuardRecord][accessor][language]" => "MSSQL" }} # MSSQL is a Guardium internal code for MSSQL @@ -94,13 +101,12 @@ filter{ mutate { add_field => { "[GuardRecord][accessor][dbProtocolVersion]" => "" }} mutate { add_field => { "[GuardRecord][accessor][clientMac]" => ""} } mutate { add_field => { "[GuardRecord][accessor][serverDescription]" => "" } } - mutate { add_field => { "[GuardRecord][accessor][serviceName]" => ""} } if [application_name]{ mutate { add_field => { "[GuardRecord][accessor][sourceProgram]" => "%{application_name}"} } } else{ - mutate { add_field => { "[GuardRecord][accessor][sourceProgram]" => "NA"} } + mutate { add_field => { "[GuardRecord][accessor][sourceProgram]" => "N.A."} } } mutate {add_field => { "isSuccess" => "%{succeeded}" }} @@ -114,7 +120,7 @@ filter{ else { mutate { add_field => { "[GuardRecord][exception][exceptionTypeId]" => "LOGIN_FAILED" }} mutate { add_field => { "[GuardRecord][exception][description]" => "%{statement}" }} - mutate { add_field => { "[GuardRecord][exception][sqlString]" => "NA" }} + mutate { add_field => { "[GuardRecord][exception][sqlString]" => "N.A." }} ruby { code => 'event.set("[GuardRecord][data]", nil)' } diff --git a/filter-plugin/logstash-filter-mssql-guardium/MssqlOverJdbcPackage/gcpMSSQL.conf b/filter-plugin/logstash-filter-mssql-guardium/MssqlOverJdbcPackage/gcpMSSQL.conf index a4ed42f6d..6f9535f98 100644 --- a/filter-plugin/logstash-filter-mssql-guardium/MssqlOverJdbcPackage/gcpMSSQL.conf +++ b/filter-plugin/logstash-filter-mssql-guardium/MssqlOverJdbcPackage/gcpMSSQL.conf @@ -40,7 +40,7 @@ filter{ mutate { add_field => { "[GuardRecord][sessionId]" => "%{connection_id}" }} } else{ - mutate { add_field => { "[GuardRecord][sessionId]" => "NA" }} + mutate { add_field => { "[GuardRecord][sessionId]" => "" }} } if [database_name]{ @@ -50,7 +50,8 @@ filter{ } else{ - mutate { add_field => { "[GuardRecord][dbName]" => "NA" }} + mutate { add_field => { "[GuardRecord][dbName]" => "N.A." }} + mutate { add_field => { "[GuardRecord][accessor][serviceName]" => "N.A."} } } mutate { add_field => { "[GuardRecord][appUserName]" => "GCPService" }} @@ -80,11 +81,11 @@ filter{ mutate { add_field => { "[GuardRecord][accessor][dbUser]" => "%{server_principal_name}" }} } else{ - mutate { add_field => { "[GuardRecord][accessor][dbUser]" => "NA" }} + mutate { add_field => { "[GuardRecord][accessor][dbUser]" => "N.A." }} } mutate { add_field => { "[GuardRecord][accessor][dataType]" => "TEXT" }} mutate { add_field => { "[GuardRecord][accessor][osUser]" => "" }} - mutate { add_field => { "[GuardRecord][accessor][serverType]" => "MSSQL" }} + mutate { add_field => { "[GuardRecord][accessor][serverType]" => "MS SQL SERVER" }} mutate { add_field => { "[GuardRecord][accessor][commProtocol]" => "GCPApiCall" }} mutate { add_field => { "[GuardRecord][accessor][dbProtocol]" => "MS SQL SERVER" }} mutate { add_field => { "[GuardRecord][accessor][language]" => "MSSQL" }} # MSSQL is a Guardium internal code for MSSQL @@ -111,7 +112,7 @@ filter{ mutate { add_field => { "[GuardRecord][accessor][sourceProgram]" => "%{application_name}"} } } else{ - mutate { add_field => { "[GuardRecord][accessor][sourceProgram]" => "NA"} } + mutate { add_field => { "[GuardRecord][accessor][sourceProgram]" => "N.A."} } } @@ -121,7 +122,7 @@ filter{ if [isSuccess] == "true" { ruby { code => 'event.set("[GuardRecord][data][construct]", nil)' } mutate { add_field => { "[GuardRecord][data][originalSqlCommand]" => "%{statement}" }} - mutate { add_field => { "[GuardRecord][exception][exceptionTypeId]" => "NA" }} + mutate { add_field => { "[GuardRecord][exception][exceptionTypeId]" => "N.A." }} ruby { code => 'event.set("[GuardRecord][exception]", nil)' } @@ -130,7 +131,7 @@ filter{ else { mutate { add_field => { "[GuardRecord][exception][exceptionTypeId]" => "LOGIN_FAILED" }} mutate { add_field => { "[GuardRecord][exception][description]" => "%{statement}" }} - mutate { add_field => { "[GuardRecord][exception][sqlString]" => "NA" }} + mutate { add_field => { "[GuardRecord][exception][sqlString]" => "N.A." }} ruby { code => 'event.set("[GuardRecord][data]", nil)' } diff --git a/filter-plugin/logstash-filter-mysql-aws-guardium/AWSMySQLS3SQS.md b/filter-plugin/logstash-filter-mysql-aws-guardium/AWSMySQLS3SQS.md new file mode 100644 index 000000000..3a6fd47d3 --- /dev/null +++ b/filter-plugin/logstash-filter-mysql-aws-guardium/AWSMySQLS3SQS.md @@ -0,0 +1,33 @@ +# AWS MySQL S3SQS Setup + +Please follow the below link to setup S3SQS for AWS MySQL using kinesis data fire hose. +[S3SQSWithFirehose](../../input-plugin/logstash-input-s3sqs/S3SQSWithFirehose.md) guide for setup and configuration details. + +## Configuring the AWS MySQL filter in Guardium + +The Guardium universal connector is the Guardium entry point for native audit logs. The Guardium universal connector identifies and parses the received events, and converts them to a standard Guardium format. The output of the Guardium universal connector is forwarded to the Guardium sniffer on the collector for policy and auditing enforcements. + +### Before you begin + +* Configure the policies you need. For more information, see [Policies](/docs/#policies). +* You must have permissions for the S-Tap Management role. By default, the admin user is assigned the S-Tap Management role. +* Download the [logstash-filter-mysql_guardium_plugin_filter](https://github.com/IBM/universal-connectors/releases) plug-in. +* Download the [logstash-input-s3_sqs](https://github.com/IBM/universal-connectors/releases) plug-in. + +### Procedure + +1. On the collector, go to **Setup** > **Tools and Views** > **Configure Universal Connector**. +2. Enable the universal connector if it is disabled. +3. Click **Upload File** and select the offline [logstash-filter-mysql_guardium_plugin_filter](https://github.com/IBM/universal-connectors/releases) plug-in. After it is uploaded, click **OK**. This step is not necessary for Guardium Data Protection v11.0p490 or later, v11.0p540 or later, v12.0 or later. +4. Click **Upload File** and select the offline [logstash-input-s3_sqs](https://github.com/IBM/universal-connectors/releases) plug-in. After it is uploaded, click **OK**. This step is not necessary for Guardium Data Protection v11.0p490 or later, v11.0p540 or later, v12.0 or later. +5. Click the **Plus** sign to open the Connector Configuration dialog box. +6. In the **Connector name** field, enter a name. +7. Update the input section to add the details from the [MySQLOverS3SQS.conf](./MySQLOverS3SQS/MySQLOverS3SQS.conf) file's `input` section, omitting the keyword `input{` at the beginning and its corresponding `}` at the end. More details on how to configure the relevant input plugin can be found [here](../../input-plugin/logstash-input-s3sqs/README.md). +8. Update the filter section to add the details from the [MySQLOverS3SQS.conf](./MySQLOverS3SQS/MySQLOverS3SQS.conf) file's `filter` section, omitting the keyword `filter{` at the beginning and its corresponding `}` at the end. +9. Make sure that the `type` fields in the `input` and `filter` configuration sections align. This field must be unique for each connector added to the system. This is no longer required starting v12p20 and v12.1. +10. Click **Save**. Guardium validates the new connector and displays it in the Configure Universal Connector page. +11. When the offline plug-in is installed and the configuration is uploaded and saved in the Guardium machine, restart the universal connector by using the **Disable/Enable** button. + +## Limitations + +- When a login attempt fails, the MySQL audit log does not capture the database name. As a result, a new S-TAP entry may be created with the host displayed as `:unknown`. \ No newline at end of file diff --git a/filter-plugin/logstash-filter-mysql-aws-guardium/CHANGELOG.md b/filter-plugin/logstash-filter-mysql-aws-guardium/CHANGELOG.md new file mode 100644 index 000000000..29a17075a --- /dev/null +++ b/filter-plugin/logstash-filter-mysql-aws-guardium/CHANGELOG.md @@ -0,0 +1,9 @@ +# Changelog +Notable changes will be documented in this file. + +## [Unreleased] +## [1.0.1] +- Updated filter configuration to handle grok parsing error + +## [1.0.0] - 2025-08-25 + diff --git a/filter-plugin/logstash-filter-mysql-aws-guardium/Gemfile b/filter-plugin/logstash-filter-mysql-aws-guardium/Gemfile new file mode 100644 index 000000000..ea0d321ef --- /dev/null +++ b/filter-plugin/logstash-filter-mysql-aws-guardium/Gemfile @@ -0,0 +1,12 @@ +# AUTOGENERATED BY THE GRADLE SCRIPT. EDITS WILL BE OVERWRITTEN. +source 'https://rubygems.org' + +gemspec + +logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash" +use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1" + +if Dir.exist?(logstash_path) && use_logstash_source + gem 'logstash-core', :path => "#{logstash_path}/logstash-core" + gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api" +end diff --git a/filter-plugin/logstash-filter-mysql-aws-guardium/MySQLOverS3SQS/MySQLOverS3SQS.conf b/filter-plugin/logstash-filter-mysql-aws-guardium/MySQLOverS3SQS/MySQLOverS3SQS.conf new file mode 100644 index 000000000..c6c9ff625 --- /dev/null +++ b/filter-plugin/logstash-filter-mysql-aws-guardium/MySQLOverS3SQS/MySQLOverS3SQS.conf @@ -0,0 +1,118 @@ +input { + s3_sqs { + queue_url => "" # For i.e https://sqs..amazonaws.com// + region => "" + access_key_id => "" + secret_access_key => "" + role_arn => "" # Leave empty if not using role-based access + max_messages => + wait_time => # Must be >= 0 and <= 20, + polling_frequency => + type => "S3SQS_MYSQL" + } +} + +filter { +if [type] == "S3SQS_MYSQL" { + + # Step 1: Parse the JSON message from S3 event into [cloudwatch] + json { + source => "message" + target => "cloudwatch" + } + + # Step 2: Split logEvents array into separate events + split { + field => "[cloudwatch][logEvents]" + } + + # Step 3: Extract each log message and promote to top-level [message] + mutate { + rename => { "[cloudwatch][logEvents][message]" => "message" } + add_field => { + "logGroup" => "%{[cloudwatch][logGroup]}" + "logStream" => "%{[cloudwatch][logStream]}" + } + } + + # Step 4: Drop known noise events based on [message] content + if [message] =~ /(session\.transaction_read_only|information_schema\.TABLES|GLOBAL\.read_only|information_schema\.rds_events_threads_waits_current|SELECT\s+1|oscar_local_only_replica_host_status|replica_host_status)/ { + drop { } + } + + # Step 5: Parse message into fields using grok (use exact field names the plugin expects) + grok { + match => { + "message" => [ + # Format 1: QUERY with database and query + "%{NUMBER:timestamp_micro},%{DATA:db_instance},%{DATA:user},%{DATA:client_ip},%{NUMBER:thread_id},%{NUMBER:query_id},%{WORD:command},%{DATA:database},'%{GREEDYDATA:query}',%{NUMBER:status_code}", + + # Format 2: QUERY without database (empty field - double comma) + "%{NUMBER:timestamp_micro},%{DATA:db_instance},%{DATA:user},%{DATA:client_ip},%{NUMBER:thread_id},%{NUMBER:query_id},%{WORD:command},,'%{GREEDYDATA:query}',%{NUMBER:status_code}", + + # Format 3: READ/WRITE operations (ends with comma, no query) + "%{NUMBER:timestamp_micro},%{DATA:db_instance},%{DATA:user},%{DATA:client_ip},%{NUMBER:thread_id},%{NUMBER:query_id},%{WORD:command},%{DATA:database},%{DATA:table_name},", + + # Format 4: CONNECT/DISCONNECT (multiple empty fields) + "%{NUMBER:timestamp_micro},%{DATA:db_instance},%{DATA:user},%{DATA:client_ip},%{NUMBER:thread_id},%{NUMBER:query_id},%{WORD:command},,,,%{NUMBER:status_code}" + ] + } + remove_field => ["message"] + tag_on_failure => ["_grokparsefailure_custom"] + } + + # Step 5.1: Convert microsecond timestamp to datetime + date { + match => ["timestamp_micro", "UNIX_MS"] + target => "timestamp" + } + + # Step 5.2: Drop events that failed grok parsing + if "_grokparsefailure_custom" in [tags] { + drop { } + } + + # Step 5.3: Drop events from rdsadmin user (case-insensitive) + if [user] =~ /(?i)rdsadmin/ { + drop { } + } + + # Step 6: Set defaults for missing fields (CRITICAL: prevent NullPointerException) + if ![user] or [user] == "" { + mutate { replace => { "user" => "unknown" } } + } + + if ![command] or [command] == "" { + mutate { replace => { "command" => "UNKNOWN" } } + } + + if ![database] or [database] == "" { + mutate { replace => { "database" => "unknown" } } + } + + # CRITICAL: Set default status_code to prevent NullPointerException in Parser.java:83 + if ![status_code] { + mutate { add_field => { "status_code" => "0" } } + } + + # Step 7: Escape quotes in query field to prevent JSON malformation + if [query] { + ruby { + code => ' + query = event.get("query") + if query + query = query.gsub("\\", "\\\\\\\\").gsub("\"", "\\\\\"") + event.set("query", query) + end + ' + } + } + + # Step 8: Run Guardium plugin + mysql_guardium_plugin_filter{} + + # Optional: Keep only the GuardRecord field if desired + prune { + whitelist_names => [ "GuardRecord" ] + } +} diff --git a/filter-plugin/logstash-filter-mysql-aws-guardium/MysqlOverCloudwatchLogsPackage/MySQL/filter.conf b/filter-plugin/logstash-filter-mysql-aws-guardium/MysqlOverCloudwatchLogsPackage/MySQL/filter.conf index 1eca8cfb9..537f3a972 100644 --- a/filter-plugin/logstash-filter-mysql-aws-guardium/MysqlOverCloudwatchLogsPackage/MySQL/filter.conf +++ b/filter-plugin/logstash-filter-mysql-aws-guardium/MysqlOverCloudwatchLogsPackage/MySQL/filter.conf @@ -1,32 +1,51 @@ filter { if [type] == "aws_mysql" { + mutate { + gsub => ["message", "\\n", " "] + gsub => ["message", "\\t", ""] + } #QUERIES if "QUERY" in [message] { grok { match => { - "message" => "(?\d{4})(?\d{2})(?

\d{2})\s%{TIME:time}(,)(ip-\d{1,3}-\d{1,3}-\d{1,3}-\d{1,3},)%{USER:db_user}(,)%{IPV4:clientIp}(,)%{NUMBER}(,)%{NUMBER}(,)%{WORD:operation}(,)([']?)([`]?)(%{WORD:dbName})?([']?)([`]?)(,)(')*([']?)(?.*)([']+,[0-9]+)" + "message" => "(?\d{4})(?\d{2})(?
\d{2})\s%{TIME:time}(,)(?ip-\d{1,3}-\d{1,3}-\d{1,3}-\d{1,3})(,)%{USER:db_user}(,)%{NOTSPACE:clientIp}(,)%{NUMBER}(,)%{NUMBER}(,)%{WORD:operation}(,)([']?)([`]?)(%{WORD:dbName})?([']?)([`]?)(,)'(?.*)'(,.*)?$" } } } - #CONNECT / DISCONNECT - else if "DISCONNECT" in [message] or "CONNECT" in [message] { + #FAILED_CONNECT + else if "FAILED_CONNECT" in [message] { grok { match => { - "message" => "([0-9]+\s)%{TIME}(,)(ip-\d{1,3}-\d{1,3}-\d{1,3}-\d{1,3},)(%{USER:db_user})?(,)%{IPV4:clientIp}(,)%{NUMBER}(,)%{NUMBER}(,)%{WORD:operation}(,)([']?)([`]?)(%{WORD:dbName})?([']?)([`]?)(,,0)(,)?([^,]*)" - } - } - } - # "Access denied" - else { - grok { - match => { - "message" => "%{TIMESTAMP_ISO8601}(\s)+(?%{NUMBER}(\s)([^\s]+)(\s)+([^']*)(')%{WORD:db_user}%{GREEDYDATA})" + "message" => "(?\d{4})(?\d{2})(?
\d{2})\s%{TIME:time}(,)(?ip-\d{1,3}-\d{1,3}-\d{1,3}-\d{1,3})(,)(%{USER:db_user})?(,)%{NOTSPACE:clientIp}(,)%{NUMBER}(,)%{NUMBER}(,)%{WORD:operation}(,)(,)(,)%{NUMBER}(,)%{WORD}" } } } + + # Drop logs from rdsadmin user querying mysql database + if [db_user] == "rdsadmin" and [dbName] == "mysql" and "_grokparsefailure" not in [tags] { + drop {} + } + mutate {add_field => { "logGroup" => "%{[cloudwatch_logs][log_group]}" } } grok { match => { "logGroup" => "(?[^\/]*)\/(?[^\/]*)\/(?[^\/]*)\/(?[^\/]*)\/(?[^\/]*)\/(?[^\/]*)" } } + + # Extract server and client IPs + grok { + match => { "clientIp" => ".*-(?\d{1,3}-\d{1,3}-\d{1,3}-\d{1,3})\..*" } + tag_on_failure => [] + } + + # Store serverHostname in a temporary field before transformation mutate { + add_field => { "serverIpTemp" => "%{serverHostname}" } + } + + mutate { + gsub => [ + "serverIpTemp", "ip-", "", + "serverIpTemp", "-", ".", + "clientIpNumeric", "-", "." + ] add_field => { "sessionLocator" => {} "time" => {} @@ -34,19 +53,20 @@ filter { "data" => {} "ExceptionRecord" => {} "GuardRecord" => {} - "serverIp" => "0.0.0.0" - "serverPort" => "-1" - "clientPort" => "-1" + "[GuardRecord][sessionLocator][serverIp]" => "%{serverIpTemp}" + "[GuardRecord][sessionLocator][serverPort]" => "-1" + "[GuardRecord][sessionLocator][clientPort]" => "-1" + "[GuardRecord][sessionLocator][clientIp]" => "%{[clientIpNumeric]}" } + rename => { "db_user" => "[GuardRecord][accessor][dbUser]" } + } + + # Fallback to original hostname if IP extraction failed + if ![GuardRecord][sessionLocator][clientIp] or [GuardRecord][sessionLocator][clientIp] == "%{[clientIpNumeric]}" { + mutate { replace => { "[GuardRecord][sessionLocator][clientIp]" => "%{clientIp}" } } } + mutate { - rename => { - "serverIp" => "[GuardRecord][sessionLocator][serverIp]" - "serverPort" => "[GuardRecord][sessionLocator][serverPort]" - "clientPort" => "[GuardRecord][sessionLocator][clientPort]" - "clientIp" => "[GuardRecord][sessionLocator][clientIp]" - "db_user" => "[GuardRecord][accessor][dbUser]" - } add_field => { "[GuardRecord][sessionId]" => "%{[cloudwatch_logs][event_id]}" "[GuardRecord][accessor][dbProtocol]" => "MYSQL" @@ -69,6 +89,12 @@ filter { "[GuardRecord][sessionLocator][isIpv6]" => "false" } } + + # Ensure dbUser is not empty + if ![GuardRecord][accessor][dbUser] or [GuardRecord][accessor][dbUser] == "" { + mutate { replace => { "[GuardRecord][accessor][dbUser]" => "N.A." } } + } + mutate { add_field => { "myTimestamp" => "%{yy}-%{mm}-%{dd}T%{time}Z" } } @@ -80,21 +106,57 @@ filter { ruby { code => "event.set('[GuardRecord][time][timstamp]', event.get('finalTime').to_i * 1000)" } if [dbName] { - mutate {add_field => { "[GuardRecord][dbName]" => "%{account_id}:%{instance}:%{dbName}" } } + mutate { + add_field => { "[GuardRecord][dbName]" => "%{account_id}:%{instance}:%{dbName}" } + replace => { "[GuardRecord][accessor][serviceName]" => "%{account_id}:%{instance}:%{dbName}" } + } } else { - mutate {add_field => { "[GuardRecord][dbName]" => "%{account_id}:%{instance}" } } + mutate { + add_field => { "[GuardRecord][dbName]" => "%{account_id}:%{instance}" } + replace => { "[GuardRecord][accessor][serviceName]" => "%{account_id}:%{instance}" } + } } #Exceptions - if "Access denied" in [message] { + if "FAILED_CONNECT" in [message] { mutate { add_field => { "[GuardRecord][exception][exceptionTypeId]" => "LOGIN_FAILED" "[GuardRecord][exception][sqlString]" => "" - "[GuardRecord][exception][description]" => "%{data}" + "[GuardRecord][exception][description]" => "FAILED_CONNECT" } } } else if "QUERY" in [message] { + # Check if SQL starts with comment containing ApplicationName + if [originalSqlCommand] =~ /^\/\*\s*ApplicationName=/ { + # Extract ApplicationName value and clean SQL using dissect + dissect { + mapping => { + "originalSqlCommand" => "/* ApplicationName=%{[GuardRecord][accessor][sourceProgram]} */ %{cleanSql}" + } + tag_on_failure => ["_dissect_failure"] + } + # Update originalSqlCommand with clean SQL + if [cleanSql] and [cleanSql] != "" { + mutate { + replace => { "originalSqlCommand" => "%{cleanSql}" } + remove_field => ["cleanSql"] + } + } + # Fallback if dissect failed + if "_dissect_failure" in [tags] { + mutate { + add_field => { "[GuardRecord][accessor][sourceProgram]" => "N.A." } + remove_tag => ["_dissect_failure"] + } + } + } else { + # No ApplicationName comment - set to N.A. + mutate { + add_field => { "[GuardRecord][accessor][sourceProgram]" => "N.A." } + } + } + mutate { gsub => ["originalSqlCommand", "'", "\""] gsub => ["originalSqlCommand", "\\n", " "] @@ -105,25 +167,18 @@ filter { "[GuardRecord][data][originalSqlCommand]" => "%{originalSqlCommand}" } } - } else if "DISCONNECT" in [message] or "CONNECT" in [message] { - mutate { - add_field => { - "[GuardRecord][data][originalSqlCommand]" => "%{operation}" - } - } - } - else { + }else { drop {} } if "_grokparsefailure" in [tags] { drop {} } mutate { - remove_field => [ "message", "object", "@version", "cloudwatch_logs", "operation", "type", "data","account_id", "data13", "instance", "dbName", "data14", "logGroup", "data15", "data12", "yy", "mm", "dd" , "myTimestamp" , "finalTime" , "@timestamp" ] - } + remove_field => [ "message", "object", "@version", "cloudwatch_logs", "operation", "type", "data","account_id", "data13", "instance", "dbName", "data14", "logGroup", "data15", "data12", "yy", "mm", "dd" , "myTimestamp" , "finalTime" , "@timestamp", "serverHostname", "serverIpTemp", "clientIp", "clientIpNumeric" ] + } json_encode { source => "[GuardRecord]" target => "[GuardRecord]" } - } + } } \ No newline at end of file diff --git a/filter-plugin/logstash-filter-mysql-aws-guardium/README.md b/filter-plugin/logstash-filter-mysql-aws-guardium/README.md index 1ce3dc5c8..5a705f744 100644 --- a/filter-plugin/logstash-filter-mysql-aws-guardium/README.md +++ b/filter-plugin/logstash-filter-mysql-aws-guardium/README.md @@ -71,7 +71,7 @@ To add the MariaDB plug-in to a MySQL instance, follow the instructions describe * Currently, this plugin supports only audit logs and "Login Failed" error logs. * Guardium Data Protection requires installation of the [json_encode](https://www.elastic.co/guide/en/logstash-versioned-plugins/current/v3.0.3-plugins-filters-json_encode.html) filter plug-in. * The `use` statement does not display the account ID in the 'Database Name' column on the reports page. -* +* The RDS MYSQL over CloudWatch plug-in does not classify SQL errors as error-level events in the database logs. ## Configuring the AWS MySQL Guardium Logstash filters in Guardium Data Security Center To configure this plug-in for Guardium Data Security Center, follow [this guide.](/docs/Guardium%20Insights/3.2.x/UC_Configuration_GI.md) For the input configuration step, refer to the [CloudWatch_logs section](/docs/Guardium%20Insights/3.2.x/UC_Configuration_GI.md#configuring-a-CloudWatch-input-plug-in). diff --git a/filter-plugin/logstash-filter-mysql-aws-guardium/VERSION b/filter-plugin/logstash-filter-mysql-aws-guardium/VERSION new file mode 100644 index 000000000..3eefcb9dd --- /dev/null +++ b/filter-plugin/logstash-filter-mysql-aws-guardium/VERSION @@ -0,0 +1 @@ +1.0.0 diff --git a/filter-plugin/logstash-filter-mysql-aws-guardium/build.gradle b/filter-plugin/logstash-filter-mysql-aws-guardium/build.gradle new file mode 100644 index 000000000..dcf73b77c --- /dev/null +++ b/filter-plugin/logstash-filter-mysql-aws-guardium/build.gradle @@ -0,0 +1,209 @@ +import java.nio.file.Files +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING + +apply plugin: 'java' + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + + dependencies { + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' + classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:4.0.1" + classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' + } +} + +def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + +apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" + +apply plugin: 'com.github.johnrengelman.shadow' +apply plugin: 'jacoco' +apply plugin: 'org.barfuin.gradle.jacocolog' + +// Plugin Info +group = 'com.ibm.guardium.mysql' +version = file("VERSION").text.trim() +description = "mysql-Guardium filter plugin" +pluginInfo.licenses = ['Apache-2.0'] +pluginInfo.longDescription = "This gem is a Logstash mysql filter plugin required to be installed as part of IBM Security Guardium, Guardium Universal Connector configuration. This gem is not a stand-alone program." +pluginInfo.authors = ['IBM'] +pluginInfo.email = [''] +pluginInfo.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html" +pluginInfo.pluginType = "filter" +pluginInfo.pluginClass = "MySQLGuardiumPluginFilter" +pluginInfo.pluginName = "mysql_guardium_plugin_filter" + +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 + +// Code coverage +def jacocoVersion = '0.8.4' +def minimumCoverageStr = System.getenv("MINIMUM_COVERAGE") ?: "50.0%" +if (minimumCoverageStr.endsWith("%")) { + minimumCoverageStr = minimumCoverageStr.substring(0, minimumCoverageStr.length() - 1) +} +def minimumCoverage = Float.valueOf(minimumCoverageStr) / 100 + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + dependencies { + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' + classpath 'org.barfuin.gradle.jacocolog:gradle-jacoco-log:4.0.1' + } +} + +repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } +} + +dependencies { + // Runtime dependencies + implementation 'commons-beanutils:commons-beanutils:1.11.0' + implementation 'commons-validator:commons-validator:1.7' + implementation 'org.apache.logging.log4j:log4j-core:2.17.1' + implementation 'org.apache.commons:commons-text:1.10.0' + implementation 'com.google.code.gson:gson:2.8.9' + implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "build/libs/logstash-core*.jar") + implementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "guardium-universalconnector-commons*.jar") + + // Test dependencies + testImplementation 'org.junit.jupiter:junit-jupiter:5.10.0' + testImplementation 'org.mockito:mockito-core:5.12.0' + testRuntimeOnly 'org.junit.platform:junit-platform-launcher:1.10.0' + testImplementation 'org.jruby:jruby-complete:9.2.7.0' + testImplementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "guardium-universalconnector-commons*.jar") +} + +test { + useJUnitPlatform() + testLogging { + events "passed", "skipped", "failed" + exceptionFormat "full" + showStandardStreams = true + } + finalizedBy jacocoTestReport +} + +jacoco { + toolVersion = "${jacocoVersion}" +} + +jacocoTestReport { + reports { + html.required = true + xml.required = true + csv.required = true + html.destination file("${buildDir}/reports/jacoco") + csv.destination file("${buildDir}/reports/jacoco/all.csv") + } + executionData.from fileTree(dir: "${buildDir}/jacoco/", includes: ['**/*.exec']) + afterEvaluate { + classDirectories.setFrom(files(classDirectories.files.collect { + fileTree(dir: it, exclude: []) + })) + } + doLast { + println "Code coverage report -> file://${buildDir}/reports/jacoco/index.html" + } +} + +jacocoTestCoverageVerification { + violationRules { + rule { + limit { + minimum = minimumCoverage + } + } + } + executionData.from fileTree(dir: "${buildDir}/jacoco/", includes: ['**/*.exec']) + afterEvaluate { + classDirectories.setFrom(files(classDirectories.files.collect { + fileTree(dir: it, exclude: []) + })) + } +} + +check.dependsOn jacocoTestCoverageVerification, jacocoTestReport + +shadowJar { + archiveClassifier = null +} + +// Custom Tasks +tasks.register("vendor") { + dependsOn shadowJar + doLast { + String vendorPathPrefix = "vendor/jar-dependencies" + String projectGroupPath = project.group.replaceAll('\\.', '/') + File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") + projectJarFile.mkdirs() + Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) + validatePluginJar(projectJarFile, project.group) + } +} + +tasks.register("generateRubySupportFiles") { + doLast { + generateRubySupportFilesForPlugin(project.description, project.group, version) + } +} + +tasks.register("removeObsoleteJars") { + doLast { + new FileNameFinder().getFileNames( + projectDir.toString(), + "vendor/**/" + pluginInfo.pluginFullName() + "*.jar", + "vendor/**/" + pluginInfo.pluginFullName() + "-" + version + ".jar").each { f -> + delete f + } + } +} + +tasks.register("gem") { + dependsOn = [downloadAndInstallJRuby, removeObsoleteJars, vendor, generateRubySupportFiles] + doLast { + buildGem(projectDir, buildDir, pluginInfo.pluginFullName() + ".gemspec") + } +} + +clean { + delete "${projectDir}/Gemfile" + delete "${projectDir}/${pluginInfo.pluginFullName()}.gemspec" + delete "${projectDir}/lib/" + delete "${projectDir}/vendor/" + new FileNameFinder().getFileNames(projectDir.toString(), pluginInfo.pluginFullName() + "-*.*.*.gem").each { + delete it + } +} + +tasks.withType(JavaCompile) { + options.encoding = 'UTF-8' +} \ No newline at end of file diff --git a/filter-plugin/logstash-filter-mysql-aws-guardium/gradle/wrapper/gradle-wrapper.jar b/filter-plugin/logstash-filter-mysql-aws-guardium/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 000000000..7454180f2 Binary files /dev/null and b/filter-plugin/logstash-filter-mysql-aws-guardium/gradle/wrapper/gradle-wrapper.jar differ diff --git a/filter-plugin/logstash-filter-mysql-aws-guardium/gradle/wrapper/gradle-wrapper.properties b/filter-plugin/logstash-filter-mysql-aws-guardium/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 000000000..81aa1c044 --- /dev/null +++ b/filter-plugin/logstash-filter-mysql-aws-guardium/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,5 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/filter-plugin/logstash-filter-mysql-aws-guardium/gradlew b/filter-plugin/logstash-filter-mysql-aws-guardium/gradlew new file mode 100755 index 000000000..cccdd3d51 --- /dev/null +++ b/filter-plugin/logstash-filter-mysql-aws-guardium/gradlew @@ -0,0 +1,172 @@ +#!/usr/bin/env sh + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS="" + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin, switch paths to Windows format before running java +if $cygwin ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=$((i+1)) + done + case $i in + (0) set -- ;; + (1) set -- "$args0" ;; + (2) set -- "$args0" "$args1" ;; + (3) set -- "$args0" "$args1" "$args2" ;; + (4) set -- "$args0" "$args1" "$args2" "$args3" ;; + (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=$(save "$@") + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong +if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then + cd "$(dirname "$0")" +fi + +exec "$JAVACMD" "$@" diff --git a/filter-plugin/logstash-filter-mysql-aws-guardium/gradlew.bat b/filter-plugin/logstash-filter-mysql-aws-guardium/gradlew.bat new file mode 100644 index 000000000..f9553162f --- /dev/null +++ b/filter-plugin/logstash-filter-mysql-aws-guardium/gradlew.bat @@ -0,0 +1,84 @@ +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS= + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto init + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto init + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:init +@rem Get command-line arguments, handling Windows variants + +if not "%OS%" == "Windows_NT" goto win9xME_args + +:win9xME_args +@rem Slurp the command line arguments. +set CMD_LINE_ARGS= +set _SKIP=2 + +:win9xME_args_slurp +if "x%~1" == "x" goto execute + +set CMD_LINE_ARGS=%* + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/filter-plugin/logstash-filter-mysql-aws-guardium/gradlew.unix b/filter-plugin/logstash-filter-mysql-aws-guardium/gradlew.unix new file mode 100644 index 000000000..cccdd3d51 --- /dev/null +++ b/filter-plugin/logstash-filter-mysql-aws-guardium/gradlew.unix @@ -0,0 +1,172 @@ +#!/usr/bin/env sh + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS="" + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin, switch paths to Windows format before running java +if $cygwin ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=$((i+1)) + done + case $i in + (0) set -- ;; + (1) set -- "$args0" ;; + (2) set -- "$args0" "$args1" ;; + (3) set -- "$args0" "$args1" "$args2" ;; + (4) set -- "$args0" "$args1" "$args2" "$args3" ;; + (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=$(save "$@") + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong +if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then + cd "$(dirname "$0")" +fi + +exec "$JAVACMD" "$@" diff --git a/filter-plugin/logstash-filter-mysql-aws-guardium/mainREADME.md b/filter-plugin/logstash-filter-mysql-aws-guardium/mainREADME.md new file mode 100644 index 000000000..82e9a3eaf --- /dev/null +++ b/filter-plugin/logstash-filter-mysql-aws-guardium/mainREADME.md @@ -0,0 +1,13 @@ +# AWS MySQL Universal Connector + +## Follow this link to set up and use AWS MySQL Universal Connector over CloudWatch Logstash Plugin + +[AwsMySqlOverCloudwatch](./README.md) + + +## Follow this link to set up and use RDS MySQL Universal Connector over CloudWatch Connect + +[RDSMySqlOverConnectCloudwatch](../../docs/KafkaBasedUCs/RDSMySqlCloudwatchKafkaConnect.md) + + + diff --git a/filter-plugin/logstash-filter-mysql-aws-guardium/mysqlCloudwatch.conf b/filter-plugin/logstash-filter-mysql-aws-guardium/mysqlCloudwatch.conf index 3acd06789..f620c1362 100644 --- a/filter-plugin/logstash-filter-mysql-aws-guardium/mysqlCloudwatch.conf +++ b/filter-plugin/logstash-filter-mysql-aws-guardium/mysqlCloudwatch.conf @@ -18,33 +18,52 @@ input { filter { if [type] == "aws_mysql" { + mutate { + gsub => ["message", "\\n", " "] + gsub => ["message", "\\t", ""] + } #QUERIES if "QUERY" in [message] { grok { match => { - "message" => "(?\d{4})(?\d{2})(?
\d{2})\s%{TIME:time}(,)(ip-\d{1,3}-\d{1,3}-\d{1,3}-\d{1,3},)%{USER:db_user}(,)%{IPV4:clientIp}(,)%{NUMBER}(,)%{NUMBER}(,)%{WORD:operation}(,)([']?)([`]?)(%{WORD:dbName})?([']?)([`]?)(,)(')*([']?)(?.*)([']+,[0-9]+)" - } - } - } - #CONNECT / DISCONNECT - else if "DISCONNECT" in [message] or "CONNECT" in [message] { - grok { - match => { - "message" => "([0-9]+\s)%{TIME}(,)(ip-\d{1,3}-\d{1,3}-\d{1,3}-\d{1,3},)(%{USER:db_user})?(,)%{IPV4:clientIp}(,)%{NUMBER}(,)%{NUMBER}(,)%{WORD:operation}(,)([']?)([`]?)(%{WORD:dbName})?([']?)([`]?)(,,0)(,)?([^,]*)" + "message" => "(?\d{4})(?\d{2})(?
\d{2})\s%{TIME:time}(,)(?ip-\d{1,3}-\d{1,3}-\d{1,3}-\d{1,3})(,)%{USER:db_user}(,)%{NOTSPACE:clientIp}(,)%{NUMBER}(,)%{NUMBER}(,)%{WORD:operation}(,)([']?)([`]?)(%{WORD:dbName})?([']?)([`]?)(,)'(?.*)'(,.*)?$" } } } - # "Access denied" - else { + #FAILED_CONNECT + else if "FAILED_CONNECT" in [message] { grok { match => { - "message" => "%{TIMESTAMP_ISO8601}(\s)+(?%{NUMBER}(\s)([^\s]+)(\s)+([^']*)(')%{WORD:db_user}%{GREEDYDATA})" + "message" => "(?\d{4})(?\d{2})(?
\d{2})\s%{TIME:time}(,)(?ip-\d{1,3}-\d{1,3}-\d{1,3}-\d{1,3})(,)(%{USER:db_user})?(,)%{NOTSPACE:clientIp}(,)%{NUMBER}(,)%{NUMBER}(,)%{WORD:operation}(,)(,)(,)%{NUMBER}(,)%{WORD}" } } } + + # Drop logs from rdsadmin user querying mysql database + if [db_user] == "rdsadmin" and [dbName] == "mysql" and "_grokparsefailure" not in [tags] { + drop {} + } + mutate {add_field => { "logGroup" => "%{[cloudwatch_logs][log_group]}" } } grok { match => { "logGroup" => "(?[^\/]*)\/(?[^\/]*)\/(?[^\/]*)\/(?[^\/]*)\/(?[^\/]*)\/(?[^\/]*)" } } + + # Extract server and client IPs + grok { + match => { "clientIp" => ".*-(?\d{1,3}-\d{1,3}-\d{1,3}-\d{1,3})\..*" } + tag_on_failure => [] + } + + # Store serverHostname in a temporary field before transformation + mutate { + add_field => { "serverIpTemp" => "%{serverHostname}" } + } + mutate { + gsub => [ + "serverIpTemp", "ip-", "", + "serverIpTemp", "-", ".", + "clientIpNumeric", "-", "." + ] add_field => { "sessionLocator" => {} "time" => {} @@ -52,19 +71,20 @@ filter { "data" => {} "ExceptionRecord" => {} "GuardRecord" => {} - "serverIp" => "0.0.0.0" - "serverPort" => "-1" - "clientPort" => "-1" + "[GuardRecord][sessionLocator][serverIp]" => "%{serverIpTemp}" + "[GuardRecord][sessionLocator][serverPort]" => "-1" + "[GuardRecord][sessionLocator][clientPort]" => "-1" + "[GuardRecord][sessionLocator][clientIp]" => "%{[clientIpNumeric]}" } + rename => { "db_user" => "[GuardRecord][accessor][dbUser]" } } + + # Fallback to original hostname if IP extraction failed + if ![GuardRecord][sessionLocator][clientIp] or [GuardRecord][sessionLocator][clientIp] == "%{[clientIpNumeric]}" { + mutate { replace => { "[GuardRecord][sessionLocator][clientIp]" => "%{clientIp}" } } + } + mutate { - rename => { - "serverIp" => "[GuardRecord][sessionLocator][serverIp]" - "serverPort" => "[GuardRecord][sessionLocator][serverPort]" - "clientPort" => "[GuardRecord][sessionLocator][clientPort]" - "clientIp" => "[GuardRecord][sessionLocator][clientIp]" - "db_user" => "[GuardRecord][accessor][dbUser]" - } add_field => { "[GuardRecord][sessionId]" => "%{[cloudwatch_logs][event_id]}" "[GuardRecord][accessor][dbProtocol]" => "MYSQL" @@ -87,6 +107,12 @@ filter { "[GuardRecord][sessionLocator][isIpv6]" => "false" } } + + # Ensure dbUser is not empty + if ![GuardRecord][accessor][dbUser] or [GuardRecord][accessor][dbUser] == "" { + mutate { replace => { "[GuardRecord][accessor][dbUser]" => "N.A." } } + } + mutate { add_field => { "myTimestamp" => "%{yy}-%{mm}-%{dd}T%{time}Z" } } @@ -98,21 +124,57 @@ filter { ruby { code => "event.set('[GuardRecord][time][timstamp]', event.get('finalTime').to_i * 1000)" } if [dbName] { - mutate {add_field => { "[GuardRecord][dbName]" => "%{account_id}:%{instance}:%{dbName}" } } + mutate { + add_field => { "[GuardRecord][dbName]" => "%{account_id}:%{instance}:%{dbName}" } + replace => { "[GuardRecord][accessor][serviceName]" => "%{account_id}:%{instance}:%{dbName}" } + } } else { - mutate {add_field => { "[GuardRecord][dbName]" => "%{account_id}:%{instance}" } } + mutate { + add_field => { "[GuardRecord][dbName]" => "%{account_id}:%{instance}" } + replace => { "[GuardRecord][accessor][serviceName]" => "%{account_id}:%{instance}" } + } } #Exceptions - if "Access denied" in [message] { + if "FAILED_CONNECT" in [message] { mutate { add_field => { "[GuardRecord][exception][exceptionTypeId]" => "LOGIN_FAILED" "[GuardRecord][exception][sqlString]" => "" - "[GuardRecord][exception][description]" => "%{data}" + "[GuardRecord][exception][description]" => "FAILED_CONNECT" } } } else if "QUERY" in [message] { + # Check if SQL starts with comment containing ApplicationName + if [originalSqlCommand] =~ /^\/\*\s*ApplicationName=/ { + # Extract ApplicationName value and clean SQL using dissect + dissect { + mapping => { + "originalSqlCommand" => "/* ApplicationName=%{[GuardRecord][accessor][sourceProgram]} */ %{cleanSql}" + } + tag_on_failure => ["_dissect_failure"] + } + # Update originalSqlCommand with clean SQL + if [cleanSql] and [cleanSql] != "" { + mutate { + replace => { "originalSqlCommand" => "%{cleanSql}" } + remove_field => ["cleanSql"] + } + } + # Fallback if dissect failed + if "_dissect_failure" in [tags] { + mutate { + add_field => { "[GuardRecord][accessor][sourceProgram]" => "N.A." } + remove_tag => ["_dissect_failure"] + } + } + } else { + # No ApplicationName comment - set to N.A. + mutate { + add_field => { "[GuardRecord][accessor][sourceProgram]" => "N.A." } + } + } + mutate { gsub => ["originalSqlCommand", "'", "\""] gsub => ["originalSqlCommand", "\\n", " "] @@ -123,25 +185,18 @@ filter { "[GuardRecord][data][originalSqlCommand]" => "%{originalSqlCommand}" } } - } else if "DISCONNECT" in [message] or "CONNECT" in [message] { - mutate { - add_field => { - "[GuardRecord][data][originalSqlCommand]" => "%{operation}" - } - } - } - else { + }else { drop {} } if "_grokparsefailure" in [tags] { drop {} } mutate { - remove_field => [ "message", "object", "@version", "cloudwatch_logs", "operation", "type", "data","account_id", "data13", "instance", "dbName", "data14", "logGroup", "data15", "data12", "yy", "mm", "dd" , "myTimestamp" , "finalTime" , "@timestamp" ] - } + remove_field => [ "message", "object", "@version", "cloudwatch_logs", "operation", "type", "data","account_id", "data13", "instance", "dbName", "data14", "logGroup", "data15", "data12", "yy", "mm", "dd" , "myTimestamp" , "finalTime" , "@timestamp", "serverHostname", "serverIpTemp", "clientIp", "clientIpNumeric" ] + } json_encode { source => "[GuardRecord]" target => "[GuardRecord]" } - } -} + } +} \ No newline at end of file diff --git a/filter-plugin/logstash-filter-mysql-aws-guardium/src/main/java/com/ibm/guardium/mysql/Constants.java b/filter-plugin/logstash-filter-mysql-aws-guardium/src/main/java/com/ibm/guardium/mysql/Constants.java new file mode 100644 index 000000000..5b760ae6d --- /dev/null +++ b/filter-plugin/logstash-filter-mysql-aws-guardium/src/main/java/com/ibm/guardium/mysql/Constants.java @@ -0,0 +1,126 @@ +package com.ibm.guardium.mysql; + + +public interface Constants { + + + String NOT_AVAILABLE = "NA"; + + + + String LOGSTASH_TAG_SKIP_NOT_PROGRESS = "LOGSTASH_TAG_SKIP_NOT_PROGRESS"; + + + + String SERVER_IP = "server_ip"; + + + + String SERVER_PORT = "portNum"; + + + + + String SERVER_HOST = "host"; //db machine + + String CLIENT_HOST = "Client_Name"; + + String CLIENT_SESSION_ID = "clientSessionId"; + + + + + + + + + + String SOURCE_PROGRAM = "SOURCE_PROGRAM"; + + String USER_ID = "user"; + + + + + + String EVENT_CONTEXT = "eventContext"; + + + + String DATE_TIME_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"; + + String LOGIN_FAILED = "LOGIN_FAILED"; + + + + String SQL_TEXT = "sql_text"; + + + + String EVENT_NOW_MYSQL = "MySQL data {} "; + + + String DATABASE_NAME = "database"; + + String DATABASE_USER_NAME = "user"; + + + String TIMESTAMP = "timestamp"; + + int MIN_OFFSET_FROM_GMT = 0; + + int MIN_DST = 0; + + String CLIENT_IP = "client_ip"; + + int DEFAULT_PORT = -1; + + String DEFAULT_IP = "0.0.0.0"; + + String UNKNOWN_STRING = ""; + + String SERVER_TYPE = "MySQL"; + + String ACCOUNT_ID = "account_id"; + + String DB_PROTOCOL = "MYSQL"; + + String TEXT = "TEXT"; + + String QUERY = "query"; + + String QUERY_CONST = "QUERY"; + + String STATUS_CODE = "status_code"; + + String COMMAND_TYPE = "command_type"; + + String FAILED_CONNECT = "FAILED_CONNECT"; + + String ACTION = "action"; + + String DESCRIPTION_MESSAGE = "The Query has failed with Error code "; + + String CONNECTION_FAILED_DESCRIPTION_MESSAGE = "Login Connection request failed with Error code "; + + String GUARD_RECORD = "GuardRecord {}"; + + String EVENT_DATA = "Event Data {}"; + + String RECORDS = "records"; + + String TIMESTAMP_ERROR = "Invalid timestamp format: {}"; + + String COMMAND_UNKNOWN = "UNKNOWN"; + + String COMMAND = "command"; + + String RDS_ADMIN = "rdsadmin"; + + String LOG_GROUP = "logGroup"; + + String MESSAGE = "message"; + + String SQL_ERROR = "SQL_ERROR"; +} + diff --git a/filter-plugin/logstash-filter-mysql-aws-guardium/src/main/java/com/ibm/guardium/mysql/MySQLGuardiumPluginFilter.java b/filter-plugin/logstash-filter-mysql-aws-guardium/src/main/java/com/ibm/guardium/mysql/MySQLGuardiumPluginFilter.java new file mode 100644 index 000000000..6af3276d5 --- /dev/null +++ b/filter-plugin/logstash-filter-mysql-aws-guardium/src/main/java/com/ibm/guardium/mysql/MySQLGuardiumPluginFilter.java @@ -0,0 +1,101 @@ +// +// Copyright 2020-2021 IBM Inc. All rights reserved +// SPDX-License-Identifier: Apache2.0 +// + +package com.ibm.guardium.mysql; + +import java.io.File; +import java.text.ParseException; +import java.util.*; + +import com.google.gson.*; +import org.apache.commons.text.StringEscapeUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.core.LoggerContext; +import com.ibm.guardium.universalconnector.commons.GuardConstants; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import co.elastic.logstash.api.Configuration; +import co.elastic.logstash.api.Context; +import co.elastic.logstash.api.Event; +import co.elastic.logstash.api.Filter; +import co.elastic.logstash.api.FilterMatchListener; +import co.elastic.logstash.api.LogstashPlugin; +import co.elastic.logstash.api.PluginConfigSpec; + +@LogstashPlugin(name = "mysql_guardium_plugin_filter") +public class MySQLGuardiumPluginFilter implements Filter { + + public static final String LOG42_CONF = "log4j2uc.properties"; + + static { + try { + String uc_etc = System.getenv("UC_ETC"); + LoggerContext context = (LoggerContext) LogManager.getContext(false); + File file = new File(uc_etc + File.separator + LOG42_CONF); + context.setConfigLocation(file.toURI()); + } catch (Exception e) { + e.printStackTrace(); + } + } + + private String id; + public static final PluginConfigSpec SOURCE_CONFIG = PluginConfigSpec.stringSetting("source", "message"); + private static Logger log = LogManager.getLogger(MySQLGuardiumPluginFilter.class); + + public MySQLGuardiumPluginFilter(String id, Configuration config, Context context) { + this.id = id; + } + + @Override + public Collection> configSchema() { + // should return a list of all configuration options for this plugin + return Collections.singletonList(SOURCE_CONFIG); + } + + @Override + public String getId() { + return this.id; + } + + + @Override + public Collection filter(Collection events, FilterMatchListener matchListener) { + for (Event e : events) { + log.info(Constants.EVENT_NOW_MYSQL, e.getData()); + try { + + if(null != e.getData() + && null != e.getData().get(Constants.COMMAND) && e.getData().get(Constants.COMMAND).toString() + .equals(Constants.COMMAND_UNKNOWN) && null != e.getData().get(Constants.DATABASE_USER_NAME) + && e.getData().get(Constants.DATABASE_USER_NAME).toString().equals(Constants.RDS_ADMIN)){ + continue; + } + + Record record = Parser.parseRecord(e); + + final GsonBuilder builder = new GsonBuilder(); + + builder.serializeNulls(); + + final Gson gson = builder.disableHtmlEscaping().create(); + + String jsonRecord = gson.toJson(record); + + jsonRecord = StringEscapeUtils.unescapeJson(jsonRecord); + + e.setField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME, jsonRecord); + + matchListener.filterMatched(e); + + } catch (ParseException ex) { + log.error("Given Event Is Not An Instance Of String " + e.getField(Constants.RECORDS)); + ex.printStackTrace(); + } + } + return events; + } + +} + diff --git a/filter-plugin/logstash-filter-mysql-aws-guardium/src/main/java/com/ibm/guardium/mysql/Parser.java b/filter-plugin/logstash-filter-mysql-aws-guardium/src/main/java/com/ibm/guardium/mysql/Parser.java new file mode 100644 index 000000000..6d45eed40 --- /dev/null +++ b/filter-plugin/logstash-filter-mysql-aws-guardium/src/main/java/com/ibm/guardium/mysql/Parser.java @@ -0,0 +1,286 @@ +package com.ibm.guardium.mysql; + +import co.elastic.logstash.api.Event; + +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.google.gson.JsonSyntaxException; +import com.ibm.guardium.universalconnector.commons.structures.Accessor; +import com.ibm.guardium.universalconnector.commons.structures.Construct; +import com.ibm.guardium.universalconnector.commons.structures.Data; +import com.ibm.guardium.universalconnector.commons.structures.ExceptionRecord; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import com.ibm.guardium.universalconnector.commons.structures.Sentence; +import com.ibm.guardium.universalconnector.commons.structures.SentenceObject; +import com.ibm.guardium.universalconnector.commons.structures.SessionLocator; +import com.ibm.guardium.universalconnector.commons.structures.Time; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + + +import java.text.ParseException; +import java.time.*; + +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + + +public class Parser { + + private static final Logger log = LogManager.getLogger(Parser.class); + + public static Record parseRecord(final Event event) throws ParseException { + + Map data = event.getData(); + + log.debug(Constants.EVENT_DATA, data); + + Record record = new Record(); + if (data != null) { + // set default value since we don't have sessionId + record.setSessionId(Constants.UNKNOWN_STRING); + + // accountId:dbName + record.setDbName(getAccountIdDBName(data)); + + + if(null != data.get(Constants.DATABASE_USER_NAME) + && !data.get(Constants.DATABASE_USER_NAME).toString().isEmpty()){ + record.setAppUserName(data.get(Constants.DATABASE_USER_NAME).toString()); + } + + record.setTime(getTime(data)); + + record.setSessionLocator(getSessionLocator(data)); + + record.setAccessor(getAccessor(data)); + + setQueryORExceptionRecord(data, record); + } + log.debug(Constants.GUARD_RECORD, record); + + return record; + + } + + private static void setQueryORExceptionRecord(Map data, Record record) { + if(null != data.get(Constants.STATUS_CODE) && null != data.get(Constants.COMMAND) + && Integer.parseInt(data.get(Constants.STATUS_CODE).toString()) == 0 + && !data.get(Constants.COMMAND).toString().isEmpty() + && data.get(Constants.COMMAND).toString().equals(Constants.QUERY_CONST)){ + setQueryData(data, record); + } else if (null != data.get(Constants.COMMAND) + && !data.get(Constants.COMMAND).toString().isEmpty() + && data.get(Constants.COMMAND).toString().equals(Constants.FAILED_CONNECT)) { + setExceptionRecord(data, record); + } else if (null != data.get(Constants.STATUS_CODE) + && Integer.parseInt(data.get(Constants.STATUS_CODE).toString()) != 0) { + setExceptionRecord (data, record); + } + } + + private static void setExceptionRecord(Map data, Record record) { + ExceptionRecord exceptionRecord = new ExceptionRecord(); + + String description = Constants.DESCRIPTION_MESSAGE + data.get(Constants.STATUS_CODE).toString(); + + exceptionRecord.setExceptionTypeId(Constants.SQL_ERROR); + + exceptionRecord.setDescription(description); + + if(null != data.get(Constants.QUERY) && !data.get(Constants.QUERY).toString().isEmpty()){ + exceptionRecord.setSqlString(data.get(Constants.QUERY).toString()); + } else { + exceptionRecord.setSqlString(Constants.NOT_AVAILABLE); + } + + if (null != data.get(Constants.COMMAND_TYPE) + && !data.get(Constants.COMMAND_TYPE).toString().isEmpty() + && null != data.get(Constants.ACTION) + && data.get(Constants.ACTION).toString().equals(Constants.FAILED_CONNECT)) { + exceptionRecord.setExceptionTypeId(Constants.LOGIN_FAILED); + description = Constants.CONNECTION_FAILED_DESCRIPTION_MESSAGE + data.get(Constants.STATUS_CODE).toString(); + exceptionRecord.setDescription((description)); + exceptionRecord.setSqlString(Constants.NOT_AVAILABLE); + } + record.setException(exceptionRecord); + } + + private static void setQueryData(Map data, Record record) { + if (data.get(Constants.QUERY) != null && !data.get(Constants.QUERY).toString().isEmpty()) { + + String rawQuery = data.get(Constants.QUERY).toString(); + + Data queryData = new Data(); + queryData.setConstruct(null); + queryData.setOriginalSqlCommand(rawQuery); + + record.setData(queryData); + } + } + + + private static Accessor getAccessor(Map data) { + Accessor accessor = new Accessor(); + + if(null != data.get(Constants.DATABASE_USER_NAME) + && !data.get(Constants.DATABASE_USER_NAME).toString().isEmpty()){ + accessor.setDbUser(data.get(Constants.DATABASE_USER_NAME).toString()); + } + + accessor.setServerType(Constants.SERVER_TYPE); + accessor.setServerOs(Constants.UNKNOWN_STRING); + accessor.setClientOs(Constants.UNKNOWN_STRING); + accessor.setClientHostName(Constants.UNKNOWN_STRING); + + // accountId:dbName + accessor.setServerHostName(getAccountIdDBName(data)); + + accessor.setCommProtocol(Constants.UNKNOWN_STRING); + accessor.setDbProtocol(Constants.DB_PROTOCOL); + accessor.setDbProtocolVersion(Constants.UNKNOWN_STRING); + accessor.setOsUser(Constants.UNKNOWN_STRING); + accessor.setSourceProgram(Constants.UNKNOWN_STRING); + accessor.setClient_mac(Constants.UNKNOWN_STRING); + + // accountId:dbName + accessor.setServiceName(getAccountIdDBName(data)); + + + accessor.setLanguage(Constants.DB_PROTOCOL); + accessor.setDataType(Constants.TEXT); + + return accessor; + } + + private static String getAccountIdDBName(Map data) { + + String dbNameAccountId = Constants.UNKNOWN_STRING; + + + if(null != data.get(Constants.ACCOUNT_ID) + && !data.get(Constants.ACCOUNT_ID).toString().isEmpty() + && null != data.get(Constants.DATABASE_NAME) + && !data.get(Constants.DATABASE_NAME).toString().isEmpty()){ + + String accountId = Constants.UNKNOWN_STRING; + String database = data.get(Constants.DATABASE_NAME).toString(); + + if(data.get(Constants.ACCOUNT_ID) instanceof String){ + accountId = data.get(Constants.ACCOUNT_ID).toString(); + } + else if (data.get(Constants.ACCOUNT_ID) instanceof List) { + List rawList = (List) data.get(Constants.ACCOUNT_ID); + List arrayList = new ArrayList<>(rawList); + + if(!arrayList.isEmpty()){ + accountId = String.valueOf(arrayList.get(0)); + } + } + dbNameAccountId = accountId+":"+database; + } + return dbNameAccountId; + } + + private static Time getTime(Map data) { + if(null != data.get(Constants.TIMESTAMP) && !data.get(Constants.TIMESTAMP).toString().isEmpty()){ + return getEpochTime(data.get(Constants.TIMESTAMP).toString()); + } + return null; + } + + private static SessionLocator getSessionLocator(Map data) { + String serverIp = getAccountIdDBName(data); + SessionLocator sessionLocator = new SessionLocator(); + if(null != data.get(Constants.CLIENT_IP) + && !data.get(Constants.CLIENT_IP).toString().isEmpty()){ + sessionLocator.setClientIp(data.get(Constants.CLIENT_IP).toString()); + } + sessionLocator.setClientPort(Constants.DEFAULT_PORT); + sessionLocator.setServerPort(Constants.DEFAULT_PORT); + sessionLocator.setServerIp(serverIp); + sessionLocator.setIpv6(false); + sessionLocator.setClientIpv6(Constants.UNKNOWN_STRING); + sessionLocator.setServerIpv6(Constants.UNKNOWN_STRING); + return sessionLocator; + } + + private static Time getEpochTime(String timeStamp) { + ZonedDateTime zonedDateTime; + + try { + // Try to parse as ISO-8601 (with 'Z' and nanoseconds) i.e. 2025-07-22T13:14:45.644605897Z + Instant instant = Instant.parse(timeStamp); + zonedDateTime = instant.atZone(ZoneId.systemDefault()); + } catch (DateTimeParseException e) { + // Fallback: try to parse using the custom format + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMdd HH:mm:ss"); + LocalDateTime localDateTime = LocalDateTime.parse(timeStamp, formatter); + zonedDateTime = localDateTime.atZone(ZoneId.systemDefault()); + } + + long millis = zonedDateTime.toInstant().toEpochMilli(); + int minOffset = zonedDateTime.getOffset().getTotalSeconds() / 60; + + return new Time(millis, minOffset, 0); + } + + private static String getDBInstanceName(String logGroupName) { + if (logGroupName == null || logGroupName.isEmpty()) { + return null; + } + + // Split by "/" and get the element at index 4 + String[] parts = logGroupName.split("/"); + if (parts.length >= 5) { + return parts[4]; + } + return null; + } + + + private static JsonObject getJSON(String jsonString) { + try { + JsonElement element = JsonParser.parseString(jsonString); + + if (element.isJsonObject()) { + return element.getAsJsonObject(); + } else { + throw new IllegalArgumentException("Provided string is not a JSON object"); + } + + } catch (JsonSyntaxException | IllegalArgumentException e) { + // Handle invalid JSON or wrong type + log.error("Error parsing JSON: " + e.getMessage()); + return null; + } + } + + private static boolean isValidJson(String json) { + if (json == null || json.trim().isEmpty()) { + log.warn("JSON string is null or empty"); + return false; + } + + try { + JsonElement element = JsonParser.parseString(json); + + if (element == null || element.isJsonNull()) { + log.warn("Parsed JSON is null: {}", json); + return false; + } + + return true; + } catch (JsonSyntaxException e) { + log.warn("Invalid JSON syntax: {} , {}", json, e); + return false; + } + } + +} diff --git a/filter-plugin/logstash-filter-mysql-aws-guardium/src/test/java/com/ibm/guardium/mysql/ParserTest.java b/filter-plugin/logstash-filter-mysql-aws-guardium/src/test/java/com/ibm/guardium/mysql/ParserTest.java new file mode 100644 index 000000000..670fd3969 --- /dev/null +++ b/filter-plugin/logstash-filter-mysql-aws-guardium/src/test/java/com/ibm/guardium/mysql/ParserTest.java @@ -0,0 +1,324 @@ +package com.ibm.guardium.mysql; + +import co.elastic.logstash.api.Event; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.text.ParseException; +import java.util.*; + +import static org.junit.jupiter.api.Assertions.*; + +class ParserTest { + + @Test + void testParseRecord_withValidInput_shouldReturnValidRecord() throws ParseException { + // Mock the Event + Event event = Mockito.mock(Event.class); + + // Prepare data for the mocked Event + Map mockData = new HashMap<>(); + + // Nested cloudwatch data as a Map + Map logEvents = new HashMap<>(); + logEvents.put("id", "12345678901234567890123456789012345678901234567890123456"); + logEvents.put("timestamp", 1753323600000L); + + Map cloudwatch = new HashMap<>(); + cloudwatch.put("logEvents", logEvents); + cloudwatch.put("logStream", "perfmysqlsv"); + cloudwatch.put("messageType", "DATA_MESSAGE"); + + List subscriptionFiltersList = new ArrayList(); + subscriptionFiltersList.add("PostgresCloudWatchLogstoS3"); + Map> subscriptionFiltersMap = new HashMap(); + subscriptionFiltersMap.put("delegate",subscriptionFiltersList); + + cloudwatch.put("subscriptionFilters",subscriptionFiltersMap); + + cloudwatch.put("owner", "123456789012"); + cloudwatch.put("logGroup", "/aws/rds/instance/sample-mysql-instance/audit"); + + mockData.put("cloudwatch", cloudwatch); + mockData.put("query_timestamp", "20250724 02:20:00"); + mockData.put("client_port", 734); + mockData.put("GuardRecord_did_not_exist", true); + mockData.put("logStream", "sample-mysql-instance"); + mockData.put("db_user", "unknown"); + mockData.put("client_ip", "192.168.1.100"); + mockData.put("fileKey", "mysql-logs/2025/07/24/CloudWatchLogstoS3-22-2025-07-24-02-19-30-sample-file-key.gz"); + mockData.put("command_type", "UNKNOWN"); + mockData.put("bucketName", "sample-mysql-audit-bucket"); + mockData.put("command", "QUERY"); + mockData.put("@timestamp", "2025-07-24T02:20:35.622185651Z"); + mockData.put("user", "admin"); + mockData.put("timestamp", "2025-07-24T02:20:35.622126440Z"); + mockData.put("database", "testdb"); + mockData.put("host", "sample-host-01"); + mockData.put("connection_id", 366361); + mockData.put("@version", 1); + mockData.put("type", "MYSQL_S3SQS"); + mockData.put("status_code", 0); + + List accountId = new ArrayList(); + accountId.add("123456789012"); + accountId.add("123456789012"); + + mockData.put("account_id", accountId); + mockData.put("query", "/* ApplicationName=DBeaver 25.0.3 - SQLEditor */ CREATE TABLE employees (\n" + + " id INT AUTO_INCREMENT PRIMARY KEY,\n" + + " first_name VARCHAR(50),\n" + + " last_name VARCHAR(50),\n" + + " email VARCHAR(100),\n" + + " hire_date DATE\n)"); + + // Embed the cloudwatch object as JSON string under "message" for getAccountIdDBName + mockData.put("message", new com.google.gson.Gson().toJson(cloudwatch)); + + // Set mock behavior + Mockito.when(event.getData()).thenReturn(mockData); + + // Act + Record record = Parser.parseRecord(event); + + // Assert + assertNotNull(record); + assertNotNull(record.getAccessor()); + assertNotNull(record.getSessionLocator()); + assertEquals("admin", record.getAccessor().getDbUser()); + assertNotNull(record.getData()); + assertTrue(record.getData().getOriginalSqlCommand().startsWith("/* ApplicationName=DBeaver")); + + // Validate db name format + assertTrue(record.getDbName().contains("123456789012:testdb")); + } + + @Test + void testParseRecord_withQuery() throws ParseException { + // Mock the Event + Event event = Mockito.mock(Event.class); + + // Prepare data for the mocked Event + Map mockData = new HashMap<>(); + + // Nested cloudwatch data as a Map + Map logEvents = new HashMap<>(); + logEvents.put("id", "12345678901234567890123456789012345678901234567890123456"); + logEvents.put("timestamp", 1753323600000L); + + Map cloudwatch = new HashMap<>(); + cloudwatch.put("logEvents", logEvents); + cloudwatch.put("logStream", "perfmysqlsv"); + cloudwatch.put("messageType", "DATA_MESSAGE"); + + List subscriptionFiltersList = new ArrayList(); + subscriptionFiltersList.add("PostgresCloudWatchLogstoS3"); + Map> subscriptionFiltersMap = new HashMap(); + subscriptionFiltersMap.put("delegate",subscriptionFiltersList); + + cloudwatch.put("subscriptionFilters",subscriptionFiltersMap); + + cloudwatch.put("owner", "123456789012"); + cloudwatch.put("logGroup", "/aws/rds/instance/sample-mysql-instance/audit"); + + mockData.put("cloudwatch", cloudwatch); + mockData.put("query_timestamp", "20250724 02:20:00"); + mockData.put("client_port", 734); + mockData.put("GuardRecord_did_not_exist", true); + mockData.put("logStream", "sample-mysql-instance"); + mockData.put("db_user", "unknown"); + mockData.put("client_ip", "192.168.1.100"); + mockData.put("fileKey", "mysql-logs/2025/07/24/CloudWatchLogstoS3-22-2025-07-24-02-19-30-sample-file-key.gz"); + mockData.put("command_type", "UNKNOWN"); + mockData.put("bucketName", "sample-mysql-audit-bucket"); + mockData.put("command", "QUERY"); + mockData.put("@timestamp", "2025-07-24T02:20:35.622185651Z"); + mockData.put("user", "admin"); + mockData.put("timestamp", "2025-07-24T02:20:35.622126440Z"); + mockData.put("database", "testdb"); + mockData.put("host", "sample-host-01"); + mockData.put("connection_id", 366361); + mockData.put("@version", 1); + mockData.put("type", "MYSQL_S3SQS"); + mockData.put("status_code", 0); + + List accountId = new ArrayList(); + accountId.add("123456789012"); + accountId.add("123456789012"); + + mockData.put("account_id", accountId); + mockData.put("query", "/* ApplicationName=DBeaver 25.0.3 - SQLEditor */ INSERT INTO Star123 (first_name, last_name, email, hire_date) VALUES(\\'John\\', \\'Doe\\', \\'john.doe@example.com\\', \\'2022-01-15\\')"); + + // Embed the cloudwatch object as JSON string under "message" for getAccountIdDBName + mockData.put("message", new com.google.gson.Gson().toJson(cloudwatch)); + + // Set mock behavior + Mockito.when(event.getData()).thenReturn(mockData); + + // Act + Record record = Parser.parseRecord(event); + + // Assert + assertNotNull(record); + assertNotNull(record.getAccessor()); + assertNotNull(record.getSessionLocator()); + assertEquals("admin", record.getAccessor().getDbUser()); + assertNotNull(record.getData()); + assertTrue(record.getData().getOriginalSqlCommand().startsWith("/* ApplicationName=DBeaver")); + + // Validate db name format + assertTrue(record.getDbName().contains("123456789012:testdb")); + } + + + @Test + void testParseRecordSQLSyntaxError() throws ParseException { + // Mock the Event + Event event = Mockito.mock(Event.class); + + // Prepare data for the mocked Event + Map mockData = new HashMap<>(); + + // Nested cloudwatch data as a Map + Map logEvents = new HashMap<>(); + logEvents.put("id", "98765432109876543210987654321098765432109876543210987654"); + logEvents.put("timestamp", 1753342566000L); + + Map cloudwatch = new HashMap<>(); + cloudwatch.put("logEvents", logEvents); + cloudwatch.put("logStream", "perfmysqlsv"); + cloudwatch.put("messageType", "DATA_MESSAGE"); + + List subscriptionFiltersList = new ArrayList<>(); + subscriptionFiltersList.add("PostgresCloudWatchLogstoS3"); + + Map> subscriptionFiltersMap = new HashMap<>(); + subscriptionFiltersMap.put("delegate", subscriptionFiltersList); + + cloudwatch.put("subscriptionFilters", subscriptionFiltersMap); + cloudwatch.put("owner", "123456789012"); + cloudwatch.put("logGroup", "/aws/rds/instance/sample-mysql-instance/audit"); + + mockData.put("cloudwatch", cloudwatch); + mockData.put("query_timestamp", "20250724 07:36:06"); + mockData.put("client_port", 734); + mockData.put("GuardRecord_did_not_exist", true); + mockData.put("logStream", "sample-mysql-instance"); + mockData.put("db_user", "unknown"); + mockData.put("client_ip", "192.168.1.100"); + mockData.put("fileKey", "mysql-logs/2025/07/24/CloudWatchLogstoS3-22-2025-07-24-07-35-25-sample-file-key.gz"); + mockData.put("command_type", "UNKNOWN"); + mockData.put("bucketName", "sample-mysql-audit-bucket"); + mockData.put("command", "QUERY"); + mockData.put("@timestamp", "2025-07-24T07:36:30.492230782Z"); + mockData.put("timestamp", "2025-07-24T07:36:30.492191791Z"); + mockData.put("user", "admin"); + mockData.put("host", "sample-host-01"); + mockData.put("connection_id", 417016); + mockData.put("database", "testdb"); + mockData.put("@version", 1); + mockData.put("type", "S3SQS__MYSQL"); + mockData.put("status_code", 1064); + + List accountId = new ArrayList<>(); + accountId.add("123456789012"); + accountId.add("123456789012"); + mockData.put("account_id", accountId); + + mockData.put("query", "/* ApplicationName=DBeaver 25.0.3 - SQLEditor */ DRGOP TABLE 24July2025_05"); + + // Embed the cloudwatch object as JSON string under "message" + mockData.put("message", new com.google.gson.Gson().toJson(cloudwatch)); + + // Set mock behavior + Mockito.when(event.getData()).thenReturn(mockData); + + // Act + Record record = Parser.parseRecord(event); + + // Assert + assertNotNull(record); + assertNotNull(record.getAccessor()); + assertNotNull(record.getSessionLocator()); + assertEquals("admin", record.getAccessor().getDbUser()); + assertNotNull(record.getException().getDescription()); + + assertTrue(record.getDbName().contains("123456789012:testdb")); + } + + @Test + void testParseRecordFailedConnect() throws ParseException { + // Mock the Event + Event event = Mockito.mock(Event.class); + + // Prepare data for the mocked Event + Map mockData = new HashMap<>(); + + // Nested cloudwatch data as a Map + Map logEvents = new HashMap<>(); + logEvents.put("id", "11111111111111111111111111111111111111111111111111111111"); + logEvents.put("timestamp", 1753346386000L); + + Map cloudwatch = new HashMap<>(); + cloudwatch.put("logEvents", logEvents); + cloudwatch.put("logStream", "perfmysqlsv"); + cloudwatch.put("messageType", "DATA_MESSAGE"); + + List subscriptionFiltersList = new ArrayList<>(); + subscriptionFiltersList.add("PostgresCloudWatchLogstoS3"); + + Map> subscriptionFiltersMap = new HashMap<>(); + subscriptionFiltersMap.put("delegate", subscriptionFiltersList); + + cloudwatch.put("subscriptionFilters", subscriptionFiltersMap); + cloudwatch.put("owner", "123456789012"); + cloudwatch.put("logGroup", "/aws/rds/instance/sample-mysql-instance/audit"); + + mockData.put("cloudwatch", cloudwatch); + mockData.put("client_port", 876); + mockData.put("GuardRecord_did_not_exist", true); + mockData.put("logStream", "sample-mysql-instance"); + mockData.put("db_user", "sample_user"); + mockData.put("client_ip", "192.168.1.100"); + mockData.put("fileKey", "mysql-logs/2025/07/24/CloudWatchLogstoS3-22-2025-07-24-08-39-00-sample-file-key.gz"); + mockData.put("command_type", "UNKNOWN"); + mockData.put("bucketName", "sample-mysql-audit-bucket"); + mockData.put("action", "FAILED_CONNECT"); + mockData.put("@timestamp", "2025-07-24T08:40:05.632510305Z"); + mockData.put("timestamp", "2025-07-24T08:40:05.632260750Z"); + mockData.put("connection_type", "TCP/IP"); + mockData.put("database", "unknown"); + mockData.put("status_code", 1045); + mockData.put("type", "S3SQS__MYSQL"); + mockData.put("@version", 1); + mockData.put("error_timestamp", "20250724 08:39:46"); + mockData.put("db_instance", "sample-host-01"); + + List accountIdList = new ArrayList<>(); + accountIdList.add("123456789012"); + accountIdList.add("123456789012"); + mockData.put("account_id", accountIdList); + + // Embed the cloudwatch object as JSON string under "message" + mockData.put("message", new com.google.gson.Gson().toJson(cloudwatch)); + + // Set mock behavior + Mockito.when(event.getData()).thenReturn(mockData); + + // Act + Record record = Parser.parseRecord(event); + + // Assert + assertNotNull(record); + assertNotNull(record.getAccessor()); + assertNotNull(record.getSessionLocator()); + + assertEquals("192.168.1.100", record.getSessionLocator().getClientIp()); + + assertTrue(record.getDbName().contains("123456789012:unknown")); + assertNotNull(record.getException()); + assertEquals("LOGIN_FAILED", record.getException().getExceptionTypeId()); + } + +} diff --git a/filter-plugin/logstash-filter-mysql-azure-guardium/azure_mysql.conf b/filter-plugin/logstash-filter-mysql-azure-guardium/azure_mysql.conf index 9fa08ba0d..2d805d382 100644 --- a/filter-plugin/logstash-filter-mysql-azure-guardium/azure_mysql.conf +++ b/filter-plugin/logstash-filter-mysql-azure-guardium/azure_mysql.conf @@ -9,12 +9,15 @@ input config_mode => "basic" #Insert primary connection string from shared access policies in event hub namespace from azure portal event_hub_connections => [";EntityPath="] - initial_position => "end" + initial_position => "beginning" threads => 16 decorate_events => true - consumer_group => "$Default" - #Insert the Connection string in storage_connection from the Access Keys present in the Storage account from azure portal. + storage_container => "logstash-checkpoints" + #Create a custom consumer group in event hub namespace, example: logstash-processor + consumer_group => "$Default" + #Insert the Connection string in storage_connection from the Access Keys present in the Storage account from azure portal storage_connection => "" + checkpoint_interval => 5 type => "azure_mysql" } } diff --git a/filter-plugin/logstash-filter-mysql-azure-guardium/build.gradle b/filter-plugin/logstash-filter-mysql-azure-guardium/build.gradle index d9562de63..14ebeb084 100644 --- a/filter-plugin/logstash-filter-mysql-azure-guardium/build.gradle +++ b/filter-plugin/logstash-filter-mysql-azure-guardium/build.gradle @@ -2,6 +2,29 @@ import java.nio.file.Files import static java.nio.file.StandardCopyOption.REPLACE_EXISTING apply plugin: 'java' + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + + dependencies { + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' + classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' + } +} + +def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + apply plugin: 'jacoco' apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" // =========================================================================== @@ -20,35 +43,25 @@ pluginInfo.pluginClass = "AzureMysqlGuardiumFilter" pluginInfo.pluginName = "azure_mysql_guardium_filter" // must match the @LogstashPlugin annotation in the main plugin class // =========================================================================== -sourceCompatibility = 1.8 -targetCompatibility = 1.8 - -buildscript { - repositories { - maven { - url "https://plugins.gradle.org/m2/" - } - mavenCentral() - jcenter() - } - - dependencies { - classpath 'com.github.jengelman.gradle.plugins:shadow:4.0.4' - classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' - } -} +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 -def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); -def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) repositories { - mavenCentral() + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } } apply plugin: 'com.github.johnrengelman.shadow' shadowJar { - classifier = null + archiveClassifier = null transform(com.github.jengelman.gradle.plugins.shadow.transformers.Log4j2PluginsCacheFileTransformer) } @@ -57,6 +70,7 @@ dependencies { implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: versions.dependencies.log4jCore implementation 'org.apache.commons:commons-lang3:' + versions.dependencies.commonsLang implementation 'com.google.code.gson:gson:' + versions.dependencies.gson + implementation group: 'commons-beanutils', name: 'commons-beanutils', version: versions.dependencies.commonsBeanutils implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "build/libs/logstash-core-*.*.*.jar") implementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "common-*.*.*.jar") @@ -81,6 +95,17 @@ tasks.withType(JavaCompile) { options.encoding = 'UTF-8' } +test { + jvmArgs = [ + '--add-opens=java.base/java.lang=ALL-UNNAMED', + '--add-opens=java.base/java.util=ALL-UNNAMED', + '--add-opens=java.base/java.text=ALL-UNNAMED', + '--add-opens=java.base/java.lang.reflect=ALL-UNNAMED', + '--add-opens=java.base/sun.util.resources=ALL-UNNAMED', + '--add-opens=java.base/sun.util.cldr=ALL-UNNAMED' + ] +} + tasks.register("vendor"){ dependsOn shadowJar doLast { @@ -89,7 +114,6 @@ tasks.register("vendor"){ File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") projectJarFile.mkdirs() Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) - validatePluginJar(projectJarFile, project.group) } } @@ -118,8 +142,8 @@ tasks.register("gem"){ } jacocoTestReport { reports { - xml.enabled true - html.enabled true + xml.required = true + html.required = true } afterEvaluate { // (optional) : to exclude classes / packages from coverage diff --git a/filter-plugin/logstash-filter-mysql-azure-guardium/gradle/wrapper/gradle-wrapper.properties b/filter-plugin/logstash-filter-mysql-azure-guardium/gradle/wrapper/gradle-wrapper.properties index aa991fcea..81aa1c044 100644 --- a/filter-plugin/logstash-filter-mysql-azure-guardium/gradle/wrapper/gradle-wrapper.properties +++ b/filter-plugin/logstash-filter-mysql-azure-guardium/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/filter-plugin/logstash-filter-mysql-azure-guardium/mainREADME.md b/filter-plugin/logstash-filter-mysql-azure-guardium/mainREADME.md new file mode 100644 index 000000000..5e9c9a495 --- /dev/null +++ b/filter-plugin/logstash-filter-mysql-azure-guardium/mainREADME.md @@ -0,0 +1,10 @@ +# Azure MySQL Universal Connector + +## Follow this link to set up and use Azure MySQL Universal Connector over Azure Eventhub Logstash Plugin + +[MySqlOverEventHub](../logstash-filter-mysql-guardium/README.md) + +## Follow this link to set up and use Azure MySQL Universal Connector over Azure Eventhub Connect + +[MySqlOverConnectEventHub](../../docs/KafkaBasedUCs/AzureMySQLEventHubKafkaConnect.md) + diff --git a/filter-plugin/logstash-filter-mysql-azure-guardium/src/main/java/com/ibm/guardium/azuremysql/AzureMysqlGuardiumFilter.java b/filter-plugin/logstash-filter-mysql-azure-guardium/src/main/java/com/ibm/guardium/azuremysql/AzureMysqlGuardiumFilter.java index 1cb9b9b23..179b51652 100644 --- a/filter-plugin/logstash-filter-mysql-azure-guardium/src/main/java/com/ibm/guardium/azuremysql/AzureMysqlGuardiumFilter.java +++ b/filter-plugin/logstash-filter-mysql-azure-guardium/src/main/java/com/ibm/guardium/azuremysql/AzureMysqlGuardiumFilter.java @@ -15,7 +15,15 @@ import com.ibm.guardium.azuremysql.Parser; import com.ibm.guardium.azuremysql.Constants; import com.ibm.guardium.universalconnector.commons.GuardConstants; -import com.ibm.guardium.universalconnector.commons.structures.*; +import com.ibm.guardium.universalconnector.commons.structures.Accessor; +import com.ibm.guardium.universalconnector.commons.structures.Construct; +import com.ibm.guardium.universalconnector.commons.structures.Data; +import com.ibm.guardium.universalconnector.commons.structures.ExceptionRecord; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import com.ibm.guardium.universalconnector.commons.structures.Sentence; +import com.ibm.guardium.universalconnector.commons.structures.SentenceObject; +import com.ibm.guardium.universalconnector.commons.structures.SessionLocator; +import com.ibm.guardium.universalconnector.commons.structures.Time; import com.ibm.guardium.universalconnector.commons.structures.Record; import org.apache.logging.log4j.LogManager; diff --git a/filter-plugin/logstash-filter-mysql-azure-guardium/src/main/java/com/ibm/guardium/azuremysql/Parser.java b/filter-plugin/logstash-filter-mysql-azure-guardium/src/main/java/com/ibm/guardium/azuremysql/Parser.java index 81867d821..8233031c5 100644 --- a/filter-plugin/logstash-filter-mysql-azure-guardium/src/main/java/com/ibm/guardium/azuremysql/Parser.java +++ b/filter-plugin/logstash-filter-mysql-azure-guardium/src/main/java/com/ibm/guardium/azuremysql/Parser.java @@ -21,7 +21,15 @@ import com.google.gson.JsonArray; import com.google.gson.JsonObject; import com.ibm.guardium.azuremysql.Constants; -import com.ibm.guardium.universalconnector.commons.structures.*; +import com.ibm.guardium.universalconnector.commons.structures.Accessor; +import com.ibm.guardium.universalconnector.commons.structures.Construct; +import com.ibm.guardium.universalconnector.commons.structures.Data; +import com.ibm.guardium.universalconnector.commons.structures.ExceptionRecord; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import com.ibm.guardium.universalconnector.commons.structures.Sentence; +import com.ibm.guardium.universalconnector.commons.structures.SentenceObject; +import com.ibm.guardium.universalconnector.commons.structures.SessionLocator; +import com.ibm.guardium.universalconnector.commons.structures.Time; import com.ibm.guardium.universalconnector.commons.structures.Record; import co.elastic.logstash.api.Event; import org.apache.logging.log4j.LogManager; diff --git a/filter-plugin/logstash-filter-mysql-guardium/README.md b/filter-plugin/logstash-filter-mysql-guardium/README.md index b289bb317..dc22209ba 100644 --- a/filter-plugin/logstash-filter-mysql-guardium/README.md +++ b/filter-plugin/logstash-filter-mysql-guardium/README.md @@ -5,11 +5,8 @@ * Supported Guardium versions: * Guardium Data Protection: 11.3 and above * Supported inputs: - * Syslog (push) + * Linux server only for Syslog (push) * Filebeat (push) - * Guardium Data Security Center SaaS: 1.0 - * Supported inputs: - * Filebeat (push) This is a [Logstash](https://github.com/elastic/logstash) filter plug-in for the universal connector that is featured in IBM Security Guardium. It parses events and messages from the MySQL audit into a Guardium record instance (which is a standard structure made out of several parts). The information is then sent over to Guardium. [Guardium records](https://github.com/IBM/universal-connectors/blob/main/common/src/main/java/com/ibm/guardium/universalconnector/commons/structures/Record.java) include the accessor (the person who tried to access the data), the session, data, and exceptions. If there are no errors, the data contains details about the query "construct". The contstruct details the main action (verb) and collections (objects) involved. @@ -35,6 +32,28 @@ Run the following two SQLs to install the default filter to get every log: #### SELECT audit_log_filter_set_filter('log_all', '{ "filter": { "log": true }}'); SELECT audit_log_filter_set_user('%', 'log_all'); + +Syslogs configuration: + +1. Configure rsyslog to forward the logs to Logstash. For example, you can add the following line to /etc/rsyslog.conf file: +#### + # MySQL Audit Log Forwarding + $ModLoad imfile + $InputFileName /var/lib/mysql/audit.log + $InputFileTag mysql_audit_log: + $InputFileStateFile audit_log + $InputFileSeverity info + $InputFileFacility local6 + $InputRunFileMonitor + + # Forward to Logstash on Guardium server (UDP) + local6.* @:5143 +#### +2. Restart the rsyslog service. For example, you can run the following command: +systemctl restart rsyslog +#### + systemctl restart rsyslog +#### ### Windows database-server 1. Append below code to my.ini file, located in MySQL directory (for example: "C:\ProgramData\MySQL\MySQL Server 8.0"): @@ -67,7 +86,8 @@ https://www.elastic.co/guide/en/beats/filebeat/current/directory-layout.html • Locate "filebeat.inputs" in the filebeat.yml file, then add the following parameters. filebeat.inputs: - - type: log + - type: filestream + - id: - enabled: true paths: - diff --git a/filter-plugin/logstash-filter-mysql-guardium/build.gradle b/filter-plugin/logstash-filter-mysql-guardium/build.gradle index c48812e74..2436314f0 100644 --- a/filter-plugin/logstash-filter-mysql-guardium/build.gradle +++ b/filter-plugin/logstash-filter-mysql-guardium/build.gradle @@ -2,6 +2,30 @@ import java.nio.file.Files import static java.nio.file.StandardCopyOption.REPLACE_EXISTING apply plugin: 'java' + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + + dependencies { + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' + classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.1.0" + classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' + } +} + +def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" // =========================================================================== @@ -20,10 +44,10 @@ pluginInfo.pluginClass = "MySqlFilterGuardium" pluginInfo.pluginName = "mysql_filter_guardium" // must match the @LogstashPlugin annotation in the main plugin class // =========================================================================== -sourceCompatibility = 1.8 -targetCompatibility = 1.8 +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 -def jacocoVersion = '0.8.4' +def jacocoVersion = '0.8.11' // minimumCoverage can be set by Travis ENV def minimumCoverageStr = System.getenv("MINIMUM_COVERAGE") ?: "50.0%" if (minimumCoverageStr.endsWith("%")) { @@ -31,27 +55,16 @@ if (minimumCoverageStr.endsWith("%")) { } def minimumCoverage = Float.valueOf(minimumCoverageStr) / 100 -buildscript { - repositories { - maven { - url "https://plugins.gradle.org/m2/" - } - mavenCentral() - jcenter() - } - - dependencies { - classpath 'com.github.jengelman.gradle.plugins:shadow:4.0.4' - classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.0.0-RC2" - classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' - } -} - -def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); -def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) repositories { - mavenCentral() + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } } tasks.register("vendor"){ @@ -62,14 +75,13 @@ tasks.register("vendor"){ File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") projectJarFile.mkdirs() Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) - validatePluginJar(projectJarFile, project.group) } } apply plugin: 'com.github.johnrengelman.shadow' shadowJar { - classifier = null + archiveClassifier = null } dependencies { @@ -77,6 +89,7 @@ dependencies { implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: versions.dependencies.log4jCore implementation 'org.apache.commons:commons-lang3:' + versions.dependencies.commonsLang implementation 'com.google.code.gson:gson:' + versions.dependencies.gson + implementation group: 'commons-beanutils', name: 'commons-beanutils', version: versions.dependencies.commonsBeanutils implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "build/libs/logstash-core-*.*.*.jar") implementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "common-*.*.*.jar") @@ -100,6 +113,17 @@ tasks.withType(JavaCompile) { options.encoding = 'UTF-8' } +test { + jvmArgs = [ + '--add-opens=java.base/java.lang=ALL-UNNAMED', + '--add-opens=java.base/java.util=ALL-UNNAMED', + '--add-opens=java.base/java.text=ALL-UNNAMED', + '--add-opens=java.base/java.lang.reflect=ALL-UNNAMED', + '--add-opens=java.base/sun.util.resources=ALL-UNNAMED', + '--add-opens=java.base/sun.util.cldr=ALL-UNNAMED' + ] +} + tasks.register("generateRubySupportFiles") { @@ -142,18 +166,17 @@ apply plugin: "org.barfuin.gradle.jacocolog" // ------------------------------------ jacoco { toolVersion = "${jacocoVersion}" - reportsDir = file("$buildDir/reports/jacoco") } jacocoTestReport { // You will see "Report -> file://...." at the end of a JaCoCo build // If no output, run this first: ./gradlew test reports { - html.enabled true - xml.enabled true - csv.enabled true - html.destination file("${buildDir}/reports/jacoco") - csv.destination file("${buildDir}/reports/jacoco/all.csv") + html.required = true + xml.required = true + csv.required = true + html.outputLocation = file("${buildDir}/reports/jacoco") + csv.outputLocation = file("${buildDir}/reports/jacoco/all.csv") } executionData.from fileTree(dir: "${buildDir}/jacoco/", includes: [ '**/*.exec' diff --git a/filter-plugin/logstash-filter-mysql-guardium/gradle/wrapper/gradle-wrapper.properties b/filter-plugin/logstash-filter-mysql-guardium/gradle/wrapper/gradle-wrapper.properties index aa991fcea..81aa1c044 100644 --- a/filter-plugin/logstash-filter-mysql-guardium/gradle/wrapper/gradle-wrapper.properties +++ b/filter-plugin/logstash-filter-mysql-guardium/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/filter-plugin/logstash-filter-mysql-guardium/src/main/java/org/logstashplugins/MySqlFilterGuardium.java b/filter-plugin/logstash-filter-mysql-guardium/src/main/java/org/logstashplugins/MySqlFilterGuardium.java index 6b948ba9d..b57feda86 100644 --- a/filter-plugin/logstash-filter-mysql-guardium/src/main/java/org/logstashplugins/MySqlFilterGuardium.java +++ b/filter-plugin/logstash-filter-mysql-guardium/src/main/java/org/logstashplugins/MySqlFilterGuardium.java @@ -27,7 +27,15 @@ import java.text.ParseException; import java.text.SimpleDateFormat; -import com.ibm.guardium.universalconnector.commons.structures.*; +import com.ibm.guardium.universalconnector.commons.structures.Accessor; +import com.ibm.guardium.universalconnector.commons.structures.Construct; +import com.ibm.guardium.universalconnector.commons.structures.Data; +import com.ibm.guardium.universalconnector.commons.structures.ExceptionRecord; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import com.ibm.guardium.universalconnector.commons.structures.Sentence; +import com.ibm.guardium.universalconnector.commons.structures.SentenceObject; +import com.ibm.guardium.universalconnector.commons.structures.SessionLocator; +import com.ibm.guardium.universalconnector.commons.structures.Time; import com.ibm.guardium.universalconnector.commons.Util; import com.ibm.guardium.universalconnector.commons.GuardConstants; @@ -54,7 +62,7 @@ public class MySqlFilterGuardium implements Filter { private static final String DATA_TYPE_CONNECTION = "connection_data"; private static final String MYSQL_AUDIT_START_SIGNAL = "mysql_audit_log: "; - public static final String DATA_PROTOCOL_STRING = "MySQL native audit"; + public static final String DATA_PROTOCOL_STRING = "MySQL"; public static final String UNKNOWN_STRING = ""; public static final String SERVER_TYPE_STRING = "MySql"; private static final String MASK_STRING = "?"; diff --git a/filter-plugin/logstash-filter-mysql-percona-guardium/README.md b/filter-plugin/logstash-filter-mysql-percona-guardium/README.md index 46f42f354..371dea767 100755 --- a/filter-plugin/logstash-filter-mysql-percona-guardium/README.md +++ b/filter-plugin/logstash-filter-mysql-percona-guardium/README.md @@ -51,7 +51,8 @@ https://www.elastic.co/guide/en/beats/filebeat/current/directory-layout.html • Locate "filebeat.inputs" in the filebeat.yml file, and then add the following parameters. ``` - type: log + type: filestream + id: #Change to true to enable this input configuration. enabled: true paths: diff --git a/filter-plugin/logstash-filter-mysql-percona-guardium/build.gradle b/filter-plugin/logstash-filter-mysql-percona-guardium/build.gradle index 7a35e4560..dee6a1202 100644 --- a/filter-plugin/logstash-filter-mysql-percona-guardium/build.gradle +++ b/filter-plugin/logstash-filter-mysql-percona-guardium/build.gradle @@ -2,6 +2,30 @@ import java.nio.file.Files import static java.nio.file.StandardCopyOption.REPLACE_EXISTING apply plugin: 'java' + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + + dependencies { + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' + classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.1.0" + classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' + } +} + +def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" // =========================================================================== @@ -20,10 +44,10 @@ pluginInfo.pluginClass = "MySqlPerconaFilter" pluginInfo.pluginName = "mysql_percona_filter" // must match the @LogstashPlugin annotation in the main plugin class // =========================================================================== -sourceCompatibility = 1.8 -targetCompatibility = 1.8 +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 -def jacocoVersion = '0.8.4' +def jacocoVersion = '0.8.11' // minimumCoverage can be set by Travis ENV def minimumCoverageStr = System.getenv("MINIMUM_COVERAGE") ?: "50.0%" if (minimumCoverageStr.endsWith("%")) { @@ -31,27 +55,16 @@ if (minimumCoverageStr.endsWith("%")) { } def minimumCoverage = Float.valueOf(minimumCoverageStr) / 100 -buildscript { - repositories { - maven { - url "https://plugins.gradle.org/m2/" - } - mavenCentral() - jcenter() - } - - dependencies { - classpath 'com.github.jengelman.gradle.plugins:shadow:4.0.4' - classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.0.0-RC2" - classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' - } -} - -def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); -def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) repositories { - mavenCentral() + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } } tasks.register("vendor"){ @@ -62,14 +75,13 @@ tasks.register("vendor"){ File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") projectJarFile.mkdirs() Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) - validatePluginJar(projectJarFile, project.group) } } apply plugin: 'com.github.johnrengelman.shadow' shadowJar { - classifier = null + archiveClassifier = null } dependencies { @@ -77,6 +89,7 @@ dependencies { implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: versions.dependencies.log4jCore implementation 'org.apache.commons:commons-lang3:' + versions.dependencies.commonsLang implementation 'com.google.code.gson:gson:' + versions.dependencies.gson + implementation group: 'commons-beanutils', name: 'commons-beanutils', version: versions.dependencies.commonsBeanutils implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "build/libs/logstash-core-*.*.*.jar") implementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "common-*.*.*.jar") @@ -100,6 +113,17 @@ tasks.withType(JavaCompile) { options.encoding = 'UTF-8' } +test { + jvmArgs = [ + '--add-opens=java.base/java.lang=ALL-UNNAMED', + '--add-opens=java.base/java.util=ALL-UNNAMED', + '--add-opens=java.base/java.text=ALL-UNNAMED', + '--add-opens=java.base/java.lang.reflect=ALL-UNNAMED', + '--add-opens=java.base/sun.util.resources=ALL-UNNAMED', + '--add-opens=java.base/sun.util.cldr=ALL-UNNAMED' + ] +} + tasks.register("generateRubySupportFiles") { @@ -142,17 +166,16 @@ apply plugin: "org.barfuin.gradle.jacocolog" // ------------------------------------ jacoco { toolVersion = "${jacocoVersion}" - reportsDir = file("$buildDir/reports/jacoco") } jacocoTestReport { // You will see "Report -> file://...." at the end of a JaCoCo build // If no output, run this first: ./gradlew test reports { - html.enabled true - xml.enabled true - csv.enabled true - html.destination file("${buildDir}/reports/jacoco") - csv.destination file("${buildDir}/reports/jacoco/all.csv") + html.required = true + xml.required = true + csv.required = true + html.outputLocation = file("${buildDir}/reports/jacoco") + csv.outputLocation = file("${buildDir}/reports/jacoco/all.csv") } executionData.from fileTree(dir: "${buildDir}/jacoco/", includes: [ '**/*.exec' diff --git a/filter-plugin/logstash-filter-mysql-percona-guardium/gradle/wrapper/gradle-wrapper.properties b/filter-plugin/logstash-filter-mysql-percona-guardium/gradle/wrapper/gradle-wrapper.properties index aa991fcea..81aa1c044 100644 --- a/filter-plugin/logstash-filter-mysql-percona-guardium/gradle/wrapper/gradle-wrapper.properties +++ b/filter-plugin/logstash-filter-mysql-percona-guardium/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/filter-plugin/logstash-filter-mysql-percona-guardium/src/main/java/com/ibm/guardium/mysql/percona/MySqlPerconaFilter.java b/filter-plugin/logstash-filter-mysql-percona-guardium/src/main/java/com/ibm/guardium/mysql/percona/MySqlPerconaFilter.java index 7cb0abf1f..c7d347ccc 100755 --- a/filter-plugin/logstash-filter-mysql-percona-guardium/src/main/java/com/ibm/guardium/mysql/percona/MySqlPerconaFilter.java +++ b/filter-plugin/logstash-filter-mysql-percona-guardium/src/main/java/com/ibm/guardium/mysql/percona/MySqlPerconaFilter.java @@ -24,7 +24,15 @@ import java.io.File; import java.time.ZonedDateTime; -import com.ibm.guardium.universalconnector.commons.structures.*; +import com.ibm.guardium.universalconnector.commons.structures.Accessor; +import com.ibm.guardium.universalconnector.commons.structures.Construct; +import com.ibm.guardium.universalconnector.commons.structures.Data; +import com.ibm.guardium.universalconnector.commons.structures.ExceptionRecord; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import com.ibm.guardium.universalconnector.commons.structures.Sentence; +import com.ibm.guardium.universalconnector.commons.structures.SentenceObject; +import com.ibm.guardium.universalconnector.commons.structures.SessionLocator; +import com.ibm.guardium.universalconnector.commons.structures.Time; import com.ibm.guardium.universalconnector.commons.Util; import com.ibm.guardium.universalconnector.commons.GuardConstants; @@ -43,7 +51,7 @@ public class MySqlPerconaFilter implements Filter { public static final String LOGSTASH_TAG_MYSQL_IGNORE = "_mysqlguardium_ignore"; private static final String MYSQL_AUDIT_START_SIGNAL = "percona-audit: "; - public static final String DATA_PROTOCOL_STRING = "MySQL Percona audit"; + public static final String DATA_PROTOCOL_STRING = "MySQL Percona"; public static final String UNKNOWN_STRING = ""; public static final String SERVER_TYPE_STRING = "MySql"; diff --git a/filter-plugin/logstash-filter-neo4j-guardium/README.md b/filter-plugin/logstash-filter-neo4j-guardium/README.md index 3e782006c..425c85433 100644 --- a/filter-plugin/logstash-filter-neo4j-guardium/README.md +++ b/filter-plugin/logstash-filter-neo4j-guardium/README.md @@ -71,7 +71,8 @@ To use Logstash to perform additional processing on the data collected by Filebe • Locate "filebeat.inputs" in the filebeat.yml file, then add the following parameters. filebeat.inputs: - - type: log + - type: filestream + - id: enabled: true paths : - /path/to/query.log @@ -120,6 +121,7 @@ To use Logstash to perform additional processing on the data collected by Filebe 3. Neo4j logs some queries multiple times, since they are executed in pipeline, so the same will be reflected in the Reports 4. Multiple system related queries are logged, which cannot be skipped, so will be seen in the Reports 5. Neo4j does not support Failed Login. +6. Syntactically incorrect queries are executed as success queries and not as Sql Error. #### For details on configuring Filebeat connection over SSL, refer [Configuring Filebeat to push logs to Guardium](https://github.com/IBM/universal-connectors/blob/main/input-plugin/logstash-input-beats/README.md#configuring-filebeat-to-push-logs-to-guardium). diff --git a/filter-plugin/logstash-filter-neo4j-guardium/build.gradle b/filter-plugin/logstash-filter-neo4j-guardium/build.gradle index afedb8349..25b63362e 100644 --- a/filter-plugin/logstash-filter-neo4j-guardium/build.gradle +++ b/filter-plugin/logstash-filter-neo4j-guardium/build.gradle @@ -2,6 +2,30 @@ import java.nio.file.Files import static java.nio.file.StandardCopyOption.REPLACE_EXISTING apply plugin: 'java' + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + + dependencies { + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' + classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.1.0" + classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' + } +} + +def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" // =========================================================================== @@ -21,10 +45,10 @@ pluginInfo.pluginName = "neodb_guardium_filter" // must match the @Logstash // =========================================================================== -sourceCompatibility = 1.8 -targetCompatibility = 1.8 +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 -def jacocoVersion = '0.8.4' +def jacocoVersion = '0.8.11' // minimumCoverage can be set by Travis ENV def minimumCoverageStr = System.getenv("MINIMUM_COVERAGE") ?: "50.0%" if (minimumCoverageStr.endsWith("%")) { @@ -32,28 +56,17 @@ if (minimumCoverageStr.endsWith("%")) { } def minimumCoverage = Float.valueOf(minimumCoverageStr) / 100 -buildscript { - repositories { - maven { - url "https://plugins.gradle.org/m2/" - } - mavenCentral() - jcenter() - } - - dependencies { - classpath 'com.github.jengelman.gradle.plugins:shadow:4.0.4' - classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.0.0-RC2" - classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' - } -} - -def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); -def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) repositories { - mavenCentral() + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } } tasks.register("vendor"){ @@ -64,14 +77,13 @@ tasks.register("vendor"){ File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") projectJarFile.mkdirs() Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) - validatePluginJar(projectJarFile, project.group) } } apply plugin: 'com.github.johnrengelman.shadow' shadowJar { - classifier = null + archiveClassifier = null } dependencies { @@ -79,6 +91,7 @@ dependencies { implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: versions.dependencies.log4jCore implementation 'org.apache.commons:commons-lang3:' + versions.dependencies.commonsLang implementation 'com.google.code.gson:gson:' + versions.dependencies.gson + implementation group: 'commons-beanutils', name: 'commons-beanutils', version: versions.dependencies.commonsBeanutils implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "build/libs/logstash-core-*.*.*.jar") implementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "common-*.*.*.jar") @@ -102,6 +115,17 @@ tasks.withType(JavaCompile) { options.encoding = 'UTF-8' } +test { + jvmArgs = [ + '--add-opens=java.base/java.lang=ALL-UNNAMED', + '--add-opens=java.base/java.util=ALL-UNNAMED', + '--add-opens=java.base/java.text=ALL-UNNAMED', + '--add-opens=java.base/java.lang.reflect=ALL-UNNAMED', + '--add-opens=java.base/sun.util.resources=ALL-UNNAMED', + '--add-opens=java.base/sun.util.cldr=ALL-UNNAMED' + ] +} + tasks.register("generateRubySupportFiles") { @@ -144,17 +168,16 @@ apply plugin: "org.barfuin.gradle.jacocolog" // ------------------------------------ jacoco { toolVersion = "${jacocoVersion}" - reportsDir = file("$buildDir/reports/jacoco") } jacocoTestReport { // You will see "Report -> file://...." at the end of a JaCoCo build // If no output, run this first: ./gradlew test reports { - html.enabled true - xml.enabled true - csv.enabled true - html.destination file("${buildDir}/reports/jacoco") - csv.destination file("${buildDir}/reports/jacoco/all.csv") + html.required = true + xml.required = true + csv.required = true + html.outputLocation = file("${buildDir}/reports/jacoco") + csv.outputLocation = file("${buildDir}/reports/jacoco/all.csv") } executionData.from fileTree(dir: "${buildDir}/jacoco/", includes: [ '**/*.exec' diff --git a/filter-plugin/logstash-filter-neo4j-guardium/gradle/wrapper/gradle-wrapper.properties b/filter-plugin/logstash-filter-neo4j-guardium/gradle/wrapper/gradle-wrapper.properties index aa991fcea..81aa1c044 100644 --- a/filter-plugin/logstash-filter-neo4j-guardium/gradle/wrapper/gradle-wrapper.properties +++ b/filter-plugin/logstash-filter-neo4j-guardium/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/filter-plugin/logstash-filter-neo4j-guardium/logstash-filter-neodb_guardium_filter.zip b/filter-plugin/logstash-filter-neo4j-guardium/logstash-filter-neodb_guardium_filter.zip index 96693b663..110bf5855 100644 Binary files a/filter-plugin/logstash-filter-neo4j-guardium/logstash-filter-neodb_guardium_filter.zip and b/filter-plugin/logstash-filter-neo4j-guardium/logstash-filter-neodb_guardium_filter.zip differ diff --git a/filter-plugin/logstash-filter-neo4j-guardium/src/main/java/com/ibm/guardium/neodb/Constants.java b/filter-plugin/logstash-filter-neo4j-guardium/src/main/java/com/ibm/guardium/neodb/Constants.java index b2369ee9b..be11de1b6 100644 --- a/filter-plugin/logstash-filter-neo4j-guardium/src/main/java/com/ibm/guardium/neodb/Constants.java +++ b/filter-plugin/logstash-filter-neo4j-guardium/src/main/java/com/ibm/guardium/neodb/Constants.java @@ -9,7 +9,7 @@ public class Constants { public static final String UNKNOWN_STRING = ""; public static final String NOT_AVAILABLE = "NA"; - public static final String DATA_PROTOCOL_STRING = "Bolt database protocol"; + public static final String DATA_PROTOCOL_STRING = "BoltDB"; public static final String SERVER_TYPE_STRING = "NEO4J"; public static final String COMM_PROTOCOL = "Neo4JApiCall"; public static final String LOGSTASH_TAG_SKIP_NOT_NEO = "_neoguardium_skip_not_neo"; diff --git a/filter-plugin/logstash-filter-neo4j-guardium/src/main/java/com/ibm/guardium/neodb/Parser.java b/filter-plugin/logstash-filter-neo4j-guardium/src/main/java/com/ibm/guardium/neodb/Parser.java index 3d78014c9..9ecadbe50 100644 --- a/filter-plugin/logstash-filter-neo4j-guardium/src/main/java/com/ibm/guardium/neodb/Parser.java +++ b/filter-plugin/logstash-filter-neo4j-guardium/src/main/java/com/ibm/guardium/neodb/Parser.java @@ -42,13 +42,27 @@ public class Parser { private static final DateTimeFormatter DATE_TIME_FORMATTER = dateTimeFormatterBuilder.toFormatter(); - static LinkedHashMap> operations = null; - static LinkedHashMap variables = null; - static Character randomValue = 65; + // Removed static modifiers from these variables + LinkedHashMap> operations = null; + LinkedHashMap variables = null; + Character randomValue = 65; + + // Added constructor to initialize instance variables + public Parser() { + operations = null; + variables = null; + randomValue = 65; + } + // Keep this method static as an entry point public static Record parseRecord(final JsonObject data) throws ParseException { - Record record = new Record(); + Parser parser = new Parser(); + return parser.parseRecordInternal(data); + } + // New instance method that contains the original parseRecord logic + public Record parseRecordInternal(final JsonObject data) throws ParseException { + Record record = new Record(); if (data != null) { record.setAppUserName(Constants.NOT_AVAILABLE); @@ -63,29 +77,29 @@ public static Record parseRecord(final JsonObject data) throws ParseException { record.setDbName(dbName); // Time - String dateString = Parser.parseTimestamp(data); + String dateString = parseTimestamp(data); String timeZone = null; if (data.has(Constants.MIN_OFF)) { timeZone = data.get(Constants.MIN_OFF).getAsString(); } - Time time = Parser.getTime(dateString, timeZone); + Time time = getTime(dateString, timeZone); if (time != null) { record.setTime(time); } // SeessionLocator - record.setSessionLocator(Parser.parseSessionLocator(data)); + record.setSessionLocator(parseSessionLocator(data)); // Accessor - record.setAccessor(Parser.parseAccessor(data)); + record.setAccessor(parseAccessor(data)); - Parser.parseSessionId(record); + record.setSessionId(Constants.UNKNOWN_STRING); // Data if (data.get(Constants.LOG_LEVEL).toString().contains("INFO")) { - record.setData(Parser.parseData(data)); + record.setData(parseData(data)); } else { - record.setException(Parser.parseException(data)); + record.setException(parseException(data)); } } @@ -94,7 +108,7 @@ public static Record parseRecord(final JsonObject data) throws ParseException { // ---------------------- Session Id ----------------------- - public static void parseSessionId(Record record) { + public void parseSessionId(Record record) { Integer hashCode = (record.getSessionLocator().getClientIp() + record.getSessionLocator().getClientPort() + record.getDbName()).hashCode(); @@ -103,7 +117,7 @@ public static void parseSessionId(Record record) { // -----------------------------------------------Accessor----------------- - public static Accessor parseAccessor(JsonObject data) { + public Accessor parseAccessor(JsonObject data) { Accessor accessor = new Accessor(); accessor.setDbProtocol(Constants.DATA_PROTOCOL_STRING); @@ -142,7 +156,7 @@ public static Accessor parseAccessor(JsonObject data) { // -----------------------------------------------SessionLocator----------------- - public static SessionLocator parseSessionLocator(JsonObject data) { + public SessionLocator parseSessionLocator(JsonObject data) { SessionLocator sessionLocator = new SessionLocator(); sessionLocator.setIpv6(false); @@ -195,7 +209,7 @@ public static SessionLocator parseSessionLocator(JsonObject data) { return sessionLocator; } - private static boolean isIPv6Address(String ip) { + private boolean isIPv6Address(String ip) { try { InetAddress inetAddress = InetAddress.getByName(ip); return inetAddress instanceof java.net.Inet6Address; @@ -206,7 +220,7 @@ private static boolean isIPv6Address(String ip) { // -----------------------------------------------Timestamp----------------- - public static String parseTimestamp(final JsonObject data) { + public String parseTimestamp(final JsonObject data) { String dateString = null; if (data.has(Constants.TIMESTAMP)) { dateString = data.get(Constants.TIMESTAMP).getAsString(); @@ -215,7 +229,7 @@ public static String parseTimestamp(final JsonObject data) { } - public static Time getTime(String dateString, String timeZone) { + public Time getTime(String dateString, String timeZone) { if (dateString != null) { JsonObject data = new JsonObject(); LocalDateTime dt = LocalDateTime.parse(dateString, DATE_TIME_FORMATTER); @@ -233,16 +247,19 @@ public static Time getTime(String dateString, String timeZone) { // -----------------------------------------------Exception----------------- - public static ExceptionRecord parseException(JsonObject data) { + public ExceptionRecord parseException(JsonObject data) { ExceptionRecord exceptionRecord = new ExceptionRecord(); exceptionRecord.setExceptionTypeId(Constants.SQL_ERROR); String queryStatement = data.get(Constants.QUERY_STATEMENT).getAsString(); String[] queryData = queryStatement.split("'} - "); + if(queryData.length == 1){ + queryData = queryStatement.split("} - "); + } String query = queryData[0].toString(); - String exceptionDes = queryData[1].toString(); + String exceptionDes = queryData[queryData.length - 1].toString(); exceptionRecord.setDescription(exceptionDes); exceptionRecord.setSqlString(query); @@ -251,7 +268,7 @@ public static ExceptionRecord parseException(JsonObject data) { // -----------------------------------------------Data Construct---------------- - public static Data parseData(JsonObject inputJSON) { + public Data parseData(JsonObject inputJSON) { Data data = new Data(); try { Construct construct = parseAsConstruct(inputJSON); @@ -272,9 +289,9 @@ public static Data parseData(JsonObject inputJSON) { return data; } - public static Construct parseAsConstruct(final JsonObject data) { + public Construct parseAsConstruct(final JsonObject data) { try { - final Sentence sentence = Parser.parseSentence(data); + final Sentence sentence = parseSentence(data); final Construct construct = new Construct(); construct.sentences.add(sentence); @@ -286,14 +303,14 @@ public static Construct parseAsConstruct(final JsonObject data) { construct.setFullSql(fullSql[0].substring(1).trim()); else construct.setFullSql(query[0].substring(1).trim()); - construct.setRedactedSensitiveDataSql(Parser.parseRedactedSensitiveDataSql(data)); + construct.setRedactedSensitiveDataSql(parseRedactedSensitiveDataSql(data)); return construct; } catch (final Exception e) { throw e; } } - protected static Sentence parseSentence(final JsonObject data) { + protected Sentence parseSentence(final JsonObject data) { Sentence sentence = null; @@ -301,7 +318,7 @@ protected static Sentence parseSentence(final JsonObject data) { String queryStatement = data.get(Constants.QUERY_STATEMENT).getAsString(); sentence = parseQuery(queryStatement.trim()); - + //For Match (n) return n if(sentence == null) { Sentence validSentence = new Sentence("MATCH"); @@ -317,7 +334,7 @@ protected static Sentence parseSentence(final JsonObject data) { return sentence; } - public static Sentence parseQuery(String query) { + public Sentence parseQuery(String query) { ArrayList gfg = new ArrayList() { { @@ -415,42 +432,41 @@ else if (queryString.startsWith(Constants.DETACH_DELET)) List alKeys = new ArrayList(operations.keySet()); SentenceObject sentenceObject = null; - - int j = 1; Sentence originalSentence = null; + int j = 1; + + if (operations.isEmpty() || variables.isEmpty() || alKeys.isEmpty()) return originalSentence; for (String keys : alKeys) { - ArrayList operation = operations.get(keys); + if (!operations.containsKey(keys)) continue; + String name = Constants.EVERYTHING; + if (variables.containsKey(keys) && variables.get(keys) != null) + name = variables.get(keys); + ArrayList operation = operations.get(keys); for (String string : operation) { - - if(j == 1) { + if (j == 1) { originalSentence = new Sentence(string); - sentenceObject = new SentenceObject(variables.get(keys)); - if(sentenceObject.getName() == null) - sentenceObject.setName(Constants.EVERYTHING); + sentenceObject = new SentenceObject(name); sentenceObject.setType(Constants.TYPE); // In graph database, graphs are equivalent to tables in RDBMS originalSentence.getObjects().add(sentenceObject); j++; - } - else { + } else { Sentence decendant = new Sentence(string); - sentenceObject = new SentenceObject(variables.get(keys)); - if(sentenceObject.getName() == null) - sentenceObject.setName(Constants.EVERYTHING); + sentenceObject = new SentenceObject(name); sentenceObject.setType(Constants.TYPE); // In graph database, graphs are equivalent to tables in RDBMS decendant.getObjects().add(sentenceObject); originalSentence.getDescendants().add(decendant); + } } } - } return originalSentence; } - private static void remove(String queryString) { + private void remove(String queryString) { ArrayList operationPerfomed = new ArrayList<>(); String arr[] = queryString.split(Constants.REMOVE); @@ -471,7 +487,7 @@ private static void remove(String queryString) { operations.put(alias, operationPerfomed); } - private static void delete(String queryString) { + private void delete(String queryString) { ArrayList operationPerfomed = new ArrayList<>(); String arr[] = queryString.split(Constants.DELETE); @@ -488,7 +504,7 @@ private static void delete(String queryString) { operations.put(alias, operationPerfomed); } - private static void detachDelete(String queryString) { + private void detachDelete(String queryString) { ArrayList operationPerfomed = new ArrayList<>(); String arr[] = queryString.split(Constants.DETACH_DELET); @@ -505,7 +521,7 @@ private static void detachDelete(String queryString) { operations.put(alias, operationPerfomed); } - private static void onMatchSet(String queryString) { + private void onMatchSet(String queryString) { ArrayList operationPerfomed = new ArrayList<>(); String arr[] = queryString.split(Constants.ON_MATC_ST); @@ -519,7 +535,7 @@ private static void onMatchSet(String queryString) { operations.put(alias, operationPerfomed); } - private static void onCreateSet(String queryString) { + private void onCreateSet(String queryString) { ArrayList operationPerfomed = new ArrayList<>(); String arr[] = queryString.split(Constants.ON_CREAT_ST); @@ -534,7 +550,7 @@ private static void onCreateSet(String queryString) { } - private static void set(String queryString) { + private void set(String queryString) { ArrayList operationPerfomed = new ArrayList<>(); String arr[] = queryString.split(Constants.SET); @@ -548,7 +564,7 @@ private static void set(String queryString) { operations.put(alias, operationPerfomed); } - private static void match(String query) { + private void match(String query) { String closingBracket = query; @@ -566,7 +582,7 @@ else if (squareIndex != -1 && (roundIndex > squareIndex || roundIndex == -1)) } - private static void create(String query) { + private void create(String query) { String closingBracket = query; @@ -584,7 +600,7 @@ else if (squareIndex != -1 && (roundIndex > squareIndex || roundIndex == -1)) } - private static void merge(String query) { + private void merge(String query) { String closingBracket = query; @@ -608,8 +624,8 @@ else if (squareIndex != -1 && (roundIndex > squareIndex || roundIndex == -1)) * and nodeName. Operation map is used to store the alias and operation * performed. */ - - private static String squareBracket(String query, String operation) { + + private String squareBracket(String query, String operation) { String closingBracket = ""; String value = ""; @@ -657,7 +673,7 @@ private static String squareBracket(String query, String operation) { } - protected static String roundBracket(String query, String operation) { + protected String roundBracket(String query, String operation) { String closingBracket = ""; String value = ""; @@ -708,7 +724,7 @@ protected static String roundBracket(String query, String operation) { } - protected static String parseRedactedSensitiveDataSql(JsonObject data) { + protected String parseRedactedSensitiveDataSql(JsonObject data) { String redactedData = ""; redactedData = data.get(Constants.MESSAGE).getAsString(); diff --git a/filter-plugin/logstash-filter-neo4j-guardium/src/test/java/com/ibm/guardium/neodb/ParserTest.java b/filter-plugin/logstash-filter-neo4j-guardium/src/test/java/com/ibm/guardium/neodb/ParserTest.java index 0c69b8428..7f591e6bc 100644 --- a/filter-plugin/logstash-filter-neo4j-guardium/src/test/java/com/ibm/guardium/neodb/ParserTest.java +++ b/filter-plugin/logstash-filter-neo4j-guardium/src/test/java/com/ibm/guardium/neodb/ParserTest.java @@ -7,10 +7,21 @@ import co.elastic.logstash.api.Event; import com.google.gson.JsonObject; -import com.ibm.guardium.universalconnector.commons.structures.*; +import com.ibm.guardium.universalconnector.commons.structures.Accessor; +import com.ibm.guardium.universalconnector.commons.structures.Construct; +import com.ibm.guardium.universalconnector.commons.structures.Data; +import com.ibm.guardium.universalconnector.commons.structures.ExceptionRecord; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import com.ibm.guardium.universalconnector.commons.structures.Sentence; +import com.ibm.guardium.universalconnector.commons.structures.SentenceObject; +import com.ibm.guardium.universalconnector.commons.structures.SessionLocator; +import com.ibm.guardium.universalconnector.commons.structures.Time; +import com.ibm.guardium.universalconnector.commons.structures.Record; import org.junit.Assert; import org.junit.Test; - +import java.util.ArrayList; +import java.util.List; +import java.util.LinkedHashMap; import java.util.HashMap; import java.util.Map; import java.util.regex.Matcher; @@ -34,19 +45,20 @@ public class ParserTest { @Test public void testParseAsConstruct_Match() { - + Event e = getParsedEvent(neoSuccessString_grokOutput, neoSuccessString); - + JsonObject inputData = inputData(e); - final Construct result = Parser.parseAsConstruct(inputData); - + Parser parser = new Parser(); + final Construct result = parser.parseAsConstruct(inputData); + final Sentence sentence = result.sentences.get(0); - + Assert.assertEquals("MATCH", sentence.getVerb().trim()); Assert.assertEquals("player", sentence.getObjects().get(0).name.trim()); Assert.assertEquals("graph", sentence.getObjects().get(0).type); } - + @Test public void testParseAsConstruct_Create() { String neoString = "2021-08-06 15:57:11.502+0000 INFO 2 ms: (planning: 1, waiting: 0) - 1704 B - 0 page hits, 0 page faults - bolt-session bolt neo4j-browser/v4.3.1 client/127.0.0.1:54356 server/127.0.0.1:11004> neo4j - neo4j - CREATE (friend:Person {name: 'Mark'}) RETURN friend - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}"; @@ -63,18 +75,19 @@ public void testParseAsConstruct_Create() { " \"queryStatement\": \"CREATE (friend:Person {name: 'Mark'}) RETURN friend - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}\"\n" + " }"; Event e = getParsedEvent(neoString_grokOutput,neoString); - + JsonObject inputData = inputData(e); - - final Construct result = Parser.parseAsConstruct(inputData); - + + Parser parser = new Parser(); + final Construct result = parser.parseAsConstruct(inputData); + final Sentence sentence = result.sentences.get(0); - + Assert.assertEquals("CREATE", sentence.getVerb().trim()); Assert.assertEquals("Person", sentence.getObjects().get(0).name.trim()); Assert.assertEquals("graph", sentence.getObjects().get(0).type); } - + @Test public void testParseAsConstruct_Merge() { String neoString = "2021-08-06 15:56:12.097+0000 INFO 4 ms: (planning: 1, waiting: 0) - 0 B - 6 page hits, 0 page faults - bolt-session bolt neo4j-browser/v4.3.1 client/127.0.0.1:54356 server/127.0.0.1:11004> neo4j - neo4j - MERGE (mark:Person {name: 'Mark'}) RETURN mark - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}"; @@ -93,16 +106,17 @@ public void testParseAsConstruct_Merge() { Event e = getParsedEvent(neoString_grokOutput,neoString); e.setField("minoff", "+04:00"); JsonObject inputData = inputData(e); - - final Construct result = Parser.parseAsConstruct(inputData); - + + Parser parser = new Parser(); + final Construct result = parser.parseAsConstruct(inputData); + final Sentence sentence = result.sentences.get(0); - + Assert.assertEquals("MERGE", sentence.getVerb().trim()); Assert.assertEquals("Person", sentence.getObjects().get(0).name.trim()); Assert.assertEquals("graph", sentence.getObjects().get(0).type); } - + @Test public void testParseException_DELETE() { String neoString = "2021-03-03 08:49:32.367+0000 ERROR 7 ms: (planning: 7, waiting: 0) - 0 B - 0 page hits, 0 page faults - bolt-session bolt neo4j-browser/v4.2.1 client/127.0.0.1:62845 server/127.0.0.1:7687> - neo4j - DETACH DELETE node - {} - runtime=null - {type: 'user-action', app: 'neo4j-browser_v4.2.1'} - Variable `node` not defined (line 1, column 23 (offset: 22))"; @@ -121,53 +135,56 @@ public void testParseException_DELETE() { Event e = getParsedEvent(neoString_grokOutput, neoString); e.setField("minoff", "+07:00"); JsonObject inputData = inputData(e); - - final ExceptionRecord exceptionRecord = Parser.parseException(inputData); + + Parser parser = new Parser(); + final ExceptionRecord exceptionRecord = parser.parseException(inputData); Assert.assertEquals("Variable `node` not defined (line 1, column 23 (offset: 22))", exceptionRecord.getDescription().trim()); Assert.assertEquals("DETACH DELETE node - {} - runtime=null - {type: 'user-action', app: 'neo4j-browser_v4.2.1", exceptionRecord.getSqlString().trim()); } - + @Test public void testParseAccessor() { Event e = getParsedEvent(neoSuccessString_grokOutput,neoSuccessString); - JsonObject inputData = inputData(e); - - final Accessor accessor = Parser.parseAccessor(inputData); - - Assert.assertEquals("Bolt database protocol", accessor.getDbProtocol().toString().trim()); + + Parser parser = new Parser(); + final Accessor accessor = parser.parseAccessor(inputData); + + Assert.assertEquals("BoltDB", accessor.getDbProtocol().toString().trim()); Assert.assertEquals("NEO4J", accessor.getServerType().toString().trim()); Assert.assertEquals("neo4j", accessor.getDbUser().toString().trim()); Assert.assertEquals("FREE_TEXT", accessor.getLanguage().toString().trim()); } - + @Test public void testParseSessionLocator() { Event e = getParsedEvent(neoSuccessString_grokOutput, neoSuccessString); - + JsonObject inputData = inputData(e); - - final SessionLocator sessionLocator = Parser.parseSessionLocator(inputData); - + + Parser parser = new Parser(); + final SessionLocator sessionLocator = parser.parseSessionLocator(inputData); + Assert.assertEquals("127.0.0.1", sessionLocator.getClientIp().toString().trim()); Assert.assertEquals(51372, sessionLocator.getClientPort()); Assert.assertEquals("127.0.0.1", sessionLocator.getServerIp().toString().trim()); Assert.assertEquals(11004, sessionLocator.getServerPort()); - + } - + @Test public void testParseTimestamp() { Event e = getParsedEvent(neoSuccessString_grokOutput, neoSuccessString); e.setField(Constants.TIMESTAMP, "2021-01-25 11:17:09.099+0000"); JsonObject inputData = inputData(e); - - final String timestamp = Parser.parseTimestamp(inputData); - + + Parser parser = new Parser(); + final String timestamp = parser.parseTimestamp(inputData); + Assert.assertEquals("2021-01-25 11:17:09.099+0000", timestamp); } @@ -177,47 +194,58 @@ public void testGetTime() { String dateString = "2021-01-25 11:17:09.099+0000"; String timeZone = "-04:00"; - final Time time = Parser.getTime(dateString, timeZone); + + Parser parser = new Parser(); + final Time time = parser.getTime(dateString, timeZone); Assert.assertEquals(0, time.getMinDst()); Assert.assertEquals(-240, time.getMinOffsetFromGMT()); Assert.assertEquals(1611587829099L, time.getTimstamp()); } - + @Test public void testParseSentence() { Event e = getParsedEvent(neoSuccessString_grokOutput, neoSuccessString); - + JsonObject inputData = inputData(e); - - final Sentence sentence = Parser.parseSentence(inputData); - + + Parser parser = new Parser(); + final Sentence sentence = parser.parseSentence(inputData); + Assert.assertEquals("MATCH", sentence.getVerb()); Assert.assertEquals("player", sentence.getObjects().get(0).name.trim()); Assert.assertEquals("graph", sentence.getObjects().get(0).type); - + } - + @Test public void testParseRedactedSensitiveDataSql() { - + Event e = getParsedEvent(neoSuccessString_grokOutput, neoSuccessString); - + JsonObject inputData = inputData(e); - - final String redacted = Parser.parseRedactedSensitiveDataSql(inputData); - + + Parser parser = new Parser(); + final String redacted = parser.parseRedactedSensitiveDataSql(inputData); + Assert.assertEquals("2021-08-06 17:09:40.008+0000 INFO 9 ms: (planning: 1, waiting: 0) - 0 B - 4 page hits, 0 page faults - bolt-session bolt neo4j-browser/v4.3.1 client/127.0.0.1:51372 server/127.0.0.1:11004> neo4j - neo4j - MATCH (Ishant:player {name: 'Ishant Sharma', YOB: 1988, POB: 'Delhi'}) DETACH DELETE Ishant - {} - runtime=slotted - {type: 'user-direct', app: 'neo4j-browser_v4.3.1'}", redacted); } - + + @Test + public void testEmptyOperations() { + LinkedHashMap> operations = new LinkedHashMap>(); + List alKeys = new ArrayList(operations.keySet()); + Assert.assertEquals(true, alKeys.isEmpty()); + } + // ----------------------------------- --------------------------------------------------- - + private JsonObject inputData(Event e){ JsonObject data = new JsonObject(); - + if(e.getField(Constants.CLIENT_IP).toString() != null && !e.getField(Constants.CLIENT_IP).toString().isEmpty()){ data.addProperty(Constants.CLIENT_IP, e.getField(Constants.CLIENT_IP).toString()); } diff --git a/filter-plugin/logstash-filter-neptune-aws-guardium/build.gradle b/filter-plugin/logstash-filter-neptune-aws-guardium/build.gradle index 7b341fc76..c6a848738 100644 --- a/filter-plugin/logstash-filter-neptune-aws-guardium/build.gradle +++ b/filter-plugin/logstash-filter-neptune-aws-guardium/build.gradle @@ -2,6 +2,36 @@ import java.nio.file.Files import static java.nio.file.StandardCopyOption.REPLACE_EXISTING apply plugin: 'java' + +buildscript { + + ext { + snakeYamlVersion = '2.2' + } + + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + + dependencies { + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' + classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.1.0" + classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' + + } +} + +def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream()) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" // =========================================================================== @@ -20,10 +50,10 @@ pluginInfo.pluginClass = "NeptuneGuardiumFilter" pluginInfo.pluginName = "neptune_guardium_filter" // must match the @LogstashPlugin annotation in the main plugin class // =========================================================================== -sourceCompatibility = JavaVersion.VERSION_11 -targetCompatibility = JavaVersion.VERSION_11 +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 -def jacocoVersion = '0.8.4' +def jacocoVersion = '0.8.11' // minimumCoverage can be set by Travis ENV def minimumCoverageStr = System.getenv("MINIMUM_COVERAGE") ?: "50.0%" if (minimumCoverageStr.endsWith("%")) { @@ -31,28 +61,16 @@ if (minimumCoverageStr.endsWith("%")) { } def minimumCoverage = Float.valueOf(minimumCoverageStr) / 100 -buildscript { - repositories { - maven { - url "https://plugins.gradle.org/m2/" - } - mavenCentral() - jcenter() - } - - dependencies { - classpath 'com.github.jengelman.gradle.plugins:shadow:4.0.4' - classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.0.0-RC2" - classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' - - } -} - -def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); -def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream()) repositories { - mavenCentral() + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } } tasks.register("vendor"){ @@ -63,14 +81,13 @@ tasks.register("vendor"){ File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") projectJarFile.mkdirs() Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) - validatePluginJar(projectJarFile, project.group) } } apply plugin: 'com.github.johnrengelman.shadow' shadowJar { - classifier = null + archiveClassifier = null } dependencies { @@ -78,8 +95,10 @@ dependencies { implementation group: 'org.apache.logging.log4j', name: 'log4j-api', version: versions.dependencies.log4jApi implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: versions.dependencies.log4jCore implementation 'com.google.code.gson:gson:' + versions.dependencies.gson + implementation group: 'commons-beanutils', name: 'commons-beanutils', version: versions.dependencies.commonsBeanutils implementation group: 'org.apache.commons', name: 'commons-text', version: versions.dependencies.commonsText implementation group: 'org.apache.tinkerpop', name: 'tinkergraph-gremlin', version: versions.dependencies.tinkergraphGremlin + implementation group: 'org.apache.commons', name: 'commons-configuration2', version: '2.13.0' implementation group: 'org.eclipse.rdf4j', name: 'rdf4j-queryparser-sparql', version: versions.dependencies.rdf4jQueryparserSparql implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "build/libs/logstash-core-*.*.*.jar") @@ -153,17 +172,16 @@ apply plugin: "org.barfuin.gradle.jacocolog" // ------------------------------------ jacoco { toolVersion = "${jacocoVersion}" - reportsDir = file("$buildDir/reports/jacoco") } jacocoTestReport { // You will see "Report -> file://...." at the end of a JaCoCo build // If no output, run this first: ./gradlew test reports { - html.enabled true - xml.enabled true - csv.enabled true - html.destination file("${buildDir}/reports/jacoco") - csv.destination file("${buildDir}/reports/jacoco/all.csv") + html.required = true + xml.required = true + csv.required = true + html.outputLocation = file("${buildDir}/reports/jacoco") + csv.outputLocation = file("${buildDir}/reports/jacoco/all.csv") } executionData.from fileTree(dir: "${buildDir}/jacoco/", includes: [ '**/*.exec' diff --git a/filter-plugin/logstash-filter-neptune-aws-guardium/gradle/wrapper/gradle-wrapper.properties b/filter-plugin/logstash-filter-neptune-aws-guardium/gradle/wrapper/gradle-wrapper.properties index 60c76b340..ba9ccfe4c 100644 --- a/filter-plugin/logstash-filter-neptune-aws-guardium/gradle/wrapper/gradle-wrapper.properties +++ b/filter-plugin/logstash-filter-neptune-aws-guardium/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists \ No newline at end of file diff --git a/filter-plugin/logstash-filter-onPremGreenplumdb-guardium/README.md b/filter-plugin/logstash-filter-onPremGreenplumdb-guardium/README.md index e73586f96..a132e9281 100644 --- a/filter-plugin/logstash-filter-onPremGreenplumdb-guardium/README.md +++ b/filter-plugin/logstash-filter-onPremGreenplumdb-guardium/README.md @@ -70,7 +70,8 @@ Filebeat must be configured to send the output to the chosen Logstash host and p ``` • Locate "filebeat.inputs" in the filebeat.yml file, then add the following parameters. filebeat.inputs: - - type: log + - type: filestream + - id: enabled: true paths: - /home/ec2-user/greenplum-db-node/gpmaster/gpsne-1/pg_log/*.csv diff --git a/filter-plugin/logstash-filter-onPremGreenplumdb-guardium/build.gradle b/filter-plugin/logstash-filter-onPremGreenplumdb-guardium/build.gradle index 38a45322c..ed5f51c2a 100644 --- a/filter-plugin/logstash-filter-onPremGreenplumdb-guardium/build.gradle +++ b/filter-plugin/logstash-filter-onPremGreenplumdb-guardium/build.gradle @@ -2,6 +2,30 @@ import java.nio.file.Files import static java.nio.file.StandardCopyOption.REPLACE_EXISTING apply plugin: 'java' + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + + dependencies { + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' + classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.1.0" + classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' + } +} + +def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + apply plugin: 'jacoco' apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" @@ -22,10 +46,10 @@ pluginInfo.pluginClass = "GreenplumdbGuardiumFilter" pluginInfo.pluginName = "greenplumdb_guardium_filter" // must match the @LogstashPlugin annotation in the main plugin class // =========================================================================== -sourceCompatibility = 1.8 -targetCompatibility = 1.8 +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 -def jacocoVersion = '0.8.4' +def jacocoVersion = '0.8.11' // minimumCoverage can be set by Travis ENV def minimumCoverageStr = System.getenv("MINIMUM_COVERAGE") ?: "50.0%" if (minimumCoverageStr.endsWith("%")) { @@ -33,27 +57,16 @@ if (minimumCoverageStr.endsWith("%")) { } def minimumCoverage = Float.valueOf(minimumCoverageStr) / 100 -buildscript { - repositories { - maven { - url "https://plugins.gradle.org/m2/" - } - mavenCentral() - jcenter() - } - - dependencies { - classpath 'com.github.jengelman.gradle.plugins:shadow:4.0.4' - classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.0.0-RC2" - classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' - } -} - -def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); -def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) repositories { - mavenCentral() + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } } tasks.register("vendor"){ @@ -64,14 +77,13 @@ tasks.register("vendor"){ File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") projectJarFile.mkdirs() Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) - validatePluginJar(projectJarFile, project.group) } } apply plugin: 'com.github.johnrengelman.shadow' shadowJar { - classifier = null + archiveClassifier = null } @@ -81,6 +93,7 @@ dependencies { implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: versions.dependencies.log4jCore implementation 'org.apache.commons:commons-lang3:' + versions.dependencies.commonsLang implementation 'com.google.code.gson:gson:' + versions.dependencies.gson + implementation group: 'commons-beanutils', name: 'commons-beanutils', version: versions.dependencies.commonsBeanutils implementation group: 'org.apache.commons', name: 'commons-text', version: versions.dependencies.commonsText implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "build/libs/logstash-core-*.*.*.jar") implementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "common-*.*.*.jar") @@ -105,6 +118,17 @@ tasks.withType(JavaCompile) { options.encoding = 'UTF-8' } +test { + jvmArgs = [ + '--add-opens=java.base/java.lang=ALL-UNNAMED', + '--add-opens=java.base/java.util=ALL-UNNAMED', + '--add-opens=java.base/java.text=ALL-UNNAMED', + '--add-opens=java.base/java.lang.reflect=ALL-UNNAMED', + '--add-opens=java.base/sun.util.resources=ALL-UNNAMED', + '--add-opens=java.base/sun.util.cldr=ALL-UNNAMED' + ] +} + tasks.register("generateRubySupportFiles") { @@ -147,17 +171,16 @@ apply plugin: "org.barfuin.gradle.jacocolog" // ------------------------------------ jacoco { toolVersion = "${jacocoVersion}" - reportsDir = file("$buildDir/reports/jacoco") } jacocoTestReport { // You will see "Report -> file://...." at the end of a JaCoCo build // If no output, run this first: ./gradlew test reports { - html.enabled true - xml.enabled true - csv.enabled true - html.destination file("${buildDir}/reports/jacoco") - csv.destination file("${buildDir}/reports/jacoco/all.csv") + html.required = true + xml.required = true + csv.required = true + html.outputLocation = file("${buildDir}/reports/jacoco") + csv.outputLocation = file("${buildDir}/reports/jacoco/all.csv") } executionData.from fileTree(dir: "${buildDir}/jacoco/", includes: [ '**/*.exec' diff --git a/filter-plugin/logstash-filter-onPremGreenplumdb-guardium/gradle/wrapper/gradle-wrapper.properties b/filter-plugin/logstash-filter-onPremGreenplumdb-guardium/gradle/wrapper/gradle-wrapper.properties index 60c76b340..ba9ccfe4c 100644 --- a/filter-plugin/logstash-filter-onPremGreenplumdb-guardium/gradle/wrapper/gradle-wrapper.properties +++ b/filter-plugin/logstash-filter-onPremGreenplumdb-guardium/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists \ No newline at end of file diff --git a/filter-plugin/logstash-filter-onPremPostgres-guardium/EDBPostgres_README.md b/filter-plugin/logstash-filter-onPremPostgres-guardium/EDBPostgres_README.md index 448d5717e..0b7b9ec2a 100644 --- a/filter-plugin/logstash-filter-onPremPostgres-guardium/EDBPostgres_README.md +++ b/filter-plugin/logstash-filter-onPremPostgres-guardium/EDBPostgres_README.md @@ -46,7 +46,8 @@ https://www.elastic.co/guide/en/beats/filebeat/current/directory-layout.html For example:- filebeat.inputs: - - type: log + - type: filestream + - id: enabled: true paths: - diff --git a/filter-plugin/logstash-filter-onPremPostgres-guardium/FEPostgres_README.md b/filter-plugin/logstash-filter-onPremPostgres-guardium/FEPostgres_README.md index 6d2e1bd43..bb13f96c9 100644 --- a/filter-plugin/logstash-filter-onPremPostgres-guardium/FEPostgres_README.md +++ b/filter-plugin/logstash-filter-onPremPostgres-guardium/FEPostgres_README.md @@ -57,7 +57,8 @@ https://www.elastic.co/guide/en/beats/filebeat/current/directory-layout.html For example:- filebeat.inputs: - - type: log + - type: filestream + - id: enabled: true paths: - diff --git a/filter-plugin/logstash-filter-opensearch-guardium/CHANGELOG.md b/filter-plugin/logstash-filter-opensearch-guardium/CHANGELOG.md new file mode 100644 index 000000000..143dd1602 --- /dev/null +++ b/filter-plugin/logstash-filter-opensearch-guardium/CHANGELOG.md @@ -0,0 +1,10 @@ + +# Changelog +Notable changes will be documented in this file. + + + +## [] + +### Added +- Initial release, in parallel to Guardium . diff --git a/filter-plugin/logstash-filter-opensearch-guardium/LICENSE b/filter-plugin/logstash-filter-opensearch-guardium/LICENSE new file mode 100644 index 000000000..261eeb9e9 --- /dev/null +++ b/filter-plugin/logstash-filter-opensearch-guardium/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/filter-plugin/logstash-filter-opensearch-guardium/OpenSearchOverCloudwatchPackage/opensearch.conf b/filter-plugin/logstash-filter-opensearch-guardium/OpenSearchOverCloudwatchPackage/opensearch.conf new file mode 100644 index 000000000..1d877aa00 --- /dev/null +++ b/filter-plugin/logstash-filter-opensearch-guardium/OpenSearchOverCloudwatchPackage/opensearch.conf @@ -0,0 +1,21 @@ +input{ + cloudwatch_logs { + log_group => [""] #example: ["/aws/rds/instance/mariadb-aws-database/audit"] + region => "" #Region that has the DB,example: ap-south-1a + codec => plain + sincedb_path => "NUL" + access_key_id => "" + secret_access_key => "" + type => "opensearch" + event_filter => '' + start_position => "end" + add_field => {"account_id" => ""} + type => "opensearch" + } +} +filter { + if ([type] == "opensearch"){ + opensearch_guardium_filter{} + } +} + diff --git a/filter-plugin/logstash-filter-opensearch-guardium/README.md b/filter-plugin/logstash-filter-opensearch-guardium/README.md new file mode 100644 index 000000000..5ab06a551 --- /dev/null +++ b/filter-plugin/logstash-filter-opensearch-guardium/README.md @@ -0,0 +1,96 @@ +# Amazon OpenSearch - Guardium Logstash filter plug-in + +### Meet OpenSearch + +* Tested versions: v1 +* Environment: AWS +* Supported inputs: CloudWatch (pull) +* Supported Guardium versions: + * Guardium Data Protection 12.2 and later + +This is a [Logstash](https://github.com/elastic/logstash) filter plug-in for the universal connector that is featured in +IBM Security Guardium. It parses events and messages from the Amazon OpenSearch audit log into +a Guardium Record. + +The plug-in is free and open-source (Apache 2.0). It can be used as a starting point to develop additional filter +plug-ins for Guardium universal connector. + +## Configuration + +### OpenSearch Setup + +1. [Prerequisites](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/setting-up.html) +2. Go to https://console.aws.amazon.com/. +3. Search and navigate to ```Amazon OpenSearch Service```. +4. To create an OpenSearch domain, refer to the [Getting started with Amazon OpenSearch Service guide](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/gsg.html). + +### Enabling Audit Logs + +1. Enable audit logs for **CloudWatch Logs** and **OpenSearch Dashboard**, refer to the [Enabling Audit logs](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/audit-logs.html#audit-log-enabling). + +### Viewing Audit Logs on CloudWatch + +By default, each database instance has an associated log group with a name in this format: `/aws/OpenSearchService//audit` and `/aws/OpenSearchService//profiler`. + +#### Procedure + +1. Open the CloudWatch console https://console.aws.amazon.com/cloudwatch/. +2. In the navigation pane, choose ```Log groups```. +3. Choose the ```log group``` that you specified while enabling audit logs. Within the log group, OpenSearch Service creates a log stream for each node in your domain. +4. In the ```Log streams```, select ```Search all```. +5. For the read and write events, see the corresponding logs. This process may take several seconds. + +#### Supported Audit Log Types + +Cluster communication occurs over two separate layers: **REST layer** and **Transport layer**. The following is the list of Audit log Categories, with their availability determined by the communication layers. + +* FAILED_LOGIN +* MISSING_PRIVILEGES +* BAD_HEADERS +* SSL_EXCEPTION +* GRANTED_PRIVILEGES +* OPENSEARCH_SECURITY_INDEX_ATTEMPT +* AUTHENTICATED +* INDEX_EVENT +* COMPLIANCE_DOC_READ +* COMPLIANCE_DOC_WRITE +* COMPLIANCE_INTERNAL_CONFIG_READ +* COMPLIANCE_INTERNAL_CONFIG_WRITE + + + +For more information about the audit logging category and layers, refer to the [Audit log layers and categories](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/audit-logs.html#audit-log-layers). + +For more information about the audit logging fields, refer to the [Audit log field reference](https://docs.opensearch.org/docs/latest/security/audit-logs/field-reference/). + +**Note:** OpenSearch generates a large volume of background audit logs by default. We recommend configuring the audit settings appropriately to limit unnecessary entries in the audit logs. + +### Limitations +- Audit logging in OpenSearch can be accessed in two different ways – via the OpenSearch Dashboards or through CloudWatch Logs. However, this filter plugin only parses and processes audit logs that are streamed to CloudWatch. Audit logs stored directly in OpenSearch indices or viewed in the Dashboards are not supported for parsing. +- FAILED_LOGIN REST messages will appear in 'Full SQL' and 'Failed Logins' report. +- Certain reserved keywords (template, mappings, get, aliases, user) are automatically prefixed with an underscore (_) during sanitization to prevent OpenSearch URI parsing errors or endpoint conflicts. +- ClientHostName is not available in the audit logs for OpenSearch. + +## Guardium Data Protection + +The Guardium universal connector is the Guardium entry point for native audit/data_access logs. The Guardium universal connector identifies and parses the received events, and converts them to a standard Guardium format. The output of the Guardium universal connector is forwarded to the Guardium sniffer on the collector, for policy and auditing enforcements. + +### Before you begin +* Configure the policies you require. See [policies](/docs/#policies) for more information. +* You must have permission for the S-Tap Management role. The admin user includes this role by default +* Download the [logstash-filter-aws_opensearch_guardium_filter](https://github.com/IBM/universal-connectors/releases/download/v1.7.0/logstash-filter-opensearch_guardium_filter.zip) plug-in. + +### Procedure +1. On the collector, go to ```Setup``` > ```Tools and Views``` > ```Configure Universal Connector```. +2. Enable the universal connector if it is disabled. +3. Click ```Upload File``` and select the offline [logstash-filter-aws_opensearch_guardium_filter](https://github.com/IBM/universal-connectors/releases/download/v1.7.0/logstash-filter-opensearch_guardium_filter.zip) plug-in. After it is uploaded, click ```OK```. +4. Click ```Upload File``` and select the key.json file. After it is uploaded, click ```OK```. +5. Click the Plus sign to open the Connector Configuration dialog box. +6. Type a name in the Connector name field. +7. Update the input section to add the details from the [opensearch.conf](OpenSearchOverCloudwatchPackage/opensearch.conf) file's input part, omitting the keyword "input{" at the beginning and its corresponding "}" at the end. +8. Update the filter section to add the details from the [opensearch.conf](OpenSearchOverCloudwatchPackage/opensearch.conf) file's filter part, omitting the keyword "filter{" at the beginning and its corresponding "}" at the end. +9. The 'type' fields should match in the input and filter configuration sections. This field should be unique for every individual connector added. +10. Click ```Save```. Guardium validates the new connector and displays it in the Configure Universal Connector page. +11. After the offline plug-in is installed and the configuration is uploaded and saved in the Guardium machine, restart the Universal Connector using the ```Disable/Enable``` button. + + diff --git a/filter-plugin/logstash-filter-opensearch-guardium/VERSION b/filter-plugin/logstash-filter-opensearch-guardium/VERSION new file mode 100644 index 000000000..afaf360d3 --- /dev/null +++ b/filter-plugin/logstash-filter-opensearch-guardium/VERSION @@ -0,0 +1 @@ +1.0.0 \ No newline at end of file diff --git a/filter-plugin/logstash-filter-opensearch-guardium/build.gradle b/filter-plugin/logstash-filter-opensearch-guardium/build.gradle new file mode 100644 index 000000000..9828188fc --- /dev/null +++ b/filter-plugin/logstash-filter-opensearch-guardium/build.gradle @@ -0,0 +1,211 @@ +import java.nio.file.Files +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING + +apply plugin: 'java' +apply plugin: 'jacoco' + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + + dependencies { + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' + classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.1.0" + classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' + } + +} + +def universalConnectorsDir = project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load(new File("${universalConnectorsDir}/versions.yml").newInputStream()) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + +apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" +apply plugin: "eclipse" + + +// =========================================================================== +// plugin info +// =========================================================================== +group "com.ibm.guardium.aws.opensearch" // must match the package of the main plugin class +version "${file("VERSION").text.trim()}" // read from required VERSION file +description = "AWS OpenSearch Guardium Filter Plugin" +pluginInfo.licenses = ['Apache-2.0'] // list of SPDX license IDs +pluginInfo.longDescription = "This gem is a Logstash OpenSearch filter plugin required to be installed as part of IBM Security Guardium, Guardium Universal connector configuration. This gem is not a stand-alone program." +pluginInfo.authors = ['IBM', '', ''] +pluginInfo.email = [''] +pluginInfo.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html" +pluginInfo.pluginType = "filter" +pluginInfo.pluginClass = "OpensearchGuardiumFilter" +pluginInfo.pluginName = "opensearch_guardium_filter" // must match the @LogstashPlugin annotation in the main plugin class +// =========================================================================== + +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 + +def jacocoVersion = '0.8.11' +// minimumCoverage can be set by Travis ENV +def minimumCoverageStr = System.getenv("MINIMUM_COVERAGE") ?: "50.0%" +if (minimumCoverageStr.endsWith("%")) { + minimumCoverageStr = minimumCoverageStr.substring(0, minimumCoverageStr.length() - 1) +} +def minimumCoverage = Float.valueOf(minimumCoverageStr) / 100 + + +repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } +} + +tasks.register("vendor") { + dependsOn shadowJar + doLast { + String vendorPathPrefix = "vendor/jar-dependencies" + String projectGroupPath = project.group.replaceAll('\\.', '/') + File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") + projectJarFile.mkdirs() + Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) + } +} + +apply plugin: 'com.github.johnrengelman.shadow' + +shadowJar { + archiveClassifier = null +} + + +dependencies { + implementation 'com.google.code.gson:gson:' + versions.dependencies.gson + implementation 'org.apache.commons:commons-lang3:' + versions.dependencies.commonsLang + implementation 'commons-validator:commons-validator:' + versions.dependencies.commonsValidator + implementation group: 'commons-beanutils', name: 'commons-beanutils', version: versions.dependencies.commonsBeanutils + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.17.2' + + testImplementation group: 'org.mockito', name: 'mockito-all', version: versions.dependencies.mockitoAll + testImplementation 'org.junit.jupiter:junit-jupiter:' + versions.dependencies.junitJupiter + testImplementation 'org.jruby:jruby-complete:' + versions.dependencies.jrubyComplete + implementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "common-*.*.*.jar") + implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "build/libs/logstash-core-*.*.*.jar") + implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: versions.dependencies.log4jCore + implementation group: 'org.json', name: 'json', version: versions.dependencies.json + implementation group: 'org.parboiled', name: 'parboiled-java', version: versions.dependencies.parboiledJava + implementation group: 'org.glassfish', name: 'javax.json', version: versions.dependencies.javaxJson + testImplementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "common-*.*.*.jar") + testImplementation 'junit:junit:4.13.1' +} + +clean { + delete "${projectDir}/Gemfile" + delete "${projectDir}/" + pluginInfo.pluginFullName() + ".gemspec" + delete "${projectDir}/lib/" + delete "${projectDir}/vendor/" + new FileNameFinder().getFileNames(projectDir.toString(), pluginInfo.pluginFullName() + "-*.*.*.gem").each { filename -> + delete filename + } +} + +tasks.withType(JavaCompile) { + options.encoding = 'UTF-8' +} +test { + useJUnitPlatform() +} +tasks.register("generateRubySupportFiles") { + doLast { + generateRubySupportFilesForPlugin(project.description, project.group, version) + } +} + +tasks.register("removeObsoleteJars") { + doLast { + new FileNameFinder().getFileNames( + projectDir.toString(), + "vendor/**/" + pluginInfo.pluginFullName() + "*.jar", + "vendor/**/" + pluginInfo.pluginFullName() + "-" + version + ".jar").each { f -> + delete f + } + } +} + +tasks.register("gem") { + dependsOn = [downloadAndInstallJRuby, removeObsoleteJars, vendor, generateRubySupportFiles] + doLast { + buildGem(projectDir, buildDir, pluginInfo.pluginFullName() + ".gemspec") + } +} + +tasks.register("copyDependencyLibs", Copy) { + into "dependenciesLib" + from configurations.compileClasspath + from configurations.runtimeClasspath + from configurations.testCompileClasspath + from configurations.testRuntimeClasspath +} + +apply plugin: 'jacoco' +//apply plugin: 'org.barfuin.gradle.jacocolog' version '2.0.0' +apply plugin: "org.barfuin.gradle.jacocolog" +// ------------------------------------ +// JaCoCo is a code coverage tool +// ------------------------------------ +jacoco { + toolVersion = "${jacocoVersion}" +} +jacocoTestReport { + // You will see "Report -> file://...." at the end of a JaCoCo build + // If no output, run this first: ./gradlew test + reports { + html.required = true + xml.required = true + csv.required = true + html.outputLocation = file("${buildDir}/reports/jacoco") + csv.outputLocation = file("${buildDir}/reports/jacoco/all.csv") + } + executionData.from fileTree(dir: "${buildDir}/jacoco/", includes: [ + '**/*.exec' + ]) + afterEvaluate { + // objective is to test TicketingService class + classDirectories.setFrom(files(classDirectories.files.collect { + fileTree(dir: it, exclude: []) + })) + } + doLast { + println "Report -> file://${buildDir}/reports/jacoco/index.html" + } +} +test.finalizedBy jacocoTestReport +jacocoTestCoverageVerification { + violationRules { + rule { + limit { + minimum = minimumCoverage + } + } + } + executionData.from fileTree(dir: "${buildDir}/jacoco/", includes: [ + '**/*.exec' + ]) + afterEvaluate { + // objective is to test TicketingService class + classDirectories.setFrom(files(classDirectories.files.collect { + fileTree(dir: it, exclude: []) + })) + } +} +project.tasks.check.dependsOn(jacocoTestCoverageVerification, jacocoTestReport) \ No newline at end of file diff --git a/filter-plugin/logstash-filter-opensearch-guardium/gradle/wrapper/gradle-wrapper.jar b/filter-plugin/logstash-filter-opensearch-guardium/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 000000000..7454180f2 Binary files /dev/null and b/filter-plugin/logstash-filter-opensearch-guardium/gradle/wrapper/gradle-wrapper.jar differ diff --git a/filter-plugin/logstash-filter-opensearch-guardium/gradle/wrapper/gradle-wrapper.properties b/filter-plugin/logstash-filter-opensearch-guardium/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 000000000..81aa1c044 --- /dev/null +++ b/filter-plugin/logstash-filter-opensearch-guardium/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,5 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/filter-plugin/logstash-filter-opensearch-guardium/gradlew b/filter-plugin/logstash-filter-opensearch-guardium/gradlew new file mode 100755 index 000000000..744e882ed --- /dev/null +++ b/filter-plugin/logstash-filter-opensearch-guardium/gradlew @@ -0,0 +1,185 @@ +#!/usr/bin/env sh + +# +# Copyright 2015 the original author or authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MSYS* | MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin or MSYS, switch paths to Windows format before running java +if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=`expr $i + 1` + done + case $i in + 0) set -- ;; + 1) set -- "$args0" ;; + 2) set -- "$args0" "$args1" ;; + 3) set -- "$args0" "$args1" "$args2" ;; + 4) set -- "$args0" "$args1" "$args2" "$args3" ;; + 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=`save "$@"` + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +exec "$JAVACMD" "$@" diff --git a/filter-plugin/logstash-filter-opensearch-guardium/gradlew.bat b/filter-plugin/logstash-filter-opensearch-guardium/gradlew.bat new file mode 100644 index 000000000..107acd32c --- /dev/null +++ b/filter-plugin/logstash-filter-opensearch-guardium/gradlew.bat @@ -0,0 +1,89 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto execute + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/filter-plugin/logstash-filter-opensearch-guardium/logstash-filter-opensearch_guardium_filter.zip b/filter-plugin/logstash-filter-opensearch-guardium/logstash-filter-opensearch_guardium_filter.zip new file mode 100644 index 000000000..3cef2b0e1 Binary files /dev/null and b/filter-plugin/logstash-filter-opensearch-guardium/logstash-filter-opensearch_guardium_filter.zip differ diff --git a/filter-plugin/logstash-filter-opensearch-guardium/mainREADME.md b/filter-plugin/logstash-filter-opensearch-guardium/mainREADME.md new file mode 100644 index 000000000..543eec667 --- /dev/null +++ b/filter-plugin/logstash-filter-opensearch-guardium/mainREADME.md @@ -0,0 +1,10 @@ +# OpenSearch Universal Connector + +## Follow this link to set up and use OpenSearch Universal Connector over CloudWatch Logstash Plugin + +[OpenSearchOverCloudwatch](./README.md) + +## Follow this link to set up and use OpenSearch Universal Connector over CloudWatch Connect + +[OpenSearchOverConnectCloudwatch](../../docs/KafkaBasedUCs/OpensearchCloudwatchKafkaConnect.md) + diff --git a/filter-plugin/logstash-filter-opensearch-guardium/settings.gradle b/filter-plugin/logstash-filter-opensearch-guardium/settings.gradle new file mode 100644 index 000000000..c65c1ca4e --- /dev/null +++ b/filter-plugin/logstash-filter-opensearch-guardium/settings.gradle @@ -0,0 +1,11 @@ +/* + * This file was generated by the Gradle 'init' task. + * + * The settings file is used to specify which projects to include in your build. + * + * Detailed information about configuring a multi-project build in Gradle can be found + * in the user manual at https://docs.gradle.org/7.1.1/userguide/multi_project_builds.html + */ + +rootProject.name = 'logstash-filter-opensearch-guardium' +include('lib') diff --git a/filter-plugin/logstash-filter-opensearch-guardium/src/main/java/com/ibm/guardium/aws/opensearch/CommonUtils.java b/filter-plugin/logstash-filter-opensearch-guardium/src/main/java/com/ibm/guardium/aws/opensearch/CommonUtils.java new file mode 100644 index 000000000..77a7e171f --- /dev/null +++ b/filter-plugin/logstash-filter-opensearch-guardium/src/main/java/com/ibm/guardium/aws/opensearch/CommonUtils.java @@ -0,0 +1,34 @@ +/* +Copyright IBM Corp. 2021, 2025 All rights reserved. +SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.guardium.aws.opensearch; + +import org.json.JSONException; +import org.json.JSONObject; + +public class CommonUtils { + + /** + * + */ + public CommonUtils() { + super(); + } + + /** + * isJSONValid() method is used to validate input string is valid JSON or NOT + * + * @return Boolean value TRUE/FALSE + * @methodName @isJSONValid + */ + public static boolean isJSONValid(String value) { + try { + new JSONObject(value); + } catch (JSONException ex) { + return false; + } + return true; + } +} diff --git a/filter-plugin/logstash-filter-opensearch-guardium/src/main/java/com/ibm/guardium/aws/opensearch/ConfigFileContent.java b/filter-plugin/logstash-filter-opensearch-guardium/src/main/java/com/ibm/guardium/aws/opensearch/ConfigFileContent.java new file mode 100644 index 000000000..e439f38d3 --- /dev/null +++ b/filter-plugin/logstash-filter-opensearch-guardium/src/main/java/com/ibm/guardium/aws/opensearch/ConfigFileContent.java @@ -0,0 +1,52 @@ +package com.ibm.guardium.aws.opensearch; + +public class ConfigFileContent { + + public String getConfigFileContent() { + StringBuilder config = new StringBuilder("{\n"); + + // Common fields across all categories + config.append(" \"app_user_name\": \"audit_request_effective_user\",\n"); + config.append(" \"client_hostname\": \"audit_node_host_name\",\n"); + config.append(" \"client_ip\": \"audit_request_remote_address\",\n"); + config.append(" \"client_ipv6\": \"\",\n"); + config.append(" \"client_mac\": \"\",\n"); + config.append(" \"client_os\": \"\",\n"); + config.append(" \"client_port\": \"{-1}\",\n"); + config.append(" \"comm_protocol\": \"audit_request_layer\",\n"); + config.append(" \"construct\": \"audit_category\",\n"); + config.append(" \"db_name\": \"audit_cluster_name\",\n"); + config.append(" \"db_protocol\": \"{OPSEARCH}\",\n"); + config.append(" \"db_user\": \"audit_request_effective_user\",\n"); + config.append(" \"db_user_initiating_user\": \"audit_request_effective_user\",\n"); + config.append(" \"server_hostname\": \"{opensearch.aws.com}\",\n"); + config.append(" \"server_ip\": \"\",\n"); + config.append(" \"server_port\": \"{-1}\",\n"); + config.append(" \"server_type\": \"{Opensearch}\",\n"); + config.append(" \"service_name\": \"audit_cluster_name\",\n"); + config.append(" \"session_id\": \"\",\n"); + config.append(" \"source_program\": \"audit_request_origin\",\n"); + config.append(" \"sql_parsing_active\": \"true\",\n"); + config.append(" \"timestamp\": \"@timestamp\",\n"); + + // REST + config.append(" \"REST_PATH\": \"audit_rest_request_path\",\n"); + config.append(" \"REST_METHOD\": \"audit_rest_request_method\",\n"); + + // Transport + config.append(" \"TRANSPORT_AUTHENTICATED\": \"audit_transport_request_type\",\n"); + config.append(" \"TRANSPORT_FAILED_LOGIN\": \"audit_request_exception_stacktrace\",\n"); + config.append(" \"TRANSPORT_PRIVILEGE\": \"audit_request_privilege\",\n"); + + config.append(" \"COMPLIANCE_OPERATION\": \"audit_compliance_operation\",\n"); + config.append(" \"COMPLIANCE_DOC_INDEX\": \"audit_trace_resolved_indices[0]\",\n"); + + config.append(" \"parsing_format\": \"JSON\",\n"); + config.append(" \"parsing_type\": \"SNIFFER\",\n"); + config.append(" \"sniffer_parser\": \"OPEN_SEARCH\",\n"); + config.append(" \"TEXT\": \"TEXT\"\n"); + + config.append("}\n"); + return config.toString(); + } +} diff --git a/filter-plugin/logstash-filter-opensearch-guardium/src/main/java/com/ibm/guardium/aws/opensearch/Constants.java b/filter-plugin/logstash-filter-opensearch-guardium/src/main/java/com/ibm/guardium/aws/opensearch/Constants.java new file mode 100644 index 000000000..c8587555a --- /dev/null +++ b/filter-plugin/logstash-filter-opensearch-guardium/src/main/java/com/ibm/guardium/aws/opensearch/Constants.java @@ -0,0 +1,46 @@ +package com.ibm.guardium.aws.opensearch; + +public class Constants { + public static final String LOGSTASH_TAG_JSON_PARSE_ERROR = "_opensearch_guardium_json_parse_error"; + public static final String AUDIT_CATEGORY = "audit_category"; + public static final String AUDIT_REQUEST_LAYER = "audit_request_layer"; + //request type + public static final String REQUEST_TYPE_REST = "REST"; + public static final String REQUEST_TYPE_TRANSPORT = "TRANSPORT"; + public static final String CATEGORY_FAILED_LOGIN = "FAILED_LOGIN"; + public static final String CATEGORY_MISSING_PRIVILEGES = "MISSING_PRIVILEGES"; + + + //OpenSearch event categories + public static final String CATEGORY_BAD_HEADERS = "BAD_HEADERS"; + public static final String CATEGORY_SSL_EXCEPTION = "SSL_EXCEPTION"; + public static final String CATEGORY_GRANTED_PRIVILEGES = "GRANTED_PRIVILEGES"; + public static final String CATEGORY_OPENSEARCH_SECURITY_INDEX_ATTEMPT = "OPENSEARCH_SECURITY_INDEX_ATTEMPT"; + public static final String CATEGORY_AUTHENTICATED = "AUTHENTICATED"; + //rest + public static final String CATEGORY_REST_FAILED_LOGIN = "REST_FAILED_LOGIN"; + public static final String CATEGORY_REST_AUTHENTICATED = "REST_AUTHENTICATED"; + public static final String CATEGORY_REST_SSL_EXCEPTION = "REST_SSL_EXCEPTION"; + public static final String CATEGORY_REST_BAD_HEADERS = "REST_BAD_HEADERS"; + public static final String CATEGORY_REST_MISSING_PRIVILEGES = "REST_MISSING_PRIVILEGES"; + //transport + public static final String CATEGORY_TRANSPORT_FAILED_LOGIN = "TRANSPORT_FAILED_LOGIN"; + public static final String CATEGORY_TRANSPORT_AUTHENTICATED = "TRANSPORT_AUTHENTICATED"; + public static final String CATEGORY_TRANSPORT_MISSING_PRIVILEGES = "TRANSPORT_MISSING_PRIVILEGES"; + public static final String CATEGORY_TRANSPORT_GRANTED_PRIVILEGES = "TRANSPORT_GRANTED_PRIVILEGES"; + public static final String CATEGORY_TRANSPORT_SSL_EXCEPTION = "TRANSPORT_SSL_EXCEPTION"; + public static final String CATEGORY_TRANSPORT_BAD_HEADERS = "TRANSPORT_BAD_HEADERS"; + public static final String CATEGORY_TRANSPORT_SECURITY_INDEX_ATTEMPT = "TRANSPORT_OPENSEARCH_SECURITY_INDEX"; + //standard categories + public static final String CATEGORY_INDEX_EVENT = "INDEX_EVENT"; + public static final String CATEGORY_COMPLIANCE_DOC_READ = "COMPLIANCE_DOC_READ"; + public static final String CATEGORY_COMPLIANCE_DOC_WRITE = "COMPLIANCE_DOC_WRITE"; + public static final String CATEGORY_COMPLIANCE_INTERNAL_CONFIG_READ = "COMPLIANCE_INTERNAL_CONFIG_READ"; + public static final String CATEGORY_COMPLIANCE_INTERNAL_CONFIG_WRITE = "COMPLIANCE_INTERNAL_CONFIG_WRITE"; + public static final String LANGUAGE_STRING = "OPEN_SEARCH"; + public static final String DB_PROTOCOL = "OPEN_SEARCH"; + static final String MESSAGE = "message"; + static final String INVALID_MSG_OPENSEARCH = "OPENSEARCH_EVENT_IS_INVALID"; + + +} diff --git a/filter-plugin/logstash-filter-opensearch-guardium/src/main/java/com/ibm/guardium/aws/opensearch/OpensearchGuardiumFilter.java b/filter-plugin/logstash-filter-opensearch-guardium/src/main/java/com/ibm/guardium/aws/opensearch/OpensearchGuardiumFilter.java new file mode 100644 index 000000000..941b38579 --- /dev/null +++ b/filter-plugin/logstash-filter-opensearch-guardium/src/main/java/com/ibm/guardium/aws/opensearch/OpensearchGuardiumFilter.java @@ -0,0 +1,98 @@ +/* +Copyright IBM Corp. 2021, 2023 All rights reserved. +SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.guardium.aws.opensearch; + +import co.elastic.logstash.api.*; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonObject; +import com.ibm.guardium.universalconnector.commons.GuardConstants; +import com.ibm.guardium.universalconnector.commons.custom_parsing.ParserFactory; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; + +import static com.ibm.guardium.aws.opensearch.Constants.*; + +@LogstashPlugin(name = "opensearch_guardium_filter") +public class OpensearchGuardiumFilter implements Filter { + private static Logger logger = LogManager.getLogger(OpensearchGuardiumFilter.class); + public static final PluginConfigSpec SOURCE_CONFIG = PluginConfigSpec.stringSetting("source", "message"); + private String id; + private Parser parser; + + public OpensearchGuardiumFilter(String id, Configuration config, Context context) { + this.id = id; + this.parser = new Parser(ParserFactory.ParserType.json); + } + + @Override + public Collection> configSchema() { + return Collections.singletonList(SOURCE_CONFIG); + } + + /** + * Returns the id + * + * @return id + */ + @Override + public String getId() { + return this.id; + } + + /** + * Filters the received events by skipping the invalid ones and normalizing them by parsing the provided payloads into Guardium Generic Records. + * + * @param events A list of received events + * @param filterMatchListener The listener for this plugin + * @return A list of normalized events + */ + public Collection filter(Collection events, FilterMatchListener filterMatchListener) { + ArrayList skippedEvents = new ArrayList<>(); + for (Event event : events) { + if (logger.isDebugEnabled()) { + logger.debug("Received event: {}", event.getData()); + } + + Object messageField = event.getField(MESSAGE); + String messageString = messageField.toString(); + + if (!CommonUtils.isJSONValid(messageString)) { + event.tag(INVALID_MSG_OPENSEARCH); + skippedEvents.add(event); + continue; + } + try { + JsonObject inputJSON = new Gson().fromJson(messageString, JsonObject.class); + Record record = parser.parseRecord(String.valueOf(inputJSON)); + if (record == null) { + event.tag(INVALID_MSG_OPENSEARCH); + skippedEvents.add(event); + continue; + } + Gson gson = new GsonBuilder() + .disableHtmlEscaping() + .serializeNulls() + .create(); + + event.setField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME, gson.toJson(record)); + filterMatchListener.filterMatched(event); + } catch (Exception ex) { + logger.error("Exception in parsing message: {}", event.getData(), + ex); + event.tag(LOGSTASH_TAG_JSON_PARSE_ERROR); + } + + } + events.removeAll(skippedEvents); + return events; + } +} \ No newline at end of file diff --git a/filter-plugin/logstash-filter-opensearch-guardium/src/main/java/com/ibm/guardium/aws/opensearch/Parser.java b/filter-plugin/logstash-filter-opensearch-guardium/src/main/java/com/ibm/guardium/aws/opensearch/Parser.java new file mode 100644 index 000000000..5cf3eac22 --- /dev/null +++ b/filter-plugin/logstash-filter-opensearch-guardium/src/main/java/com/ibm/guardium/aws/opensearch/Parser.java @@ -0,0 +1,414 @@ +/* +Copyright IBM Corp. 2021, 2023 All rights reserved. +SPDX-License-Identifier: Apache-2.0 + */ + +package com.ibm.guardium.aws.opensearch; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.ibm.guardium.universalconnector.commons.custom_parsing.CustomParser; +import com.ibm.guardium.universalconnector.commons.custom_parsing.ParserFactory; +import com.ibm.guardium.universalconnector.commons.structures.ExceptionRecord; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import com.ibm.guardium.universalconnector.commons.structures.Time; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +/** + * Parser Class will perform operation on parsing events and messages from the + * opensearch audit logs into a Guardium record instance Guardium records include + * the accessor, the sessionLocator, data, and exceptions. If there are no + * errors, the data contains details about the query "construct" + * + * @className @Parser + */ +public class Parser extends CustomParser { + private static Logger logger = LogManager.getLogger(Parser.class); + private final ObjectMapper mapper = new ObjectMapper(); + + public Parser(ParserFactory.ParserType parserType) { + super(parserType); + } + + @Override + public Record parseRecord(String payload) { + try { + String normalizedPayload = normalizeAuditCategory(payload); + return super.parseRecord(normalizedPayload); + } catch (Exception e) { + logger.error("Error extracting record: " + e.getMessage(), e); + return null; + } + } + + String normalizeAuditCategory(String payload) { + String category = getValueFromPayload(payload, Constants.AUDIT_CATEGORY); + if (category == null || category.isEmpty()) { + logger.error("Error normalizing audit category: " + category); + } + + try { + String layer = getValueFromPayload(payload, Constants.AUDIT_REQUEST_LAYER); + + + String normalizedCategory = category; + + if (layer.equals(Constants.REQUEST_TYPE_REST)) { + + if (category.equals(Constants.CATEGORY_FAILED_LOGIN)) { + normalizedCategory = Constants.CATEGORY_REST_FAILED_LOGIN; + } else if (category.equals(Constants.CATEGORY_AUTHENTICATED)) { + normalizedCategory = Constants.CATEGORY_REST_AUTHENTICATED; + } else if (category.equals(Constants.CATEGORY_SSL_EXCEPTION)) { + normalizedCategory = Constants.CATEGORY_REST_SSL_EXCEPTION; + } else if (category.equals(Constants.CATEGORY_BAD_HEADERS)) { + normalizedCategory = Constants.CATEGORY_REST_BAD_HEADERS; + } + } else if (layer.equals(Constants.REQUEST_TYPE_TRANSPORT)) { + + if (category.equals(Constants.CATEGORY_FAILED_LOGIN)) { + normalizedCategory = Constants.CATEGORY_TRANSPORT_FAILED_LOGIN; + } else if (category.equals(Constants.CATEGORY_AUTHENTICATED)) { + normalizedCategory = Constants.CATEGORY_TRANSPORT_AUTHENTICATED; + } else if (category.equals(Constants.CATEGORY_MISSING_PRIVILEGES)) { + normalizedCategory = Constants.CATEGORY_TRANSPORT_MISSING_PRIVILEGES; + } else if (category.equals(Constants.CATEGORY_GRANTED_PRIVILEGES)) { + normalizedCategory = Constants.CATEGORY_TRANSPORT_GRANTED_PRIVILEGES; + } else if (category.equals(Constants.CATEGORY_SSL_EXCEPTION)) { + normalizedCategory = Constants.CATEGORY_TRANSPORT_SSL_EXCEPTION; + } else if (category.equals(Constants.CATEGORY_REST_BAD_HEADERS)) { + normalizedCategory = Constants.CATEGORY_TRANSPORT_BAD_HEADERS; + } + } + + //standard category + if (normalizedCategory.equals(category)) { + if (category.equals(Constants.CATEGORY_INDEX_EVENT)) { + normalizedCategory = Constants.CATEGORY_INDEX_EVENT; + } else if (category.equals(Constants.CATEGORY_COMPLIANCE_DOC_READ)) { + normalizedCategory = Constants.CATEGORY_COMPLIANCE_DOC_READ; + } else if (category.equals(Constants.CATEGORY_COMPLIANCE_DOC_WRITE)) { + normalizedCategory = Constants.CATEGORY_COMPLIANCE_DOC_WRITE; + } else if (category.equals(Constants.CATEGORY_COMPLIANCE_INTERNAL_CONFIG_READ)) { + normalizedCategory = Constants.CATEGORY_COMPLIANCE_INTERNAL_CONFIG_READ; + } else if (category.equals(Constants.CATEGORY_COMPLIANCE_INTERNAL_CONFIG_WRITE)) { + normalizedCategory = Constants.CATEGORY_COMPLIANCE_INTERNAL_CONFIG_WRITE; + } + } + + if (!normalizedCategory.equals(category)) { + JsonNode rootNode = mapper.readTree(payload); + ((ObjectNode) rootNode).put(Constants.AUDIT_CATEGORY, normalizedCategory); + return rootNode.toString(); + } + + return payload; + } catch (Exception e) { + logger.error("Error normalizing audit category: " + e.getMessage(), e); + return payload; + } + } + + @Override + protected Record extractRecord(String payload) { + String category = getValueFromPayload(payload, Constants.AUDIT_CATEGORY); + if (Constants.CATEGORY_REST_FAILED_LOGIN.equals(category)) { + String dbUser = this.getValue(payload, "db_user"); + if (dbUser == null || dbUser.isEmpty() || "".equalsIgnoreCase(dbUser)) { + logger.debug("Skipping REST_FAILED_LOGIN event with db user "); + return null; + } + } + + Record record = new Record(); + record.setSessionId(this.getSessionId(payload)); + record.setDbName(this.getDbName(payload)); + record.setAppUserName(this.getAppUserName(payload)); + String sqlString = this.getSqlString(payload); + record.setException(this.getException(payload, sqlString)); + record.setAccessor(this.getAccessor(payload)); + record.setSessionLocator(this.getSessionLocator(payload)); + record.setTime(this.getTimestamp(payload)); + record.setData(this.getData(payload, sqlString)); + return record; + } + + @Override + protected String parse(String payload, String key) { + if (key == null || key.isEmpty()) { + return null; + } + + try { + return getValueFromPayload(payload, key); + } catch (Exception e) { + logger.error("Error parsing key '{}' from payload: {}", key, e.getMessage(), e); + return null; + } + } + + String getValueFromPayload(String payload, String fieldName) { + if (fieldName == null || fieldName.isEmpty()) { + return null; + } + try { + JsonNode rootNode = mapper.readTree(payload); + + if (fieldName.contains("[") && fieldName.contains("]")) { + int arrayStart = fieldName.indexOf("["); + int arrayEnd = fieldName.indexOf("]"); + String arrayField = fieldName.substring(0, arrayStart); + int index = Integer.parseInt(fieldName.substring(arrayStart + 1, arrayEnd)); + + JsonNode arrayNode = rootNode.path(arrayField); + if (arrayNode.isArray() && arrayNode.size() > index) { + return arrayNode.get(index).asText(); + } + return ""; + } + + if (rootNode.has(fieldName)) { + JsonNode fieldNode = rootNode.get(fieldName); + if (fieldNode.isArray()) { + return fieldNode.toString(); + } else { + return fieldNode.asText(); + } + } + } catch (Exception e) { + logger.error("Error getting value from payload: " + e.getMessage(), e); + } + return ""; + } + @Override + protected String getSqlString(String payload) { + StringBuilder sb = new StringBuilder(); + + String category = getValueFromPayload(payload, Constants.AUDIT_CATEGORY); + String layer = getValueFromPayload(payload, Constants.AUDIT_REQUEST_LAYER); + + boolean compliance_write = Constants.CATEGORY_COMPLIANCE_DOC_WRITE.equals(category) || Constants.CATEGORY_COMPLIANCE_INTERNAL_CONFIG_WRITE.equals(category); + boolean compliance_read = Constants.CATEGORY_COMPLIANCE_DOC_READ.equals(category) || Constants.CATEGORY_COMPLIANCE_INTERNAL_CONFIG_READ.equals(category); + String requestType = ""; + + sb.append("__OPSEARCH "); + if (layer != null && !layer.isEmpty()) { + + if (Constants.REQUEST_TYPE_REST.equals(layer)) { + String method = getValueFromPayload(payload, "audit_rest_request_method"); + String path = checkURIPath(getValueFromPayload(payload, "audit_rest_request_path")); + + sb.append(method).append(" ").append(path).append(" "); + + } else if (Constants.REQUEST_TYPE_TRANSPORT.equals(layer)) { + requestType = getValueFromPayload(payload, "audit_transport_request_type"); + String requestPrivilege = checkURIPath(getValueFromPayload(payload, "audit_request_privilege")); + + sb.append(requestType).append(" ").append(requestPrivilege).append(" "); + } + } else { + if (compliance_read) { + sb.append("GET").append(" ").append("/"); + } else if (compliance_write) { + sb.append("POST").append(" ").append("/"); + } + } + + sb.append("#"); + + sb.append("{"); + + sb.append("\"category\":\"").append(category).append("\""); + if (compliance_write) { + String complianceOperation = getValueFromPayload(payload, "audit_compliance_operation"); + sb.append(", \"action\":\"").append(complianceOperation).append("\""); + } + + String body = getValueFromPayload(payload, "audit_request_body"); + if (body != null && !body.isEmpty()) { + // Check if the body contains multiple JSON objects (bulk operations) + if (body.trim().startsWith("{") && body.contains("}\n{")) { + // Convert newline-separated JSON objects to a JSON array + String[] jsonObjects = body.split("\\n"); + sb.append(", \"_query\":["); + for (int i = 0; i < jsonObjects.length; i++) { + if (i > 0) { + sb.append(","); + } + sb.append(jsonObjects[i]); + } + sb.append("]"); + } else { + // Single JSON object, append as is + sb.append(", \"_query\":").append(body); + } + } + + String resolvedIndex = getValueFromPayload(payload, "audit_trace_resolved_indices"); + if (resolvedIndex.isEmpty()){ + resolvedIndex = getValueFromPayload(payload, "audit_trace_indices"); + } + if (!resolvedIndex.isEmpty()) { + List sanitized = sanitizeResolvedIndices(resolvedIndex); + try { + sb.append(", \"_indices\":").append(mapper.writeValueAsString(sanitized)); + } catch (JsonProcessingException e) { + logger.warn("Error serializing resolved indices: " + e.getMessage(), e); + sb.append(", \"_indices\":[]"); + } + + } + + sb.append("}"); + return sb.toString(); + } + + public List sanitizeResolvedIndices(String jsonArrayString) { + List sanitized = new ArrayList<>(); + + if (jsonArrayString == null || jsonArrayString.isEmpty()) { + return sanitized; + } + + try { + JsonNode arrayNode = mapper.readTree(jsonArrayString); + if (arrayNode.isArray()) { + for (JsonNode node : arrayNode) { + String index = node.asText(); + if (index != null && !index.startsWith(".")) { + sanitized.add(normalizeReservedKeyword(index)); + } + } + } + } catch (Exception e) { + logger.error("Failed to parse resolved indices: " + e.getMessage(), e); + } + + return sanitized; + } + + + public static String checkURIPath(String uri) { + if (uri == null || uri.isEmpty()) { + return uri; + } + + uri = uri.replaceAll("\\[.*?\\]", ""); + + + uri = uri.replaceAll("%", "_"); + + uri = uri.replace(":", "/"); + + if (uri.contains(" snifRestrictedKeywords = Set.of("template", "mappings", "get", "aliases", "user", "info", "account", "actiongroups", "actions"); + if (word != null && snifRestrictedKeywords.contains(word)) { + return "_" + word; + } + return word; + } + + + @Override + protected ExceptionRecord getException(String payload, String sqlString) { + ExceptionRecord exceptionRecord = new ExceptionRecord(); + String exceptionTypeId = this.getExceptionTypeId(payload); + String category = ""; + + if (exceptionTypeId.isEmpty()) { + category = getValueFromPayload(payload, Constants.AUDIT_CATEGORY); + exceptionTypeId = getExceptionTypeFromCategory(category); + if (exceptionTypeId == null) { + return null; + } + } + exceptionRecord.setExceptionTypeId(exceptionTypeId); + exceptionRecord.setDescription(category); + exceptionRecord.setSqlString(sqlString); + return exceptionRecord; + } + + private String getExceptionTypeFromCategory(String category) { + if (category.contains(Constants.CATEGORY_FAILED_LOGIN) || category.equals(Constants.CATEGORY_REST_FAILED_LOGIN) || category.equals(Constants.CATEGORY_TRANSPORT_FAILED_LOGIN)) { + return "LOGIN_FAILED"; + } + if (category.equals(Constants.CATEGORY_MISSING_PRIVILEGES) || category.equals(Constants.CATEGORY_BAD_HEADERS) || category.equals(Constants.CATEGORY_SSL_EXCEPTION) || category.equals(Constants.CATEGORY_REST_MISSING_PRIVILEGES) || category.equals(Constants.CATEGORY_REST_BAD_HEADERS) || category.equals(Constants.CATEGORY_REST_SSL_EXCEPTION) || category.equals(Constants.CATEGORY_TRANSPORT_MISSING_PRIVILEGES) || category.equals(Constants.CATEGORY_TRANSPORT_BAD_HEADERS) || category.equals(Constants.CATEGORY_TRANSPORT_SSL_EXCEPTION)) { + return "SQL_ERROR"; + } + return null; + } + + @Override + protected String getDbUser(String payload) { + String value = this.getValue(payload, "db_user"); + if (value == null || value.isEmpty()) { + value = this.getValue(payload, "db_user_initiating_user"); + } + if (value == null || value.isEmpty() || "".equalsIgnoreCase(value)) { + return "N.A"; + }; + return value; + } + + public static Time parseTimestamp(String timestamp) { + if (timestamp == null || timestamp.isEmpty()) { + throw new IllegalArgumentException("Timestamp cannot be null or empty"); + } + + ZonedDateTime date; + try { + date = ZonedDateTime.parse(timestamp); + } catch (Exception e) { + try { + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS"); + LocalDateTime localDateTime = LocalDateTime.parse(timestamp, formatter); + date = localDateTime.atZone(ZoneId.systemDefault()); + } catch (Exception e2) { + throw new IllegalArgumentException("Could not parse timestamp: " + timestamp, e2); + } + } + long millis = date.toInstant().toEpochMilli(); + int minOffset = date.getOffset().getTotalSeconds() / 60; + int minDst = date.getZone().getRules().isDaylightSavings(date.toInstant()) ? 60 : 0; + return new Time(millis, minOffset, minDst); + } + + @Override + public String getConfigFileContent() { + return new ConfigFileContent().getConfigFileContent(); + } +} \ No newline at end of file diff --git a/filter-plugin/logstash-filter-opensearch-guardium/src/test/java/com/ibm/guardium/aws/opensearch/OpensearchGuardiumFilterTest.java b/filter-plugin/logstash-filter-opensearch-guardium/src/test/java/com/ibm/guardium/aws/opensearch/OpensearchGuardiumFilterTest.java new file mode 100644 index 000000000..32a79b060 --- /dev/null +++ b/filter-plugin/logstash-filter-opensearch-guardium/src/test/java/com/ibm/guardium/aws/opensearch/OpensearchGuardiumFilterTest.java @@ -0,0 +1,392 @@ +package com.ibm.guardium.aws.opensearch; + +import co.elastic.logstash.api.*; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.ibm.guardium.universalconnector.commons.custom_parsing.ParserFactory; +import com.ibm.guardium.universalconnector.commons.structures.Time; +import org.junit.Before; +import org.junit.jupiter.api.Test; +import org.logstash.plugins.ConfigurationImpl; +import org.logstash.plugins.ContextImpl; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.jupiter.api.Assertions.*; + +class OpensearchGuardiumFilterTest { + + FilterMatchListener matchListener = new TestMatchListener(); + + String id = "1"; + Configuration config = new ConfigurationImpl(Collections.singletonMap("source", "")); + Context context = new ContextImpl(null, null); + OpensearchGuardiumFilter filter = new OpensearchGuardiumFilter(id, config, context); + + private Parser parser; + private ObjectMapper objectMapper; + + @Before + public void setUp() throws IOException { + parser = new Parser(ParserFactory.ParserType.json); + objectMapper = new ObjectMapper(); + } + + @Test + void testRestLayer() { + String payload = "{\n" + + " \"audit_cluster_name\": \"1245451225:myopensearchpk\",\n" + + " \"audit_transport_headers\": {\n" + + " \"X-Opaque-Id\": \"d6c4f099-4663-4633-baef-98eb0581f020\"\n" + + " },\n" + + " \"audit_node_name\": \"3c5bbacf33948a9a2c26426eb5a55c63\",\n" + + " \"audit_trace_task_id\": \"4dW-p_qGTr6MT1q9RqXThA:2669096\",\n" + + " \"audit_transport_request_type\": \"GetMappingsRequest\",\n" + + " \"audit_category\": \"INDEX_EVENT\",\n" + + " \"audit_request_origin\": \"REST\",\n" + + " \"audit_node_id\": \"4dW-p_qGTr6MT1q9RqXThA\",\n" + + " \"audit_request_layer\": \"TRANSPORT\",\n" + + " \"@timestamp\": \"2025-04-21T17:32:22.227+00:00\",\n" + + " \"audit_format_version\": 4,\n" + + " \"audit_request_remote_address\": \"216.58.113.178\",\n" + + " \"audit_request_privilege\": \"indices:admin/mappings/get\",\n" + + " \"audit_request_effective_user\": \"userpk\",\n" + + " \"audit_trace_resolved_indices\": [\n" + + " \".kibana_1\",\n" + + " \"school\",\n" + + " \".opendistro_security\",\n" + + " \"test_index\",\n" + + " \".opensearch-observability\"\n" + + " ]\n" + + "}"; + Event event = new org.logstash.Event(); + event.setField("message", payload); + Collection actualResponse = filter.filter(Collections.singletonList(event), matchListener); + + assertNotNull(actualResponse.toArray(new Event[0])[0].getField("GuardRecord")); + } + + @Test + void testArrayValuesFromIndex() { + String payload = "{\n" + + " \"audit_compliance_operation\": \"CREATE\",\n" + + " \"audit_cluster_name\": \"1245451225:myopensearchpk\",\n" + + " \"audit_node_name\": \"076ba9bbd1cfeb6c80e1d15a405869ed\",\n" + + " \"audit_category\": \"COMPLIANCE_DOC_WRITE\",\n" + + " \"audit_request_origin\": \"REST\",\n" + + " \"audit_compliance_doc_version\": 1,\n" + + " \"audit_request_body\": \"{\\n \\\"student_id\\\": \\\"101\\\",\\n \\\"name\\\": \\\"John Doe\\\",\\n \\\"age\\\": 15,\\n \\\"grade\\\": \\\"10th\\\",\\n \\\"subjects\\\": [\\\"Math\\\", \\\"Science\\\", \\\"English\\\"]\\n}\\n\",\n" + + " \"audit_node_id\": \"rgz9dKmNT9C9a1iSBM8yjg\",\n" + + " \"@timestamp\": \"2025-03-06T18:43:13.968+00:00\",\n" + + " \"audit_format_version\": 4,\n" + + " \"audit_request_remote_address\": \"69.171.141.155\",\n" + + " \"audit_trace_doc_id\": \"1\",\n" + + " \"audit_request_effective_user\": \"admin\",\n" + + " \"audit_trace_shard_id\": 0,\n" + + " \"audit_trace_indices\": [\n" + + " \"school-2025\"\n" + + " ],\n" + + " \"audit_trace_resolved_indices\": [\n" + + " \"school-2025\"\n" + + " ]\n" + + " }"; + Event event = new org.logstash.Event(); + event.setField("message", payload); + Collection actualResponse = filter.filter(Collections.singletonList(event), matchListener); + + assertNotNull(actualResponse.toArray(new Event[0])[0].getField("GuardRecord")); + } + + @Test + void testFailedLogin() { + String payload = "{\n" + + " \"audit_cluster_name\": \"1245451225:myopensearchpk\",\n" + + " \"audit_node_name\": \"3c5bbacf33948a9a2c26426eb5a55c63\",\n" + + " \"audit_rest_request_method\": \"GET\",\n" + + " \"audit_category\": \"FAILED_LOGIN\",\n" + + " \"audit_request_origin\": \"REST\",\n" + + " \"audit_node_id\": \"4dW-p_qGTr6MT1q9RqXThA\",\n" + + " \"audit_request_layer\": \"REST\",\n" + + " \"audit_rest_request_path\": \"/_plugins/_security/authinfo\",\n" + + " \"@timestamp\": \"2025-04-22T15:10:29.763+00:00\",\n" + + " \"audit_request_effective_user_is_admin\": false,\n" + + " \"audit_format_version\": 4,\n" + + " \"audit_request_remote_address\": \"216.58.113.178\",\n" + + " \"audit_rest_request_headers\": {\n" + + " \"x-opensearch-product-origin\": [\n" + + " \"opensearch-dashboards\"\n" + + " ],\n" + + " \"Connection\": [\n" + + " \"keep-alive\"\n" + + " ],\n" + + " \"x-opaque-id\": [\n" + + " \"56e3fd95-d79f-4077-a674-85fa22fed9e9\"\n" + + " ],\n" + + " \"Host\": [\n" + + " \"localhost:9200\"\n" + + " ],\n" + + " \"Content-Length\": [\n" + + " \"0\"\n" + + " ],\n" + + " \"NO_REDACT\": [\n" + + " \"false\"\n" + + " ]\n" + + " },\n" + + " \"audit_request_effective_user\": \"userpk\"\n" + + "}"; + Event event = new org.logstash.Event(); + event.setField("message", payload); + Collection actualResponse = filter.filter(Collections.singletonList(event), matchListener); + + assertNotNull(actualResponse.toArray(new Event[0])[0].getField("GuardRecord")); + } + + @Test + void testBadHeader() { + String payload = "{\n" + + " \"audit_cluster_name\": \"1245451225:myopensearchpk\",\n" + + " \"audit_rest_request_params\": {\n" + + " \"t\": \"1\",\n" + + " \"index\": \"teorema505\"\n" + + " },\n" + + " \"audit_node_name\": \"076ba9bbd1eb6c80e1d15a405869ed\",\n" + + " \"audit_rest_request_method\": \"GET\",\n" + + " \"audit_category\": \"BAD_HEADERS\",\n" + + " \"audit_request_origin\": \"REST\",\n" + + " \"audit_node_id\": \"rgz9dKmNT9C91iSBM8yjg\",\n" + + " \"audit_request_layer\": \"REST\",\n" + + " \"audit_rest_request_path\": \"/teorema505\",\n" + + " \"@timestamp\": \"2025-03-06T18:39:54.550+00:00\",\n" + + " \"audit_request_effective_user_is_admin\": false,\n" + + " \"audit_format_version\": 4,\n" + + " \"audit_request_remote_address\": \"161.35.66.151\",\n" + + " \"audit_rest_request_headers\": {\n" + + " \"content-length\": [\n" + + " \"0\"\n" + + " ],\n" + + " \"NO_REDACT\": [\n" + + " \"false\"\n" + + " ],\n" + + " \"user-agent\": [\n" + + " \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36\"\n" + + " ],\n" + + " \"accept\": [\n" + + " \"*/*\"\n" + + " ]\n" + + " },\n" + + " \"audit_request_effective_user\": \"\"\n" + + " }"; + Event event = new org.logstash.Event(); + event.setField("message", payload); + Collection actualResponse = filter.filter(Collections.singletonList(event), matchListener); + + assertNotNull(actualResponse.toArray(new Event[0])[0].getField("GuardRecord")); + } + + @Test + void testTransportLayer() { + String payload = "{\n" + + " \"audit_cluster_name\": \"1245451225:myopensearchpk\",\n" + + " \"audit_transport_headers\": {\n" + + " \"X-Opaque-Id\": \"f0075eb2-569f-4fd3-bfuu-2219571dfd9b\"\n" + + " },\n" + + " \"audit_node_name\": \"076ba9bbd1cfeb6c1d15a405869ed\",\n" + + " \"audit_trace_task_id\": \"rgz9dKmNT9C9a1iSBM8yjg:34312\",\n" + + " \"audit_transport_request_type\": \"GetAliasesRequest\",\n" + + " \"audit_category\": \"INDEX_EVENT\",\n" + + " \"audit_request_origin\": \"REST\",\n" + + " \"audit_node_id\": \"rgz9dKmNTC9a1iSBM8yjg\",\n" + + " \"audit_request_layer\": \"TRANSPORT\",\n" + + " \"@timestamp\": \"2025-03-06T16:56:39.103+00:00\",\n" + + " \"audit_format_version\": 4,\n" + + " \"audit_request_remote_address\": \"69.171.141.155\",\n" + + " \"audit_request_privilege\": \"indices:admin/aliases/get\",\n" + + " \"audit_request_effective_user\": \"admin\",\n" + + " \"audit_trace_resolved_indices\": [\n" + + " \".opendistro-reports-instances\",\n" + + " \".ql-datasources\",\n" + + " \".opendistro_security\",\n" + + " \".plugins-ml-config\",\n" + + " \"opensearch_dashboards_sample_data_flights\",\n" + + " \"school-2025\",\n" + + " \".opendistro-reports-definitions\",\n" + + " \".kibana_92668751_admin_1\",\n" + + " \".opensearch-observability\",\n" + + " \"opensearch_dashboards_sample_data_logs\",\n" + + " \".kibana_1\"\n" + + " ]\n" + + " }"; + Event event = new org.logstash.Event(); + event.setField("message", payload); + Collection actualResponse = filter.filter(Collections.singletonList(event), matchListener); + + assertNotNull(actualResponse.toArray(new Event[0])[0].getField("GuardRecord")); + } + + @Test + void testInvalidJsonHandling() { + // Arrange + String invalidJsonPayload = "{ \"@timestamp\": \"2025-03-06 15:39:02.724\", \"@message\": { \"audit_cluster_name\": \"346824953529:myopensearchpk\" "; + Event event = new org.logstash.Event(); + event.setField("message", invalidJsonPayload); + + List events = new ArrayList<>(); + events.add(event); + + Collection actualResponse = filter.filter(events, matchListener); + + assertEquals(0, actualResponse.size()); + } + + @Test + void testParse() { + String payload = "{ \"field1\": \"value1\", \"field2\": [\"value2\", \"value3\"] }"; + Parser parser = new Parser(ParserFactory.ParserType.json); + + String result = parser.parse(payload, "field1"); + assertEquals("value1", result); + + result = parser.parse(payload, "field2[1]"); + assertEquals("value3", result); + + result = parser.parse(payload, "nonexistent"); + assertEquals("", result); + + result = parser.parse(payload, "field2[5]"); + assertEquals("", result); + + result = parser.parse(payload, null); + assertNull(result); + + result = parser.parse(payload, ""); + assertNull(result); + } + + @Test + void testGetValueFromPayload() { + String payload = "{ \"field1\": \"value1\", \"field2\": [\"value2\", \"value3\"] }"; + Parser parser = new Parser(ParserFactory.ParserType.json); + + String result = parser.getValueFromPayload(payload, "field1"); + assertEquals("value1", result); + + result = parser.getValueFromPayload(payload, "nonexistent"); + assertEquals("", result); + + result = parser.getValueFromPayload(payload, "field2[1]"); + assertEquals("value3", result); + + result = parser.getValueFromPayload(payload, "field2[5]"); + assertEquals("", result); + + result = parser.getValueFromPayload(payload, null); + assertNull(result); + + result = parser.getValueFromPayload(payload, ""); + assertNull(result); + } + + @Test + void testNormalizeAuditCategory() { + Parser parser = new Parser(ParserFactory.ParserType.json); + ObjectMapper objectMapper = new ObjectMapper(); + + String restPayload = "{\"audit_category\": \"FAILED_LOGIN\", \"audit_request_layer\": \"REST\"}"; + String normalizedRest = parser.normalizeAuditCategory(restPayload); + try { + JsonNode restNode = objectMapper.readTree(normalizedRest); + assertEquals("REST_FAILED_LOGIN", restNode.path("audit_category").asText()); + } catch (Exception e) { + fail("Exception occurred while parsing JSON: " + e.getMessage()); + } + + String transportPayload = "{\"audit_category\": \"FAILED_LOGIN\", \"audit_request_layer\": \"TRANSPORT\"}"; + String normalizedTransport = parser.normalizeAuditCategory(transportPayload); + try { + JsonNode transportNode = objectMapper.readTree(normalizedTransport); + assertEquals("TRANSPORT_FAILED_LOGIN", transportNode.path("audit_category").asText()); + } catch (Exception e) { + fail("Exception occurred while parsing JSON: " + e.getMessage()); + } + + String standardPayload = "{\"audit_category\": \"INDEX_EVENT\", \"audit_request_layer\": \"REST\"}"; + String normalizedStandard = parser.normalizeAuditCategory(standardPayload); + try { + JsonNode standardNode = objectMapper.readTree(normalizedStandard); + assertEquals("INDEX_EVENT", standardNode.path("audit_category").asText()); + } catch (Exception e) { + fail("Exception occurred while parsing JSON: " + e.getMessage()); + } + + String malformedPayload = "{\"audit_category\": \"FAILED_LOGIN\"}"; + String normalizedMalformed = parser.normalizeAuditCategory(malformedPayload); + assertEquals(malformedPayload, normalizedMalformed); + } + + @Test + void testConfigSchema() { + OpensearchGuardiumFilter filter = new OpensearchGuardiumFilter("testId", null, null); + Collection> configSchema = filter.configSchema(); + assertNotNull(configSchema); + assertEquals(1, configSchema.size()); + assertTrue(configSchema.contains(OpensearchGuardiumFilter.SOURCE_CONFIG)); + } + + @Test + void testGetId() { + String expectedId = "testId"; + OpensearchGuardiumFilter filter = new OpensearchGuardiumFilter(expectedId, null, null); + String actualId = filter.getId(); + assertEquals(expectedId, actualId); + } + + @Test + void testParseTimestamp() { + String isoTimestamp = "2023-10-01T12:34:56.789Z"; + Time time = Parser.parseTimestamp(isoTimestamp); + assertNotNull(time); + + String invalidTimestamp = "invalid-timestamp"; + assertThrows(IllegalArgumentException.class, () -> Parser.parseTimestamp(invalidTimestamp)); + assertThrows(IllegalArgumentException.class, () -> Parser.parseTimestamp(null)); + } + + class TestMatchListener implements FilterMatchListener { + private AtomicInteger matchCount = new AtomicInteger(0); + + public int getMatchCount() { + return matchCount.get(); + } + + @Override + public void filterMatched(co.elastic.logstash.api.Event arg0) { + matchCount.incrementAndGet(); + + } + } + @Test + public void testNormalizeReservedKeyword() { + assertEquals("_user", Parser.normalizeReservedKeyword("user")); + assertEquals("_get", Parser.normalizeReservedKeyword("get")); + assertEquals("school", Parser.normalizeReservedKeyword("school")); + assertNull(Parser.normalizeReservedKeyword(null)); + } + + @Test + public void testCheckURIPath_basic() { + assertEquals("/students", Parser.checkURIPath("students")); + assertEquals("/_user", Parser.checkURIPath("user")); + assertEquals("/_get/_template", Parser.checkURIPath("get/template")); + } + + @Test + public void testCheckURIPath_encodedAndInvalid() { + assertEquals("/invalid/xml_input", Parser.checkURIPath("")); + assertEquals("/_mappings", Parser.checkURIPath("indices:mappings")); + } +} diff --git a/filter-plugin/logstash-filter-opensearch-guardium/src/test/java/com/ibm/guardium/aws/opensearch/ParserMultiRecordTest.java b/filter-plugin/logstash-filter-opensearch-guardium/src/test/java/com/ibm/guardium/aws/opensearch/ParserMultiRecordTest.java new file mode 100644 index 000000000..4a33e2305 --- /dev/null +++ b/filter-plugin/logstash-filter-opensearch-guardium/src/test/java/com/ibm/guardium/aws/opensearch/ParserMultiRecordTest.java @@ -0,0 +1,76 @@ +package com.ibm.guardium.aws.opensearch; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.ibm.guardium.universalconnector.commons.custom_parsing.ParserFactory; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +class ParserMultiRecordTest { + + @Test + void testSingleRecordJsonFormat() throws Exception { + // Test with a single record + String payload = "{\n" + + " \"audit_category\": \"COMPLIANCE_DOC_WRITE\",\n" + + " \"audit_request_body\": \"{\\n \\\"student_id\\\": \\\"S022\\\",\\n \\\"name\\\": \\\"William Brooks\\\",\\n \\\"grade\\\": 10,\\n \\\"class\\\": \\\"10B\\\",\\n \\\"subjects\\\": [\\\"Math\\\", \\\"Physics\\\"],\\n \\\"admission_date\\\": \\\"2021-09-20\\\",\\n \\\"address\\\": {\\n \\\"street\\\": \\\"123 Oakwood Dr\\\",\\n \\\"city\\\": \\\"Springfield\\\",\\n \\\"zip\\\": \\\"12357\\\"\\n }\\n}\\n\"\n" + + "}"; + + Parser parser = new Parser(ParserFactory.ParserType.json); + String sqlString = parser.getSqlString(payload); + + // Verify the result contains valid JSON + ObjectMapper mapper = new ObjectMapper(); + JsonNode jsonNode = mapper.readTree(sqlString.substring(sqlString.indexOf("#") + 1)); + + // Check that _query field exists and is valid JSON + assertTrue(jsonNode.has("_query")); + JsonNode queryNode = jsonNode.get("_query"); + assertTrue(queryNode.isObject()); + assertEquals("S022", queryNode.get("student_id").asText()); + } + + @Test + void testMultipleRecordsJsonFormat() throws Exception { + // Test with multiple records (bulk operation) + String payload = "{\n" + + " \"audit_category\": \"COMPLIANCE_DOC_WRITE\",\n" + + " \"audit_request_body\": \"{ \\\"index\\\": { \\\"_index\\\": \\\"school\\\", \\\"_id\\\": 1 } }\\n" + + "{ \\\"student_id\\\": \\\"S001\\\", \\\"name\\\": \\\"Alice Johnson\\\", \\\"grade\\\": 10, \\\"class\\\": \\\"10A\\\", \\\"subjects\\\": [\\\"Math\\\", \\\"Science\\\"], \\\"admission_date\\\": \\\"2021-06-15\\\", \\\"address\\\": { \\\"street\\\": \\\"123 Main St\\\", \\\"city\\\": \\\"Springfield\\\", \\\"zip\\\": \\\"12345\\\" } }\\n" + + "{ \\\"index\\\": { \\\"_index\\\": \\\"school\\\", \\\"_id\\\": 2 } }\\n" + + "{ \\\"student_id\\\": \\\"S002\\\", \\\"name\\\": \\\"Bob Smith\\\", \\\"grade\\\": 9, \\\"class\\\": \\\"9B\\\", \\\"subjects\\\": [\\\"English\\\", \\\"History\\\"], \\\"admission_date\\\": \\\"2022-07-01\\\", \\\"address\\\": { \\\"street\\\": \\\"456 Oak Ave\\\", \\\"city\\\": \\\"Springfield\\\", \\\"zip\\\": \\\"12346\\\" } }\\n" + + "{ \\\"index\\\": { \\\"_index\\\": \\\"school\\\", \\\"_id\\\": 3 } }\\n" + + "{ \\\"student_id\\\": \\\"S003\\\", \\\"name\\\": \\\"Charlie Davis\\\", \\\"grade\\\": 11, \\\"class\\\": \\\"11C\\\", \\\"subjects\\\": [\\\"Physics\\\", \\\"Chemistry\\\"], \\\"admission_date\\\": \\\"2020-09-10\\\", \\\"address\\\": { \\\"street\\\": \\\"789 Pine Rd\\\", \\\"city\\\": \\\"Shelbyville\\\", \\\"zip\\\": \\\"54321\\\" } }\\n\"\n" + + "}"; + + Parser parser = new Parser(ParserFactory.ParserType.json); + String sqlString = parser.getSqlString(payload); + + // Verify the result contains valid JSON + ObjectMapper mapper = new ObjectMapper(); + JsonNode jsonNode = mapper.readTree(sqlString.substring(sqlString.indexOf("#") + 1)); + + // Check that _query field exists and is a JSON array + assertTrue(jsonNode.has("_query")); + JsonNode queryNode = jsonNode.get("_query"); + assertTrue(queryNode.isArray()); + + // Verify we have 6 elements in the array (3 index operations + 3 documents) + assertEquals(6, queryNode.size()); + + // Check first document content + JsonNode firstDoc = queryNode.get(1); + assertEquals("S001", firstDoc.get("student_id").asText()); + + // Check second document content + JsonNode secondDoc = queryNode.get(3); + assertEquals("S002", secondDoc.get("student_id").asText()); + + // Check third document content + JsonNode thirdDoc = queryNode.get(5); + assertEquals("S003", thirdDoc.get("student_id").asText()); + } +} + +// Made with Bob diff --git a/filter-plugin/logstash-filter-oua-guardium/OracleRDSOverCloudwatch.md b/filter-plugin/logstash-filter-oua-guardium/OracleRDSOverCloudwatch.md new file mode 100644 index 000000000..d90c641a9 --- /dev/null +++ b/filter-plugin/logstash-filter-oua-guardium/OracleRDSOverCloudwatch.md @@ -0,0 +1,281 @@ +# Configuring Oracle RDS datasource profiles for Kafka Connect Plug-ins + +Create and configure datasource profiles through Central Manager for **Oracle RDS over CloudWatch Kafka Connect** plug-ins. + +### Meet Oracle RDS over CloudWatch Connect + +* Environments: AWS +* Supported inputs: Kafka Input (pull) +* Supported Guardium versions: + * Guardium Data Protection: Appliance bundle 12.2.2 or later + +Kafka-connect is a framework for streaming data between Apache Kafka and other systems. This connector enables monitoring of Oracle RDS audit logs through CloudWatch. + +## Configuring AWS RDS Oracle + +For detailed instructions on creating and configuring an AWS RDS Oracle database instance, see [Creating an Oracle DB instance and connecting to a database on an Oracle DB instance](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/CHAP_GettingStarted.CreatingConnecting.Oracle.html#CHAP_GettingStarted.Creating.Oracle). + +### Additional configuration requirements + +After creating your Oracle RDS instance following the Amazon documentation, make sure you complete the following steps. + +1. **Enable CloudWatch Log Exports:** + - In the Additional Configuration section, under Log exports, select the log type '**Audit**' from Amazon CloudWatch log options. + - Click on **Add Rule** and **Save** changes. + - **Note:** You might need to restart the database for changes to take effect. + +2. **Configure Security Group:** + - Edit the VPC security group associated with your database instance to allow traffic on port **1521**. + - In the **Inbound Rules** section, add a rule with: + - Type: Oracle-RDS + - Protocol: TCP + - Port Range: 1521 + - Source: Configure based on your security requirements (specific IP address or IP range) + +## Enabling Auditing + +### Configuring parameter group + +1. Enable auditing by setting up parameters on the parameter group and associating them with the database instance. + a. Select **Parameter Groups** from the left pane on Amazon RDS. + b. Select the newly created parameter group. + c. Click **Edit parameters** on the right corner. + d. Add the following setting: + + ``` + audit_trail = XML, EXTENDED + ``` + +### Associating DB parameter group to database instance + +1. Click **RDS** > **Databases** from the left panel. +2. Select the **Oracle database** instance to be updated. Then click **Modify**. +4. In the **Additional Configuration** section, under database options, select the newly created group from the **DB Parameter Group** drop-down. +5. Click **Continue**. +6. Select the database instance that, in its configuration section, shows the status for the DB Parameter Group as **pending-reboot**. +7. Reboot the Database instance for the changes to take effect. + +### Applying Audit Policies + +To allow the connector to parse and analyze your database queries, you must enable **Traditional Auditing** using the `AUDIT` command. Traditional Auditing records are exported to CloudWatch, making them accessible to this connector. + +**Important:** Do **not** use `CREATE AUDIT POLICY` (Unified Auditing). Unified Auditing records are stored inside the database and cannot be exported to CloudWatch. You must use the Traditional Auditing commands shown below. + +Connect to your Oracle RDS database using your master user (e.g., `ADMIN`) via a SQL client such as SQL*Plus, SQL Developer, or any JDBC-based tool. Depending on your security and compliance requirements, run the appropriate commands below. + +For more information about Traditional Auditing, see: +- [AWS Blog: Security Auditing in Amazon RDS for Oracle](https://aws.amazon.com/blogs/database/part-1-security-auditing-in-amazon-rds-for-oracle/) +- [Oracle Documentation: AUDIT (Traditional Auditing)](https://docs.oracle.com/en/database/oracle/oracle-database/19/sqlrf/AUDIT-Traditional-Auditing.html) + +#### Option 1: Audit all data modifications globally (All Users, All Tables) + +**Note:** This generates high log volume. It captures reads/writes on any table by any user. + +```sql +-- Capture all reads +AUDIT SELECT ANY TABLE BY ACCESS; + +-- Capture all writes +AUDIT INSERT ANY TABLE BY ACCESS; +AUDIT UPDATE ANY TABLE BY ACCESS; +AUDIT DELETE ANY TABLE BY ACCESS; + +-- Capture stored procedure executions +AUDIT EXECUTE ANY PROCEDURE BY ACCESS; +``` + +#### Option 2: Audit specific schema operations + +To audit operations on a specific schema (e.g., `MYSCHEMA`): + +```sql +-- Audit all operations on tables in MYSCHEMA +AUDIT SELECT TABLE, INSERT TABLE, UPDATE TABLE, DELETE TABLE BY MYSCHEMA BY ACCESS; + +-- Audit procedure executions in MYSCHEMA +AUDIT EXECUTE PROCEDURE BY MYSCHEMA BY ACCESS; +``` + +#### Option 3: Audit specific tables + +To audit operations on specific tables: + +```sql +-- Audit a specific table +AUDIT SELECT, INSERT, UPDATE, DELETE ON MYSCHEMA.MYTABLE BY ACCESS; + +-- Audit multiple specific tables +AUDIT SELECT, INSERT, UPDATE, DELETE ON MYSCHEMA.CUSTOMERS BY ACCESS; +AUDIT SELECT, INSERT, UPDATE, DELETE ON MYSCHEMA.ORDERS BY ACCESS; +``` + +#### Option 4: Audit session events (Login/Logout) + +To capture login and logout events: + +```sql +-- Audit all session connections +AUDIT SESSION BY ACCESS; + +-- Audit only failed login attempts +AUDIT SESSION WHENEVER NOT SUCCESSFUL; +``` + +#### Option 5: Audit DDL statements + +To capture Data Definition Language (DDL) operations: + +```sql +-- Audit all DDL statements +AUDIT TABLE BY ACCESS; +AUDIT VIEW BY ACCESS; +AUDIT PROCEDURE BY ACCESS; +``` + +#### To disable auditing + +To stop auditing specific operations: + +```sql +-- Disable global auditing +NOAUDIT SELECT ANY TABLE; +NOAUDIT INSERT ANY TABLE; +NOAUDIT UPDATE ANY TABLE; +NOAUDIT DELETE ANY TABLE; + +-- Stop auditing procedure executions +NOAUDIT EXECUTE ANY PROCEDURE; + +-- Disable auditing on specific tables +NOAUDIT SELECT, INSERT, UPDATE, DELETE ON MYSCHEMA.MYTABLE; + +-- Disable session auditing +NOAUDIT SESSION; +``` + +#### To check current audit settings + +To view which audit options are currently enabled: + +```sql +-- View global statement audit options +SELECT * FROM DBA_STMT_AUDIT_OPTS; + +-- View object-specific audit options +SELECT * FROM DBA_OBJ_AUDIT_OPTS; +``` + +#### To view audit trail records + +To query the audit trail (note: these records are also exported to CloudWatch): + +```sql +-- View recent audit records +SELECT EXTENDED_TIMESTAMP, DB_USER, ACTION, OBJECT_NAME, SQL_TEXT +FROM V$XML_AUDIT_TRAIL +WHERE DB_USER NOT IN ('SYS', 'SYSTEM', 'RDSADMIN') + AND DB_USER != '/' -- Exclude internal user + AND DB_USER IS NOT NULL +ORDER BY EXTENDED_TIMESTAMP DESC +``` + + +## Limitations + +1. **Data Ingestion Delay**: There will be a delay in data being observed for reports due to limitations of the Oracle RDS DB instance and CloudWatch log availability. + +2. **Filtered System Users and Operations**: To avoid unnecessary logging and reduce noise, the connector automatically filters out audit records from the following system users and operations: + - **System Users**: Records where `Object_Schema`, `Current_User`, or `DB_User` is `SYS`, `AUDSYS`, or `RDSADMIN` + - **DBMS_OUTPUT Operations**: Any SQL statements containing `DBMS_OUTPUT` calls + - **Empty or Invalid SQL**: Records with empty SQL text or containing only `/` + - **Records without Content**: Records missing both `Comment_Text` and `Sql_Text` + + These filters help focus on application-level database activities and reduce the volume of system-generated audit logs. + +## Creating datasource profiles + +You can create a new datasource profile from the **Datasource Profile Management** page. + +### Procedure + +1. Go to **Manage > Universal Connector > Datasource Profile Management** +2. Click the **➕ (Add)** button. +3. You can create a profile by using one of the following methods: + + - To **Create a new profile manually**, go to the **"Add Profile"** tab and provide values for the following fields. + - **Name** and **Description**. + - Select a **Plug-in Type** from the dropdown. For example, `Amazon RDS Oracle Over Cloudwatch Connect 2.0`. + + - To **Upload from CSV**, go to the **"Upload from CSV"** tab and upload an exported or manually created CSV file containing one or more profiles. + You can also choose from the following options: + - **Update existing profiles on name match** — Updates profiles with the same name if they already exist. + - **Test connection for imported profiles** — Automatically tests connections after profiles are created. + - **Use ELB** — Enables ELB support for imported profiles. You must provide the number of MUs to be used in the ELB process. + +**Note:** Configuration options vary based on the selected plug-in. + +## Configuring Oracle RDS Over CloudWatch Kafka Connect 2.0 + +The following table describes the fields that are specific to Oracle RDS over CloudWatch Kafka Connect 2.0 plugin. + +| Field | Description | +|--------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Name** | Unique name of the profile. | +| **Description** | Description of the profile. | +| **Plug-in** | Plug-in type for this profile. Select `Amazon RDS Oracle Over Cloudwatch Connect 2.0`. A full list of available plug-ins are available on the **Package Management** page. | +| **Credential** | Select AWS Credentials or AWS Role ARN. The credential to authenticate with AWS. Must be created in **Credential Management**, or click **➕** to create one. For more information, see [Creating Credentials](https://www.ibm.com/docs/en/SSMPHH_12.x/com.ibm.guardium.doc.stap/guc/guc_credential_management.html). | +| **Kafka Cluster** | Select the appropriate Kafka cluster from the available Kafka cluster list or create a new Kafka cluster. For more information, see [Managing Kafka clusters](https://www.ibm.com/docs/en/SSMPHH_12.x/com.ibm.guardium.doc.stap/guc/guc_kafka_cluster_management.html). | +| **Label** | Grouping label. For example, customer name or ID. | +| **AWS account region** | Specifies the AWS region where your RDS Oracle instance is located (e.g., us-east-1, eu-west-1). | +| **Log groups** | List of CloudWatch log groups to monitor. These are the log groups where Oracle audit logs are exported. | +| **Filter pattern** | CloudWatch Logs filter pattern to apply. Use "None" to retrieve all logs, or specify a pattern to filter specific log events. | +| **Account ID** | Your AWS account ID (12-digit number). This identifies your AWS account. | +| **Cluster name** | The name of your RDS Oracle cluster or instance identifier. | +| **Ingestion delay (seconds)** | Default value is 900 seconds (15 minutes). This delay accounts for the time it takes for logs to be available in CloudWatch after being generated. | +| **No-traffic threshold (minutes)** | Default value is 60. If there is no incoming traffic for an hour, S-TAP displays a red status. Once incoming traffic resumes, the status returns to green. | +| **Unmask sensitive value** | Optional boolean flag. When enabled, sensitive values in the audit logs will not be masked. | +| **Use Enterprise Load Balancing (ELB)** | Enable this if ELB support is required. | +| **Managed Unit Count** | Number of Managed Units (MUs) to allocate for ELB. | + +**Note:** +- Ensure that the **profile name** is unique. +- Required credentials must be created before or during profile creation. +- The AWS credentials must have appropriate permissions to read CloudWatch logs. + +--- + +## Testing a Connection + +After creating a profile, you must test the connection to ensure the provided configuration is valid. + +### Procedure + +1. Select the new profile. +2. From the top menu, click **Test Connection**. +3. If the test is successful, you can proceed to installing the profile. + +--- + +## Installing a Profile + +Once the connection test is successful, you can install the profile on **Managed Units (MUs)** or **Edges**. The parsed audit logs are sent to the selected Managed Unit or Edge to be consumed by the **Sniffer**. + +### Procedure + +1. Select the profile. +2. From the **Install** menu, click **Install**. +3. From the list of available MUs and Edges that is displayed, select the ones that you want to deploy the profile to. + +--- + +## Uninstalling or reinstalling profiles + +An installed profile can be uninstalled or reinstalled if needed. + +### Procedure + +1. Select the profile. +2. From the list of available actions, select the desired option: **Uninstall** or **Reinstall**. + +--- + diff --git a/filter-plugin/logstash-filter-oua-guardium/OuaOverConnectJdbcReadme.md b/filter-plugin/logstash-filter-oua-guardium/OuaOverConnectJdbcReadme.md index d7c7cc93b..74d17c6cd 100644 --- a/filter-plugin/logstash-filter-oua-guardium/OuaOverConnectJdbcReadme.md +++ b/filter-plugin/logstash-filter-oua-guardium/OuaOverConnectJdbcReadme.md @@ -1,10 +1,10 @@ # Oracle Unified Audit Universal Connector Over JDBC Connect ## Meet Oracle Unified Audit Over JDBC Connect -* Tested versions: 19,21 -* Environments: On-prem, RDS in AWS +* Tested versions: 19, 21 +* Environments: On-prem, RDS in AWS, Oracle Base Database Service in OCI * Supported inputs: Kafka Input (pull) -* Supported Oracle versions: 19,21 +* Supported Oracle versions: 19, 21 * Supported Guardium versions: * Guardium Data Protection: appliance bundle 12.1p105 or later. @@ -23,9 +23,9 @@ Detailed breakdown: ### GDP versions available with OUA over JDBC credential support | Credential types | Patch details for availability | |--------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **JDBC** | GDP version 12.1 + Appliance bundle patch 105 + Universal connector patch 1006 and later | -| **JDBC & Kerbseros** | GDP version 12.1 + Appliance bundle patch 115 + Universal connector patch 5002 and later | -| **JDBC & Kerbseros for OUA 2.0** | GDP version 12.1 + Appliance bundle patch 120 + Universal connector patch p5002 | +| **JDBC** | GDP version 12.1 + Appliance bundle patch 105 or 115 + Universal connector patch 1006 and later | +| **JDBC & Kerberos** | GDP version 12.1 + Appliance bundle patch 115 + Universal connector patch 5002 and later | +| **JDBC & Kerberos for OUA 2.0** | GDP version 12.1 + Appliance bundle patch 120 + Universal connector patch p5002 | @@ -110,21 +110,21 @@ Detailed breakdown: * Configure the policies you require. See [policies](/docs/#policies) for more information. ### Configuring Universal Connector Profile -1. See [Creating data source profile topic](https://www.ibm.com/docs/en/gdp/12.x?topic=configuration-creating-data-source-profiles) to create a datasource profile. +1. To create a datasource profile, see [Creating data source profiles](https://www.ibm.com/docs/en/SSMPHH_12.x/com.ibm.guardium.doc.stap/guc/guc_datasource_profile_management.html). 2. Select '**OUA over JDBC connect**' in the plug-ins list 3. Update the parameters as follows: | Field | Description | |--------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **Credential** | Create JDBC credentials. For more information, see [Creating Credentials](https://www.ibm.com/docs/en/gdp/12.x?topic=configuration-creating-credentials). | -| **Kafka cluster** | Select the appropriate Kafka cluster from the available Kafka cluster list or create a new Kafka cluster. For more information, see [Managing Kafka clusters](https://www.ibm.com/docs/en/gdp/12.x?topic=flow-creating-kafka-clusters). | +| **Credential** | Create JDBC credentials. For more information, see [Creating Credentials](https://www.ibm.com/docs/en/SSMPHH_12.x/com.ibm.guardium.doc.stap/guc/guc_credential_management.html). | +| **Kafka cluster** | Select the appropriate Kafka cluster from the available Kafka cluster list or create a new Kafka cluster. For more information, see [Managing Kafka clusters](https://www.ibm.com/docs/en/SSMPHH_12.x/com.ibm.guardium.doc.stap/guc/guc_kafka_cluster_management.html). | | **No traffic threshold (minutes)** | Default value is 60. If there is no incoming traffic for an hour, S-TAP displays a red status. Once incoming traffic resumes, the status returns to green. | | **Initial Time (ms)** | The timestamp from which the connector starts polling for changes in the database. Setting this to 0 means the connector starts from the earliest available data. For incremental data fetching, this ensures only new data (after the initial time) is retrieved. | | **Hostname** | Specifies the hostname or IP address of the Oracle database server. It is the address where the Oracle instance can be accessed for establishing a JDBC connection. | -| **JDBC driver library** | The Oracle JDBC driver JAR file (e.g., `ojdbc8.jar`) is required for the connector to communicate with the Oracle database. Download the [Oracle JDBC driver JAR file](https://download.oracle.com/otn-pub/otn_software/jdbc/234/ojdbc8.jar) and upload it to the Kafka Connect environment. | +| **JDBC driver library** | The Oracle JDBC driver JAR file (e.g., `ojdbc8.jar`) is required for the connector to communicate with the Oracle database. Download the [Oracle JDBC driver JAR file](https://download.oracle.com/otn-pub/otn_software/jdbc/234/ojdbc8.jar) and upload it to the Kafka Connect environment.
Note: Uploading multiple OJDBC versions simultaneously is not supported. This action causes Guardium Universal Connector to fail due to classloader conflicts. | | **Port** | Specifies the port number used to connect to the Oracle database. The default port number is 1521, but it can vary depending on the Oracle configuration. Port 1521 must be open and accessible for the connection. | | **Service Name / SID** | Specifies the Oracle service name (or SID if it's an older configuration) for the Kafka connector to connect. The service name uniquely identifies a database service within an Oracle environment and is provided by the database administrator. For OUA over JDBC data is retrived from the service itself: unified_audit_trail . | | **CDB Service Name / SID** | OUA over JDBC Connect 2.0 and OUA multitenant over JDBC Connect, data is retrived from CDB service audit log: cdb_unified_audit_trail. | -4. Continue from step 3 of [Creating data source profile topic](https://www.ibm.com/docs/en/gdp/12.x?topic=configuration-creating-data-source-profiles) to complete creating a datasource profile. +4. Continue from step 3 of [Creating data source profile topic](https://www.ibm.com/docs/en/SSMPHH_12.x/com.ibm.guardium.doc.stap/guc/guc_datasource_profile_management.html) to complete creating a datasource profile. diff --git a/filter-plugin/logstash-filter-oua-guardium/OuaOverPipe-Wallet.md b/filter-plugin/logstash-filter-oua-guardium/OuaOverPipe-Wallet.md new file mode 100644 index 000000000..473750e71 --- /dev/null +++ b/filter-plugin/logstash-filter-oua-guardium/OuaOverPipe-Wallet.md @@ -0,0 +1,208 @@ +# Oracle Unified Audit universal connector with Wallet authentication + +## Meet Oracle Unified Audit over Pipe with Wallet authentication +* Tested versions: 19C +* Environments: Oracle Cloud Infrastructure (OCI), Oracle Autonomous Database +* Supported inputs: Pipe input +* Supported Oracle versions: 19C +* Supported Guardium versions: + * Guardium Data Protection: 12.2.2 or later + +### Wallet ZIP: +1. **cwallet.sso** - Auto-login wallet (no password required) +2. **ewallet.p12** - Password-protected encrypted wallet +3. **tnsnames.ora** - Network service names configuration +4. **sqlnet.ora** - SQL*Net configuration with wallet location + + +## Before you begin + +1. Download the Oracle Wallet files from OCI or create them on-premises. +2. Enable unified auditing in the Oracle database. +3. Configure network connectivity to the Oracle database (typically port 1522 for TCPS). + +## Setting up the Wallet +### For Oracle Cloud Infrastructure (OCI): + +1. Download the Wallet from the OCI console.
+ a. Navigate to your **Autonomous Database** or **Database Service**.
+ b. Click **DB Connection**.
+ c. Download the wallet ZIP file.
+ +2. The Wallet contains the following files. + ``` + wallet/ + ├── cwallet.sso # Auto-login wallet + ├── ewallet.p12 # Encrypted wallet + ├── tnsnames.ora # Service names + ├── sqlnet.ora # Network configuration + ├── ojdbc.properties # JDBC properties + └── keystore.jks # Java keystore (optional) + ``` + + +3. Configure the `sqlnet.ora` file. + ```properties + WALLET_LOCATION = (SOURCE = (METHOD = file) (METHOD_DATA = (DIRECTORY=/usr/share/logstash/third_party/wallet/your_wallet_zip_name))) + SSL_SERVER_DN_MATCH=yes + SQLNET.AUTHENTICATION_SERVICES= (TCPS,NTS) + ``` + +## Configuring the universal connector + +### Step 1: Preparing the Wallet files + +Upload the Wallet files to Guardium system by using GRDAPI endpoint. + +1. Login to the CLI by using the following command. + This command returns a **client_secret** that you can use to generate the token. + + ``` + grdapi register_oauth_client client_id="testWallet" + ``` + +2. Generate the token. Exit the CLI, then run the following command on your MU. + ``` + curl -k -X POST -d + "client_id=testWallet&grant_type=password&client_secret=your_clien_secret&username=admin" -d password='your_password' + ``` + +3. Upload the Wallet ZIP by using the following command. + + ``` + curl -k -X POST --header "Authorization: Bearer token" -F + "walletFile=@local_file_path/your_wallet_zip_name.zip" \ + https://hostname:8443/restAPI/uploadOracleWalletZipFile + ``` +**Note:** For Guardium 12.2.1 with patch 5007, you must manually upload and extract the wallet ZIP file from the command line by completing the following steps. + + 1. Navigate to the collector at `/var/IBM/Guardium/uc/third_party`. Then create a folder named `wallet`, and upload the wallet ZIP file to this folder by using the following command: + + ``` + scp -O your_local_wallet_zip_location/your_wallet_zip_name.zip root@host:/var/IBM/Guardium/uc/third_party/wallet/ + ``` + + 2. Extract the wallet file by using the following command: + ``` + unzip your_wallet_zip_name.zip -d /var/IBM/Guardium/uc/third_party/wallet/ + ``` + +### Step 2: Configuring the Logstash legacy flow + +1. Configure the Logstash legacy flow by using the following configuration template. + + ``` + input { + pipe { + type => "oua-wallet" + command =>"${OUA_BINARY_PATH} -c ${THIRD_PARTY_PATH} -a + ${THIRD_PARTY_PATH}/wallet/ -s ${THIRD_PARTY_PATH} -r 100 -t + 1000 -p 20 -j username/password@SERVICE_NAME" + # Server configuration + add_field => {"SERVER_ADDRESS" => + ""} + add_field => {"SERVER_PORT" => ""} + # Wallet configuration - path on Guardium server where wallet is stored + add_field => {"WALLET_ZIP_PATH" => + "${THIRD_PARTY_PATH}/wallet/"} + # Service name from tnsnames.ora in wallet (e.g., mydb_high, + mydb_medium, mydb_low) + add_field => {"SERVICE_NAME" => ""} + } + } + + filter { + if [type] == "oua" { + ruby { + code => " + if event.get('message') + event.set('message', event.get('message').gsub('\\\\\\', '\\\\\\\\')) + end + " + } + mutate { + gsub => [ + "command", "\w+/[^\s@]+@", "*****@" + ] + } + json { + source => "message" + } + if [obj_owner] == "SYS" or [obj_owner] == "AUDSYS" or [obj_owner] == "RDSADMIN" + or [current_user] == "RDSADMIN" or [current_user] == "SYS" or [sql_text] =~ "DBMS_OUTPUT.GET_LINE" + + { + drop{} + } + mutate { + add_field => {"[HostName]" => "%{SERVER_ADDRESS}" } + add_field => {"[PortNumber]" => "%{SERVER_PORT}" } + add_field => {"[accountId]" => "%{ACCOUNT_ID}" } + } + if "_jsonparsefailure" not in [tags] { + oua_filter {} + } + } + } + ``` + +### Step 3: Configuring the Logstash CM flow + +1. Login to Guardium CM. Then navigate to the **Datasource Profile Management** page. +2. Create a **Profile**, select **Oracle over Pipe**. +3. In the **Credential** field, select **Oracle Autonomous Database Wallet**, and complete the required fields. + - Upload the Wallet configuration file: the `wallet.zip` file. + - Enter a **Database Username** (e.g., admin). + - Enter a **Database Password**. +4. Click **OK** to save the credential. +5. In the **Datasource Profile**, complete the required fields. + - **Server Address**: IP or hostname of the Oracle database + - **Server Port**: Port number for TCPS connection (the default is 1522) + - **Account ID**: (Optional) Account identifier for Guardium + - **Basic Instant Client package**: Download the [RPM file](https://download.oracle.com/otn_software/linux/instantclient/211000/oracle-instantclient-basic-21.1.0.0.0-1.x86_64.rpm). This release supports only Instant Client version 21.1.0.0.0 and later. + - **Wallet Zip Path**: Path on the Guardium server in which the wallet ZIP is stored (e.g., `${THIRD_PARTY_PATH}/wallet/`).
+ **Note:** This field is only required for Wallet authentication method. If you are using other authentication methods, leave this field blank. + - **Command**: Use the same command as in the legacy flow. Replace the wallet path and service name variables with your specific values. + ``` + ${OUA_BINARY_PATH} -c ${THIRD_PARTY_PATH} -a ${THIRD_PARTY_PATH}/wallet/ -s ${THIRD_PARTY_PATH} -r 1 -t 1000 -p 10 -j username/${password}@SERVICE_NAME + ``` +6. Click **OK** to save the datasource profile. +7. Select the new datasource profile in the Oracle over Pipe configuration. Then, deploy it to the collectors. +8. A secret containing the user’s password for OUA universal connector must be created - Example: grdapi universal_connector_keystore_add key=OUA_USER_PASS password= where is the OUA universal connector user’s password for the database. OUA_USER_PASS will be used in the plug-in configuration as a variable for password secret. + + Note: For CM flow, the password secret also need to add in MU by using the grdapi command. + +## Connection string format + +When you use the Wallet authentication, use the service name from `tnsnames.ora`. + +``` +jdbc:oracle:thin:@ +``` + +Example service names from OCI wallet: +- `mydb_high` - High priority connection +- `mydb_medium` - Medium priority connection +- `mydb_low` - Low priority connection + +## Troubleshooting common issues + +1. **Wallet location does not exist**
+ a. Verify that the wallet path is correct.
+ b. Check file permissions.
+ c. Ensure that the wallet files are extracted from the ZIP file.
+ +2. **Wallet directory must contain either cwallet.sso or ewallet.p12**
+ a. Verify that the wallet files are present.
+ b. Check if the ZIP extraction was successful.
+ c. Ensure that the wallet was downloaded correctly from OCI.
+ +3. **SSL handshake failed**
+ a. Verify that the SSL certificates in the wallet are valid.
+ b. Check network connectivity to database.
+ c. Ensure that the port is correct (1522 for TCPS).
+ +4. **Service name not found**
+ a. Check that the `tnsnames.ora` file contains the service name.
+ b. Verify that `TNS_ADMIN` is set correctly.
+ c. Ensure that the service name matches the OCI configuration.
diff --git a/filter-plugin/logstash-filter-oua-guardium/OuaOverPipeReadme.md b/filter-plugin/logstash-filter-oua-guardium/OuaOverPipeReadme.md index cf3ee9023..6d84d6f06 100644 --- a/filter-plugin/logstash-filter-oua-guardium/OuaOverPipeReadme.md +++ b/filter-plugin/logstash-filter-oua-guardium/OuaOverPipeReadme.md @@ -5,7 +5,9 @@ * Tested versions: 18,19 * Environments: On-prem, RDS in AWS, Oracle Autonomous Database in OCI - **Note**: Autonomous Database in OCI is supported only by Guardium Data Protection SqlGuard-12.0p7015_Bundle_May_20_2024 and SqlGuard-11.0p545_Bundle_Jul_09_2024. + **Note**: + * Autonomous Database in OCI is supported only by Guardium Data Protection SqlGuard-12.0p7015_Bundle_May_20_2024 and SqlGuard-11.0p545_Bundle_Jul_09_2024. + * For Oracle Autonomous Database, TCP is the only supported protocol, which does not provide built-in encryption. * Supported inputs: Oracle Unified Audit (pull) * Supported Oracle versions: 18, 19, and 21 * Supported Guardium versions: @@ -24,6 +26,7 @@ 5. You must create a secret containing your OUA universal connector password. - Example: `grdapi universal_connector_keystore_add key=OUA_USER_PASS password=` where `` is the OUA universal connector user’s password for the database. `OUA_USER_PASS` will be used in the plug-in configuration as a variable for the password secret. + Note: For CM flow, the password secret also need to add in MU by using the grdapi command. 6. Configure the policies you require. See [policies](https://github.com/IBM/universal-connectors/tree/main/docs#policies) for more information. @@ -52,12 +55,12 @@ Update the variables in Makefile for your environment's Java home and Logstash l - For other environments including RDS in AWS and Oracle Databases On-Premises run the following commands: ``` - CREATE USER guardium IDENTIFIED BY password; - GRANT CONNECT, RESOURCE to guardium; - GRANT SELECT ANY DICTIONARY TO guardium; - exec DBMS_NETWORK_ACL_ADMIN.APPEND_HOST_ACE(host => 'localhost', - ace => xs$ace_type(privilege_list => xs$name_list('connect', - 'resolve'), principal_name => 'guardium', principal_type => xs_acl.ptype_db)); + CREATE USER IDENTIFIED BY ; + GRANT CONNECT to ; + GRANT AUDIT_VIEWER to ; + GRANT SELECT ON v_$INSTANCE to ; + GRANT SELECT ON v_$DATABASE to ; + GRANT SELECT ON v_$MYSTAT to ; ``` - To verify your new user's privileges, connect to the Oracle instance that you planning to monitor using the name and credentials for your designated user and run the following statements: diff --git a/filter-plugin/logstash-filter-oua-guardium/README.md b/filter-plugin/logstash-filter-oua-guardium/README.md index 8e4edc7c4..e0ba63f38 100644 --- a/filter-plugin/logstash-filter-oua-guardium/README.md +++ b/filter-plugin/logstash-filter-oua-guardium/README.md @@ -6,4 +6,8 @@ ## Follow this link to set up and use Oracle Unified Audit Universal Connector over JDBC Connect -[OuaOverConnectJdbc](./OuaOverConnectJdbcReadme.md) \ No newline at end of file +[OuaOverConnectJdbc](./OuaOverConnectJdbcReadme.md) + +## Limitations + +- If a syntactically correct SQL query causes a database server error, the query appears in both the full SQL report and the exception report. \ No newline at end of file diff --git a/filter-plugin/logstash-filter-oua-guardium/build.gradle b/filter-plugin/logstash-filter-oua-guardium/build.gradle index 3051b47dd..98231ad08 100644 --- a/filter-plugin/logstash-filter-oua-guardium/build.gradle +++ b/filter-plugin/logstash-filter-oua-guardium/build.gradle @@ -2,6 +2,30 @@ import java.nio.file.Files import static java.nio.file.StandardCopyOption.REPLACE_EXISTING apply plugin: 'java' + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + + dependencies { + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' + classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.1.0" + classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' + } +} + +def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" @@ -21,10 +45,10 @@ pluginInfo.pluginClass = "OuaFilter" pluginInfo.pluginName = "oua_filter" // must match the @LogstashPlugin annotation in the main plugin class // =========================================================================== -sourceCompatibility = 1.8 -targetCompatibility = 1.8 +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 -def jacocoVersion = '0.8.4' +def jacocoVersion = '0.8.11' // minimumCoverage can be set by Travis ENV def minimumCoverageStr = System.getenv("MINIMUM_COVERAGE") ?: "50.0%" if (minimumCoverageStr.endsWith("%")) { @@ -32,27 +56,16 @@ if (minimumCoverageStr.endsWith("%")) { } def minimumCoverage = Float.valueOf(minimumCoverageStr) / 100 -buildscript { - repositories { - maven { - url "https://plugins.gradle.org/m2/" - } - mavenCentral() - jcenter() - } - - dependencies { - classpath 'com.github.jengelman.gradle.plugins:shadow:4.0.4' - classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.0.0-RC2" - classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' - } -} - -def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); -def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) repositories { - mavenCentral() + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } } tasks.register("vendor"){ @@ -63,14 +76,13 @@ tasks.register("vendor"){ File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") projectJarFile.mkdirs() Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) - validatePluginJar(projectJarFile, project.group) } } apply plugin: 'com.github.johnrengelman.shadow' shadowJar { - classifier = null + archiveClassifier = null } dependencies { @@ -78,7 +90,8 @@ dependencies { implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: versions.dependencies.log4jCore implementation 'org.apache.commons:commons-lang3:' + versions.dependencies.commonsLang implementation 'com.google.code.gson:gson:' + versions.dependencies.gson - implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "build/libs/logstash-core-*.*.*.jar") + implementation group: 'commons-beanutils', name: 'commons-beanutils', version: versions.dependencies.commonsBeanutils + implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "build/libs/logstash-core.jar") implementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "common-*.*.*.jar") testImplementation 'junit:junit:' + versions.dependencies.junit @@ -101,6 +114,17 @@ tasks.withType(JavaCompile) { options.encoding = 'UTF-8' } +test { + jvmArgs = [ + '--add-opens=java.base/java.lang=ALL-UNNAMED', + '--add-opens=java.base/java.util=ALL-UNNAMED', + '--add-opens=java.base/java.text=ALL-UNNAMED', + '--add-opens=java.base/java.lang.reflect=ALL-UNNAMED', + '--add-opens=java.base/sun.util.resources=ALL-UNNAMED', + '--add-opens=java.base/sun.util.cldr=ALL-UNNAMED' + ] +} + tasks.register("generateRubySupportFiles") { @@ -135,17 +159,16 @@ apply plugin: "org.barfuin.gradle.jacocolog" // ------------------------------------ jacoco { toolVersion = "${jacocoVersion}" - reportsDir = file("$buildDir/reports/jacoco") } jacocoTestReport { // You will see "Report -> file://...." at the end of a JaCoCo build // If no output, run this first: ./gradlew test reports { - html.enabled true - xml.enabled true - csv.enabled true - html.destination file("${buildDir}/reports/jacoco") - csv.destination file("${buildDir}/reports/jacoco/all.csv") + html.required = true + xml.required = true + csv.required = true + html.outputLocation = file("${buildDir}/reports/jacoco") + csv.outputLocation = file("${buildDir}/reports/jacoco/all.csv") } executionData.from fileTree(dir: "${buildDir}/jacoco/", includes: [ '**/*.exec' diff --git a/filter-plugin/logstash-filter-oua-guardium/gradle/wrapper/gradle-wrapper.properties b/filter-plugin/logstash-filter-oua-guardium/gradle/wrapper/gradle-wrapper.properties index aa991fcea..81aa1c044 100644 --- a/filter-plugin/logstash-filter-oua-guardium/gradle/wrapper/gradle-wrapper.properties +++ b/filter-plugin/logstash-filter-oua-guardium/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/filter-plugin/logstash-filter-oua-guardium/logstash-filter-oua_guardium_filter.zip b/filter-plugin/logstash-filter-oua-guardium/logstash-filter-oua_guardium_filter.zip new file mode 100644 index 000000000..629ee6068 Binary files /dev/null and b/filter-plugin/logstash-filter-oua-guardium/logstash-filter-oua_guardium_filter.zip differ diff --git a/filter-plugin/logstash-filter-oua-guardium/ouaPipe.conf b/filter-plugin/logstash-filter-oua-guardium/ouaPipe.conf index 0addbff73..432b5edeb 100644 --- a/filter-plugin/logstash-filter-oua-guardium/ouaPipe.conf +++ b/filter-plugin/logstash-filter-oua-guardium/ouaPipe.conf @@ -1,7 +1,7 @@ input { pipe { type => "oua" - command => "${OUA_BINARY_PATH} -c ${THIRD_PARTY_PATH} -s ${THIRD_PARTY_PATH} -r 1 -t 1000 -p 10 -j /${OUA_USER_PASS}@:/" + command => "${OUA_BINARY_PATH} -c ${THIRD_PARTY_PATH} -s ${THIRD_PARTY_PATH} -r 100 -t 1000 -p 20 -j /${OUA_USER_PASS}@:/" add_field => {"SERVER_ADDRESS" => ""} add_field => {"SERVER_PORT" => ""} #provide accountId incase of AWS ORACLE instance diff --git a/filter-plugin/logstash-filter-oua-guardium/src/main/java/com/ibm/guardium/OuaFilter.java b/filter-plugin/logstash-filter-oua-guardium/src/main/java/com/ibm/guardium/OuaFilter.java index e14a4eb69..c23cffaee 100644 --- a/filter-plugin/logstash-filter-oua-guardium/src/main/java/com/ibm/guardium/OuaFilter.java +++ b/filter-plugin/logstash-filter-oua-guardium/src/main/java/com/ibm/guardium/OuaFilter.java @@ -10,7 +10,15 @@ import com.google.gson.*; import com.ibm.guardium.universalconnector.commons.GuardConstants; import com.ibm.guardium.universalconnector.commons.Util; -import com.ibm.guardium.universalconnector.commons.structures.*; +import com.ibm.guardium.universalconnector.commons.structures.Accessor; +import com.ibm.guardium.universalconnector.commons.structures.Construct; +import com.ibm.guardium.universalconnector.commons.structures.Data; +import com.ibm.guardium.universalconnector.commons.structures.ExceptionRecord; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import com.ibm.guardium.universalconnector.commons.structures.Sentence; +import com.ibm.guardium.universalconnector.commons.structures.SentenceObject; +import com.ibm.guardium.universalconnector.commons.structures.SessionLocator; +import com.ibm.guardium.universalconnector.commons.structures.Time; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -60,6 +68,8 @@ public class OuaFilter implements Filter { public static final String DATA_PROTOCOL_STRING = "Oracle Unified Audit"; public static final String LANGUAGE_STRING = "ORACLE"; public static final String UNKNOWN_STRING = ""; + public static final String NOT_AVAILABLE = "N.A."; + public static final String CLIENT_IP_TAG = "client_host_ip"; public static final String SERVER_IP_TAG = "server_host_ip"; @@ -149,7 +159,7 @@ public static Record parseRecord(final Event event) throws ParseException { if (event.getField(OuaFilter.DB_NAME_TAG) instanceof String) { record.setDbName(event.getField(OuaFilter.DB_NAME_TAG).toString()); } else { - record.setDbName(OuaFilter.UNKNOWN_STRING); + record.setDbName(OuaFilter.NOT_AVAILABLE); } if (event.getField(OuaFilter.SESSION_ID_TAG) instanceof Long) { @@ -174,7 +184,7 @@ public static Record parseExceptionRecord(final Event event) throws ParseExcepti if (event.getField(OuaFilter.DB_NAME_TAG) instanceof String) { record.setDbName(event.getField(OuaFilter.DB_NAME_TAG).toString()); } else { - record.setDbName(OuaFilter.UNKNOWN_STRING); + record.setDbName(OuaFilter.NOT_AVAILABLE); } if (event.getField(OuaFilter.SESSION_ID_TAG) instanceof Long) { @@ -338,13 +348,13 @@ private static Accessor parseAccessor(final Event event) { if (event.getField(OuaFilter.DB_USER_TAG) instanceof String) { accessor.setDbUser(event.getField(OuaFilter.DB_USER_TAG).toString()); } else { - accessor.setDbUser(OuaFilter.UNKNOWN_STRING); + accessor.setDbUser(OuaFilter.NOT_AVAILABLE); } if (event.getField(OuaFilter.CON_NAME_TAG) instanceof String) { accessor.setServiceName(event.getField(OuaFilter.CON_NAME_TAG).toString()); } else { - accessor.setServiceName(OuaFilter.UNKNOWN_STRING); + accessor.setServiceName(OuaFilter.NOT_AVAILABLE); } return accessor; diff --git a/filter-plugin/logstash-filter-postgres-guardium/AwsPostgres_README.md b/filter-plugin/logstash-filter-postgres-guardium/AwsPostgres_README.md index faa800613..0a73f1a09 100644 --- a/filter-plugin/logstash-filter-postgres-guardium/AwsPostgres_README.md +++ b/filter-plugin/logstash-filter-postgres-guardium/AwsPostgres_README.md @@ -1,284 +1,296 @@ -# AWS postgres +# Configuring PostgreSQL audit logging for AWS RDS with Guardium ## Meet AWS Postgres * Environment: AWS * Supported inputs: CloudWatch (pull), SQS (pull) * Supported Guardium versions: - * Guardium Data Protection: 11.4 and above - * Guardium Data Security Center: 3.2 and above + * Guardium Data Protection: 11.4 and later + * Guardium Data Security Center: 3.2 and later -## Configuring native logging +## Configuring native logging (optional) - If desired, enable encryption on the database instances. In **Additional configuration** > **Log exports**, select the Postgresql log type to publish to Amazon CloudWatch. +You can enable encryption on the database instances by completing the following step. + +Click **Additional configuration** > **Log exports**. Then select the **Postgresql** log type to publish to Amazon CloudWatch. ## Enabling the PGAudit extension -There are different ways of auditing and logging in Postgres. For this exercise, we will use PGAudit, the open -source audit logging extension for PostgreSQL 9.5+. This extension supports logging for Sessions or Objects. -Configure either Session Auditing or Object Auditing. You cannot enable both at the same time. +There are different ways of auditing and logging in PostgreSQL. This procedure uses PGAudit, the open +source audit logging extension for PostgreSQL 9.5 and later. -### Procedure +This extension supports logging for sessions or objects. You can configure either session auditing or object auditing, but not both at the same time. -1. Creating the database parameter group -2. Enabling Auditing using **either one** of the following: +1. Create the database parameter group. +2. Enable auditing by using one of the following methods: - a. Enabling PGAudit Session Auditing + a. Enable PGAudit session auditing. - b. Enabling PGAudit Object Auditing + b. Enable PGAudit object auditing. -3. Associating the DB Parameter Group with the database Instance +3. Associate the DB parameter group with the database instance. -#### Creating the database parameter group +### Creating the database parameter group -When you create a database instance, it is associated with the default parameter group. Follow these -steps to create a new parameter group: +When you create a database instance, it is associated with the default parameter group. To create a new parameter group, complete the following steps. -#### Procedure -1. Go to **Services** > **Database** > **Parameter groups**. +1. From the AWS console, go to **Services** > **Database** > **Parameter groups**. 2. Click **Create Parameter Group**. -3. Enter the parameter group details. - - • Select the parameter group family. For example, postgres 12. This version should match the version of the database that is created and with which this parameter group is to be associated. +3. Configure the following parameter group fields: - • Enter the DB parameter group name. - - • Enter the DB parameter group description. -4. Click **Save**. The new group appears in the **Parameter Groups** section. + a. Select the parameter group family.
+  For example, **postgres 12**. This version must match the version of the database that is created and with which this parameter group is to be associated.
+ b. Enter the database parameter group name.
+ c. Enter the database parameter group description.
+ +5. Click **Save**. The new group appears in the **Parameter Groups** section. ### Enabling PGAudit session auditing -Session auditing allows you to log activities that are selected in the pgaudit.log for logging. Be cautious when you select which activities will be logged, as logged activities can affect the performance of the database instance. +Session auditing logs the activities that you specify in the pgaudit.log parameter. +**Note:** When you select which activities to log, as extensive logging can affect database instance performance. -#### Procedure 1. In the Amazon RDS panel, select **Parameter Groups**. 2. Select the parameter group that you created. -3. Click **Edit parameters** and add these settings: +3. Click **Edit parameters** and add the following settings: - • `pgaudit.log = all, -misc` + a. `pgaudit.log = all, -misc`
- (Select the options from the **Allowed values** list. You can specify multiple values and separate them with “,”. The values that are marked with “-” are excluded while logging.) + **Note:** Select the options from the **Allowed values** list. You can specify multiple values and separate them with commas `,`. The values that are marked with a hyphen `-` are excluded from logging.
- • `pgaudit.log_catalog = 0` + b. `pgaudit.log_catalog = 0`
- • `pgaudit.log_parameter = 0` + c. `pgaudit.log_parameter = 0`
- • `shared_preload_libraries = pgaudit` + d. `shared_preload_libraries = pgaudit`
- • `log_error_verbosity = default` + e. `log_error_verbosity = default`
-### Enabling PGAudit Object Auditing +### Enabling PGAudit object auditing Object auditing affects the performance less than session auditing, due to the fine-grained criteria of tables and columns that you can choose for auditing. -#### Procedure -1. Set these parameters: +1. Configure the following parameters: - • `pgaudit.log = none` (since this is not needed for extensive SESSION logging) + a. `pgaudit.log = none`
- • `pgaudit.role = rds_pgaudit` + b. `pgaudit.role = rds_pgaudit`
- • `pgaudit.log_catalog = 0` + c. `pgaudit.log_catalog = 0`
- • `pgaudit.log_parameter = 0` + d. `pgaudit.log_parameter = 0`
- • `shared_preload_libraries = pgaudit` + e. `shared_preload_libraries = pgaudit`
- • `log_error_verbosity = default` + f. `log_error_verbosity = default`
-2. Provide the required permissions to the rds_pgaudit role while associating it with the table that is audited. For example, grant `ALL` on `` to `rds_pgaudit` (this grant enables full `SELECT`, `INSERT`, `UPDATE`, and `DELETE` logging on the `relation_name`). +2. Provide the required permissions to the rds_pgaudit role while associating it with the table that is audited.
+ For example, grant `ALL` on `` to `rds_pgaudit`. This grant enables full `SELECT`, `INSERT`, `UPDATE`, and `DELETE` logging on `relation_name`. -### Associating the DB Parameter Group with the database instance +### Associating the DB parameter group with the database instance -#### Procedure -1. Go to **Services** > **Database** > **RDS** > **Databases**. -2. Click the Postgres database instance to be updated. -3. Click **Modify**. -4. Go to the **Additional Configuration** section > **database options** > **DB Parameter Group** menu and select the newly-created group. -5. Click **Continue**. -6. Select the database instance in its configuration section. The state of the DB Parameter Group is pending-reboot. -7. Reboot the database instance for the changes to take effect. +1. From the AWS console, go to **Services** > **Database** > **RDS** > **Databases**. +2. Select the Postgres database instance that you want to update and click **Modify**. +3. In **Additional Configuration** > **Database options**, select the newly created group from the **DB Parameter Group** menu. +4. Click **Continue**.
+ When you view the database instance in its configuration section, the state of the DB Parameter Group is `pending-reboot`. +5. Reboot the database instance for the changes to take effect. +### Logging Enabling query duration logging (optional) -## Viewing the PGAudit logs +Query duration logging tracks and logs queries based on their execution time. Use this feature for performance monitoring and identifying slow queries. + +1. From the RDS console, go to **Parameter Groups** and select your parameter group. +2. Edit the following parameters:
-The PGAudit logs (both Session and Object logs) can be seen in log files in RDS, and also on CloudWatch: + a. To log all queries with duration, set `log_min_duration_statement` to `0`.
+    * `0` - logs all queries with duration
+    * `-1` - disables duration-based logging
+    * `5000` - logs queries that run longer than 5 seconds
+
+ b. To avoid duplicate entries in CloudWatch, set `pgaudit.log` to `none`.
+3. Save the changes.
+ Changes apply immediately without requiring a database reboot. +## Viewing the PGAudit logs -### Viewing the auditing details in RDS log files +You can view PGAudit logs (both session and object logs) in RDS log files and in CloudWatch. -The RDS log files can be viewed, watched, and downloaded. The name of the RDS log file is modifiable and is controlled by parameter log_filename. +### Viewing audit details in RDS log files -#### Procedure -1. Go to **Services** > **Database** > **RDS** > **Databases**. +You can view, watch, and download RDS log files. The `log_filename` parameter specifies the log file name, which you can modify. + +1. From the AWS console, go to **Services** > **Database** > **RDS** > **Databases**. 2. Select the database instance. -3. Select the **Logs & Events** section. -4. The end of the Logs section lists the files that contain the auditing details. The newest file is the last page. +3. Select the **Logs & Events** tab. +4. In the **Logs** section, you can view the files that contain audit details. The newest file appears on the last page. -### Viewing the logs entries on CloudWatch -By default, each database instance has an associated log group with a name in this format: /aws/rds/instance//postgresql. You can use this log group, or you can create a new one and associate it with the database instance. +### Viewing log entries in CloudWatch -#### Procedure -1. On the AWS Console page, open the **Services** menu. -2. Enter the CloudWatch string in the search box. -3. Click **CloudWatch** to redirect to the CloudWatch dashboard. -4. Select **Logs**. -5. Click **Log Groups**. +By default, each database instance has an associated log group with the name `/aws/rds/instance//postgresql`. You can use this log group or create a new one and associate it with the database instance. +1. From the AWS console, open the **Services** menu. +2. Search for and select **CloudWatch**. +3. Click **Logs** > **Log Groups**. -#### Notes -* Guardium Data Protection requires installation of the [json_encode](https://www.elastic.co/guide/en/logstash-versioned-plugins/current/v3.0.3-plugins-filters-json_encode.html) filter plug-in +**Note:** Guardium Data Protection requires installation of the [json_encode](https://www.elastic.co/guide/en/logstash-versioned-plugins/current/v3.0.3-plugins-filters-json_encode.html) filter plug-in. -## Exporting CloudWatch Logs to SQS using lambda function +## Exporting CloudWatch logs to SQS by using a Lambda function -In order to achieve load balancing of audit logs between different collectors, the audit logs must exported from CloudWatch to SQS +To load balance audit logs between different collectors, export the audit logs from CloudWatch to SQS. -### Creating the SQS +### Creating the SQS queue -#### Procedure -1. Go to https://console.aws.amazon.com/ -2. Click **Services** -3. Search for SQS and click on Simple Queue Services -4. Click on **Create Queue** -5. Select the type as **Standard** -6. Enter the name for the queue -7. Keep the rest of the default settings +1. From the [AWS console](https://console.aws.amazon.com), click **Services**. +2. Search for and select **Simple Queue Service**. +3. Click **Create Queue**. +4. Select **Standard** as the queue type. +5. Enter the name for the queue. +6. Keep the remaining settings at their default values and create the queue. -#### Create Policy for the relevant IAM User -1. For the IAM User using which the SQS logs are to be accessed in Guardium, perform the below steps -2. Go to https://console.aws.amazon.com/ -3. Go to ```IAM service``` > ```Policies``` > ```Create Policy``` -4. Select ```service as SQS``` -5. Select the check boxes having actions as: ```ListQueues```, ```DeleteMessage```, ```DeleteMessageBatch```, ```GetQueueAttributes```, ```GetQueueUrl```, ```ReceiveMessage```, ```ChangeMessageVisibility```, ```ChangeMessageVisibilityBatch``` -6. In the resources, specify the ARN of the queue created in the above step -7. Click ```Review policy``` and specify the policy name -8. Click ```Create policy``` -9. Assign the policy to the user - - a. Log in to the IAM console as IAM user (https://console.aws.amazon.com/iam/) - - b. Go to ```Users``` on the console and select the relevant IAM user to whom you want to give permissions. Click the user name link - - c. In the ```Permissions``` tab, click ```Add permissions``` - - d. Click ```Attach existing policies directly``` - - e. Search for the policy created and check the checkbox next to it - - f. Click ```Next: Review``` +### Creating a policy for the IAM user + +Create a policy for the IAM user that accesses the SQS logs in Guardium. + +1. From the [AWS console](https://console.aws.amazon.com/), go to **IAM service** > **Policies** > **Create Policy**. +2. Select **SQS** as the service. +3. Select the following actions: **ListQueues**, **DeleteMessage**, **DeleteMessageBatch**, **GetQueueAttributes**, **GetQueueUrl**, **ReceiveMessage**, **ChangeMessageVisibility**, **ChangeMessageVisibilityBatch**. +4. In **Resources**, specify the ARN of the queue that you created. +5. Click **Review policy**, specify a policy name, and click **Create policy**. + +### Assigning the policy to the user - g. Click ```Add permissions``` +1. From the [IAM console](https://console.aws.amazon.com/iam/), go to **Users** and select the IAM user to whom you want to assign permissions. +2. In the **Permissions** tab, click **Add permissions**. +3. Click **Attach existing policies directly**. +4. Search for and select the policy that you created. +5. Click **Next: Review** and then click **Add permissions**. -### Creating the lambda function - -#### Create IAM Role - -Create the IAM role that will be used in the Lambda function set up. The AWS lambda service will require permission to log events and write to the SQS created. Create the IAM Role “Export-RDS-CloudWatch-to-SQS-Lambda” with “AmazonSQSFullAccess”, “CloudWatchLogsFullAccess”, and “CloudWatchEventsFullAccess” policies. - -##### Procedure -1. Go to https://console.aws.amazon.com/ -2. Go to ```IAM``` -> ```Roles``` -3. Click ```Create Role``` -4. Under ```use case``` select ```Lambda``` and click ```Next``` -5. Search for “AmazonSQSFullAccess” and select it -6. Search for “CloudWatchLogsFullAccess” and select it -7. Search for “CloudWatchEventsFullAccess” and select it -8. Set the ```Role Name```: e.g., “Export-RDS-CloudWatch-to-SQS-Lambda” and click ```Create role``` - -#### Create the lambda function - -#### Procedure -1. Go to https://console.aws.amazon.com/ -2. Go to ```Services```. Search for lambda function -3. Click ```Functions``` -4. Click ```Create Function``` -5. Keep ```Author for Scratch``` selected -6. Set ```Function name``` e.g., Export-RDS-CloudWatch-Logs-To-SQS -7. Under ```Runtime```, select ```Python 3.x``` -8. Under ```Permissions```, select ```Use an existing role``` and select the IAM role that you created in the previous step (Export-RDS-CloudWatch-to-SQS-Lambda) -9. Click ```Create function``` and navigate to ```Code view``` -10. Add the function code from [lambdaFunction](./PostgresOverSQSPackage/postgresLambda.py) -11. Click ```Configuration``` -> ```Environment Variables``` -12. Create 2 variables: - 1. Key = GROUP_NAME value = e.g., /aws/rds/instance/database-1/postgresql - 2. Key = QUEUE_NAME value = e.g., https://sqs.ap-south-1.amazonaws.com/11111111111/PostgresQueue -13. Save the function -14. Click on the **Deploy** button - -#### Automating the lambda function - -#### Procedure -1. Go to the CloudWatch dashboard -2. Go to ```Events``` -> ```Rules``` on the left pane -3. Click ```Create Rule``` -4. Enter the name for the rule e.g., cloudwatchToSqs -5. Under ```Rule Type```, select ```Schedule``` -6. Define the schedule. In ```schedule pattern``` select a schedule that runs at a regular rate, such as every 10 minutes -7. Enter the rate expression, meaning the rate at which the function should execute. This value must match the time specified in the lambda function code that calculates the time delta. (If the function code it is set to 2 minutes, set the rate to 2 minutes unless changed in the code). Click ```Next``` -8. Select the ```Target1```. Select the ```Target Type``` as ```AWS Service``` -9. Select ```Target``` as ```Lambda Function``` -10. Select the lambda function created in the above step. e.g., Export-RDS-CloudWatch-Logs-To-SQS -11. Add the tag if needed -12. Click ```Create Rule``` - -#### Note -Before making any changes to the lambda function code, first disable the above rule. Deploy the change and then re-enable the rule. +### Creating an IAM role + +Create an IAM role for the Lambda function. The AWS Lambda service requires permission to log events and write to the SQS queue. Create the IAM role with the following policies: **AmazonSQSFullAccess**, **CloudWatchLogsFullAccess**, and **CloudWatchEventsFullAccess**. + +1. From the [AWS console](https://console.aws.amazon.com/), go to **IAM** > **Roles**. +2. Click **Create Role**. +3. Under **Use case**, select **Lambda** and click **Next**. +4. Search for and select the following policies: + * **AmazonSQSFullAccess** + * **CloudWatchLogsFullAccess** + * **CloudWatchEventsFullAccess** +5. Enter a **Role Name** (for example, `Export-RDS-CloudWatch-to-SQS-Lambda`) and click **Create role**. + +### Creating the Lambda function + +1. From the [AWS console](https://console.aws.amazon.com/), go to **Services** and search for `Lambda function`. +2. Click **Functions** > **Create Function**. +3. Ensure that **Author for Scratch** is selected. +4. Enter a **Function name**. For example, ``Export-RDS-CloudWatch-Logs-To-SQS``. +5. Under **Runtime**, select **Python 3.x**. +6. Under **Permissions**, select **Use an existing role**. Then select the IAM role that you created (for example, `Export-RDS-CloudWatch-to-SQS-Lambda`). +7. Click **Create function**. +8. In the **Code** tab, add the function code from [lambdaFunction](./PostgresOverSQSPackage/postgresLambda.py). +9. Click **Configuration > Environment Variables** and create the following variables.
+ a. `GROUP_NAME` - Name of the log group in CloudWatch from where logs are exported. For example, `/aws/rds/instance/database-1/postgresql`.
+ b. `QUEUE_NAME` - Queue URL where logs are sent. For example, `https://sqs.ap-south-1.amazonaws.com/11111111/PostgresQueue`.
+ c. `PARAMETER_NAME` - Name of the parameter store in the System Manager Parameter Store. For example, `LastExecutionTimestamp`.
+ d. `ENABLE_DEBUG` - Set to `True` or `False` to control debugging statements.
+10. Click **Save**. +11. Deploy the function by using one of the following methods. + * **New Lambda Editor**: From the left sidebar, go to **EXPLORER > DEPLOY** and click **Deploy**. + * **Classic Lambda Editor**: Click **Deploy** in the top-right corner of the code editor. +12. Wait for the deployment to complete. A message is displayed when the deployment is successful. + +### Automating the Lambda function + +AWS migrated CloudWatch Events to Amazon EventBridge. Use EventBridge to create scheduling rules for Lambda functions. + +1. From the AWS console, search for and select **Amazon EventBridge**. +2. In the left navigation pane, click **Events** > **Rules**. +3. Click **Create rule**, and configure the following fields.
+ a. **Name**: Enter a name for the rule. For example, `cloudwatchToSqs`.
+ b. **Description**: (Optional) Add a description.
+ c. **Event bus**: Select **default**.
+4. In the **Rule type** field, select **Schedule** and click **Next**. +5. Define the schedule pattern.
+ a. Select **A schedule that runs at a regular rate, such as every 10 minutes**.
+ b. Enter the rate expression (for example, ``2`` minutes). This value must match the time delta in the Lambda function code. If the function code uses 2 minutes, set the rate to 2 minutes.
+ c. Click **Next**. +6. Select the target, and configure the following fields.
+ a. **Target types**: Select **AWS service**.
+ b. **Select a target**: Select **Lambda function**.
+ c. **Function**: Select the Lambda function that you created. For example, **Export-RDS-CloudWatch-Logs-To-SQS**.
+ d. Click **Next**.
+7. (Optional) Add tags and click **Next**. +8. Review the rule configuration and click **Create rule**. + +**Note:** Before you modify the Lambda function code, disable this rule. After you deploy the changes, re-enable the rule. ## Configuring the Postgres filters in Guardium -The Guardium universal connector is the Guardium entry point for native audit logs. The universal connector identifies and parses the received events, and converts them to a standard Guardium format. The output of the universal connector is forwarded to the Guardium sniffer on the collector, for policy and auditing enforcements. Configure Guardium to read the native audit logs by customizing the Postgres template. +The Guardium universal connector identifies and parses native audit log events, and converts them to standard Guardium format. The output is forwarded to the Guardium sniffer on the collector for policy and auditing enforcement. -### Authorizing outgoing traffic from AWS to Guardium +To configure Guardium to read native audit logs, customize the Postgres template. -#### Procedure -1. Log in to the Guardium Collector's API. -2. Issue these commands: - - • `grdapi add_domain_to_universal_connector_allowed_domains domain=amazonaws.com` +## Authorizing outgoing traffic from AWS to Guardium - • `grdapi add_domain_to_universal_connector_allowed_domains domain=amazon.com` +### Before you begin -#### Before you begin -• Configure the policies you require. See [policies](/docs/#policies) for more information. +1. Configure the required policies. For more information, see [policies](/docs/#policies). +2. You must have permission for the S-Tap Management role. The admin user includes this role by default. + -• You must have permission for the S-Tap Management role. The admin user includes this role by default. +### Procedure +1. Log in to the Guardium collector's API. +2. Run the following commands: + ``` + grdapi add_domain_to_universal_connector_allowed_domains domain=amazonaws.com + grdapi add_domain_to_universal_connector_allowed_domains domain=amazon.com + ``` +3. On the collector, go to **Setup** > **Tools and Views** > **Configure Universal Connector**. +4. If the universal connector is disabled, enable it. +5. Click the **plus sign (+)** to open the Connector Configuration dialog box. +6. In the **Connector name** field, enter a name for the connector. +7. Configure the input section by copying the content from the appropriate file, omitting `input{` at the beginning and the closing `}` at the end: + * To fetch audit logs from CloudWatch, use the input section from [postgresCloudwatch.conf](./PostgresOverCloudWatchPackage/postgresCloudwatch.conf).. + * To fetch audit logs from SQS, use the input section from [postgreSQS.conf](./PostgresOverSQSPackage/postgreSQS.conf). + + For more information about configuring the input plug-in, see [Cloudwatch_logs input plug-in](../../input-plugin/logstash-input-cloudwatch-logs/README.md). + + **Note**: To configure CloudWatch with `role_arn` instead of `access_key` and `secret_key`, see [Configuration for role_arn parameter in the cloudwatch_logs input plug-in](https://github.com/IBM/universal-connectors/blob/main/input-plugin/logstash-input-cloudwatch-logs/SettingsForRoleArn.md#configuration-for-role_arn-parameter-in-the-cloudwatch_logs-input-plug-in). + +8. Configure the filter section by copying the content from the appropriate file, omitting `filter{` at the beginning and the closing `}` at the end: + * To fetch audit logs from CloudWatch, use the filter section from [postgresCloudwatch.conf](./PostgresOverCloudWatchPackage/postgresCloudwatch.conf). + * To fetch audit logs from SQS, use the filter section from [postgreSQS.conf](./PostgresOverSQSPackage/postgreSQS.conf). +9. Ensure that the `type` field values match in both the input and filter sections. This field must be unique for each connector. +10. Click **Save**. Guardium validates the new connector and displays it on the Configure Universal Connector page. -• This plug-in is automatically available with Guardium Data Protection versions 12.x, 11.4 with appliance bundle 11.0p490 or later or Guardium Data Protection version 11.5 with appliance bundle 11.0p540 or later releases. +## Configuring the Postgres AWS Guardium Logstash filters in Guardium Data Security Center -• For Guardium Data Protection versions 11.0p540, 11.0p6505, 12.0 and 12p15 download the [cloudwatch_logs plug-in](../../input-plugin/logstash-input-cloudwatch-logs/CloudwatchLogsInputPackage/offline-logstash-input-cloudwatch_log_1_0_5.zip). -**Note**: For Guardium Data Protection version 11.4 without appliance bundle 11.0p490 or prior or Guardium Data Protection version 11.5 without appliance bundle 11.0p540 or prior, download the [postgres-offline-plugins-7.5.2.zip plug-in](https://github.com/IBM/universal-connectors/raw/release-v1.2.0/filter-plugin/logstash-filter-postgres-guardium/PostgresOverCloudWatchPackage/Postgres/postgres-offline-plugins-7.5.2.zip) plug-in. (Do not unzip the offline-package file throughout the procedure). +To configure this plug-in for Guardium Data Security Center, see [Configuring universal connectors.](/docs/Guardium%20Insights/3.2.x/UC_Configuration_GI.md) -#### Procedure +For more information on the input configuration step, see [CloudWatch_logs section](/docs/Guardium%20Insights/3.2.x/UC_Configuration_GI.md#configuring-a-CloudWatch-input-plug-in). -1. On the collector, go to **Setup** > **Tools and Views** > **Configure Universal Connector**. -2. Enable the universal connector if it is disabled. -3. Click **Upload File** and - * Select the [offline postgres-offline-plugins-7.5.2.zip](https://github.com/IBM/universal-connectors/raw/release-v1.2.0/filter-plugin/logstash-filter-postgres-guardium/PostgresOverCloudWatchPackage/Postgres/postgres-offline-plugins-7.5.2.zip) plug-in. After it uploads, click **OK**. This is not necessary for Guardium Data Protection v11.0p490 or later, v11.0p540 or later, v12.0 or later. - * If you have installed Guardium Data Protection version 11.0p540 and/or 11.0p6505, 12.0 and/or 12p15, select the offline [cloudwatch_logs plug-in](../../input-plugin/logstash-input-cloudwatch-logs/CloudwatchLogsInputPackage/offline-logstash-input-cloudwatch_log_1_0_5.zip). After it is uploaded, click **OK**. -4. Click the Plus sign to open the Connector Configuration dialog box. -5. Type a name in the **Connector name** field. -6. If the audit logs are to be fetched from CloudWatch directly, use the details from the [postgresCloudwatch.conf](./PostgresOverCloudWatchPackage/postgresCloudwatch.conf) file. But if the audit logs are to be fetched from SQS, use the details from the [postgreSQS.conf](./PostgresOverSQSPackage/postgreSQS.conf) file. Update the input section to add the details from the corresponding file's input part, omitting the keyword "input{" at the beginning and its corresponding "}" at the end. More details on how to configure the relevant input plugin, see [Cloudwatch_logs input plug-in](../../input-plugin/logstash-input-cloudwatch-logs/README.md). +## Configuring event filters (optional) - **Note**:If you want to configure Cloudwatch with role_arn instead of access_key and secret_key then refer to the [Configuration for role_arn parameter in the cloudwatch_logs input plug-in](https://github.com/IBM/universal-connectors/blob/main/input-plugin/logstash-input-cloudwatch-logs/SettingsForRoleArn.md#configuration-for-role_arn-parameter-in-the-cloudwatch_logs-input-plug-in) topic. +To improve data processing efficiency and avoid delays, configure event filtering to collect specific event types from AWS. Use the `event_filter` parameter in the input filter configuration. -7. If the audit logs are to be fetched from CloudWatch directly, use the details from the [postgresCloudwatch.conf](./PostgresOverCloudWatchPackage/postgresCloudwatch.conf) file. But if the audit logs are to be fetched from SQS, use the details from the [postgreSQS.conf](./PostgresOverSQSPackage/postgreSQS.conf) file. Update the filter section to add the details from the corresponding file's filter part, omitting the keyword "filter{" at the beginning and its corresponding "}" at the end -8. The "type" fields should match in the input and the filter configuration sections. This field should be unique for every individual connector added. -9. Click **Save**. Guardium validates the new connector and displays it in the Configure Universal Connector page. +For example, to filter DELETE and INSERT operations (case-insensitive) and reduce unnecessary event processing, run the following command: -## Configuring the Postgres AWS Guardium Logstash filters in Guardium Data Security Center - -To configure this plug-in for Guardium Data Security Center, follow [this guide.](/docs/Guardium%20Insights/3.2.x/UC_Configuration_GI.md) +``` +event_filter => '?delete ?DELETE ?insert ?INSERT' +``` -For the input configuration step, refer to the [CloudWatch_logs section](/docs/Guardium%20Insights/3.2.x/UC_Configuration_GI.md#configuring-a-CloudWatch-input-plug-in). +You can customize the filter based on the events that are relevant to your use case. To update the filter, modify the event types in the `event_filter` string. -### Troubleshooting +## Troubleshooting seahorse networking errors -If you encounter one of the following errors: +### Symptoms +If you encounter the following errors, the issue is typically related to AWS credentials or network configuration in your Docker container: ``` Exception: Seahorse::Client::NetworkingError @@ -290,31 +302,24 @@ or Exception: Seahorse::Client::NetworkingError: Socket closed ``` -These errors typically indicate an issue with the AWS credentials or network configuration inside your Docker container. Follow the steps below to verify the issue: - -### 1. Run `aws configure` -Ensure that your AWS credentials are correctly configured inside the Docker container(Klaus) within Collector. Run the following command to set up your AWS CLI configuration: - -```bash -aws configure -``` - -This will prompt you to enter your AWS Access Key ID, Secret Access Key, default region, and output format. Make sure to provide valid credentials with the necessary permissions to access the required AWS services. - -### 2. Verify AWS Identity with `aws sts get-caller-identity` -After configuring your AWS CLI credentials, verify that your AWS setup is correct by running the following command: - -```bash -aws sts get-caller-identity -``` - -This command returns the IAM user or role associated with the AWS credentials being used. If the command fails, it may indicate that the credentials are misconfigured or missing required permissions. - -### Additional Notes - -* If you encounter the `Seahorse::Client::NetworkingError`, it suggests a network or connectivity issue, typically indicating that the AWS CLI (in our case Plugin) cannot establish a connection to AWS services. Ensure that the container has proper network access and can reach the AWS endpoints. -* If the error is `Seahorse::Client::NetworkingError: Socket closed`, it might indicate that the AWS CLI (in our case Plugin) connection was unexpectedly closed or terminated. This could be caused by network interruptions, firewall issues, or invalid credentials. -* If the network issue is resolved, you can re-establish the connection between UC and AWS using the following CLI command: -``` -grdapi restart_universal_connector overwrite_old_instance="true" -``` +### Resolving the problem + +1. Verify your AWS credentials.
+ a. In the Docker container (Klaus) within the collector, configure your AWS CLI credentials by running the following command:
+ ``` + aws configure + ``` + b. Enter your AWS Access Key ID, Secret Access Key, default region, and output format. Ensure that you provide valid credentials with the necessary permissions.
+
+ c. Verify your AWS setup by running the following command. This command returns the IAM user or role associated with your AWS credentials. If the command fails, the credentials might be misconfigured or missing required permissions.
+ ``` + aws sts get-caller-identity + ``` +2. Review the error type: + * `Seahorse::Client::NetworkingError` indicates a network or connectivity issue. The plug-in cannot establish a connection to AWS services. Ensure that the container has network access and can reach AWS endpoints. + * `Seahorse::Client::NetworkingError:` Socket closed indicates that the connection was unexpectedly closed or terminated. This error can be caused by network interruptions, firewall issues, or invalid credentials. + +3. After you resolve the network issue, restart the connection between the universal connector and AWS by running the following command: + ``` + grdapi restart_universal_connector overwrite_old_instance="true" + ``` diff --git a/filter-plugin/logstash-filter-postgres-guardium/CHANGELOG.md b/filter-plugin/logstash-filter-postgres-guardium/CHANGELOG.md new file mode 100644 index 000000000..250b72ecf --- /dev/null +++ b/filter-plugin/logstash-filter-postgres-guardium/CHANGELOG.md @@ -0,0 +1,16 @@ +# Changelog + +## [1.0.3] +Adding support for execution time field in the GuardRecord + +## [1.0.2] +GRD-122937: Add fix in `PostgresOverCloudWatchPackage/postgresCloudwatch.conf` and `Postgres/filter.conf` for no traffic seen for Postgres over Cloudwatch. + +## [1.0.2] +Updating configuration in path `/PostgresOverSQSPackage/postgreSQS.conf` to enable custom certificate upload for UC. + +## [1.0.1] +Adding PostgresOverS3SQS filter parser implementation logic for Guardium records with remove extra quotes from SQL statement. + +## [1.0.0] +Adding PostgresOverS3SQS filter with java implementation \ No newline at end of file diff --git a/filter-plugin/logstash-filter-postgres-guardium/Gemfile b/filter-plugin/logstash-filter-postgres-guardium/Gemfile new file mode 100644 index 000000000..ea0d321ef --- /dev/null +++ b/filter-plugin/logstash-filter-postgres-guardium/Gemfile @@ -0,0 +1,12 @@ +# AUTOGENERATED BY THE GRADLE SCRIPT. EDITS WILL BE OVERWRITTEN. +source 'https://rubygems.org' + +gemspec + +logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash" +use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1" + +if Dir.exist?(logstash_path) && use_logstash_source + gem 'logstash-core', :path => "#{logstash_path}/logstash-core" + gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api" +end diff --git a/filter-plugin/logstash-filter-postgres-guardium/PostgresOverCloudWatchPackage/Postgres/filter.conf b/filter-plugin/logstash-filter-postgres-guardium/PostgresOverCloudWatchPackage/Postgres/filter.conf index f0a02fe9d..f90840893 100644 --- a/filter-plugin/logstash-filter-postgres-guardium/PostgresOverCloudWatchPackage/Postgres/filter.conf +++ b/filter-plugin/logstash-filter-postgres-guardium/PostgresOverCloudWatchPackage/Postgres/filter.conf @@ -1,5 +1,5 @@ filter { - if [type]=="postgres" { + if [type]=="postgres" { grok { match => { "message" => "%{YEAR:year}-%{MONTHNUM:month}-%{MONTHDAY:day} %{TIME:time} (?[^:]*):(?[^:]*):(?[^@]*)@(?[^:]*):\[(?[^\]]*)\]:%{GREEDYDATA:sql_full_log}" } } @@ -97,7 +97,24 @@ filter { } else if [sql_full_log] =~ /(?i)log:/ { - grok { match => { "sql_full_log" => "(?[^:]*): (?[^:]*): %{GREEDYDATA:audit_data},<" } } + # Check if this is a duration log (e.g., "LOG: duration: 0.219 ms statement: SELECT * FROM users;") + if [sql_full_log] =~ /duration:.*statement:/ { + grok { match => { "sql_full_log" => "(?[^:]*): +duration: (?[0-9.]+) ms +statement: %{GREEDYDATA:query_statement}" } } + if [duration_ms] and [query_statement] { + # Convert milliseconds to microseconds by multiplying by 1000 + ruby { code => 'event.set("duration_microseconds", (event.get("duration_ms").to_f * 1000).round)' } + + # Set log_level to indicate this is a valid log entry + mutate { add_field => { "log_level" => "duration_log" } } + + # Set query_details for processing + mutate { add_field => { "query_details" => "%{query_statement}" } } + } + } + else { + # Original audit log format + grok { match => { "sql_full_log" => "(?[^:]*): (?[^:]*): %{GREEDYDATA:audit_data},<" } } + } if [log_level] { @@ -161,12 +178,13 @@ filter { } mutate {add_field => { "[GuardRecord][appUserName]" => "AWSService" } } - + if [session_id] { mutate {add_field => { "[GuardRecord][sessionId]" => "%{session_id}" } } } else { - mutate {add_field => { "[GuardRecord][sessionId]" => "NA" } } + mutate {add_field => { "[GuardRecord][sessionId]" => "" } } + mutate {replace => { "[GuardRecord][sessionLocator][clientPort]" => "-1" }} } mutate {add_field => { "[GuardRecord][sessionLocator][serverIp]" => "0.0.0.0" } } @@ -192,6 +210,11 @@ filter { mutate { add_field => { "[GuardRecord][accessor][language]" => "PGRS" } } #PGRS is a Guardium internal code for PostreSQL mutate { add_field => { "[GuardRecord][accessor][dataType]" => "TEXT" } } + # Add executionTime to GuardRecord if duration was captured + if [duration_microseconds] { + mutate { add_field => { "[GuardRecord][executionTime]" => "%{duration_microseconds}" } } + } + json_encode { source => "[GuardRecord]" target => "[GuardRecord]" @@ -209,6 +232,6 @@ filter { drop { } } - prune { whitelist_names => [ "GuardRecord" ] } + mutate { remove_field => ["cloudwatch_logs", "rest_data", "tags", "clientIP", "session_id","db_name", "error_description", "log", "log_level", "data1", "data2", "data3", "data4", "data5", "data6", "data7", "db_user", "year", "month", "day","time", "finalTime", "myTimestamp", "ts", "epoc", "@timestamp", "@version", "error", "sql_state", "description", "rest", "type", "sequence", "query_details", "message", "clientPort", "audit_data", "sql_full_log", "data13", "number_of_elements", "data14", "instance", "log_group", "account_id", "zone", "data12", "data15", "client_IP", "client_PORT", "db_USER", "db_NAME", "client_ip", "query", "host", "duration_ms", "duration_microseconds"] } } } \ No newline at end of file diff --git a/filter-plugin/logstash-filter-postgres-guardium/PostgresOverCloudWatchPackage/postgresCloudwatch.conf b/filter-plugin/logstash-filter-postgres-guardium/PostgresOverCloudWatchPackage/postgresCloudwatch.conf index ec610efed..1f3593e22 100644 --- a/filter-plugin/logstash-filter-postgres-guardium/PostgresOverCloudWatchPackage/postgresCloudwatch.conf +++ b/filter-plugin/logstash-filter-postgres-guardium/PostgresOverCloudWatchPackage/postgresCloudwatch.conf @@ -25,6 +25,8 @@ input { type => "postgres" #Insert the account id of the AWS account add_field => {"account_id" => ""} + #Unmask is an optional parameter. To get unmasked logs, you need to set unmask = true. + #unmask => true } } @@ -128,7 +130,24 @@ filter { } else if [sql_full_log] =~ /(?i)log:/ { - grok { match => { "sql_full_log" => "(?[^:]*): (?[^:]*): %{GREEDYDATA:audit_data},<" } } + # Check if this is a duration log (e.g., "LOG: duration: 0.219 ms statement: SELECT * FROM users;") + if [sql_full_log] =~ /duration:.*statement:/ { + grok { match => { "sql_full_log" => "(?[^:]*): +duration: (?[0-9.]+) ms +statement: %{GREEDYDATA:query_statement}" } } + if [duration_ms] and [query_statement] { + # Convert milliseconds to microseconds by multiplying by 1000 + ruby { code => 'event.set("duration_microseconds", (event.get("duration_ms").to_f * 1000).round)' } + + # Set log_level to indicate this is a valid log entry + mutate { add_field => { "log_level" => "duration_log" } } + + # Set query_details for processing + mutate { add_field => { "query_details" => "%{query_statement}" } } + } + } + else { + # Original audit log format + grok { match => { "sql_full_log" => "(?[^:]*): (?[^:]*): %{GREEDYDATA:audit_data},<" } } + } if [log_level] { @@ -192,12 +211,13 @@ filter { } mutate {add_field => { "[GuardRecord][appUserName]" => "AWSService" } } - + if [session_id] { mutate {add_field => { "[GuardRecord][sessionId]" => "%{session_id}" } } } else { - mutate {add_field => { "[GuardRecord][sessionId]" => "NA" } } + mutate {add_field => { "[GuardRecord][sessionId]" => "" } } + mutate {replace => { "[GuardRecord][sessionLocator][clientPort]" => "-1" }} } mutate {add_field => { "[GuardRecord][sessionLocator][serverIp]" => "0.0.0.0" } } @@ -223,6 +243,11 @@ filter { mutate { add_field => { "[GuardRecord][accessor][language]" => "PGRS" } } #PGRS is a Guardium internal code for PostreSQL mutate { add_field => { "[GuardRecord][accessor][dataType]" => "TEXT" } } + # Add executionTime to GuardRecord if duration was captured + if [duration_microseconds] { + mutate { add_field => { "[GuardRecord][executionTime]" => "%{duration_microseconds}" } } + } + json_encode { source => "[GuardRecord]" target => "[GuardRecord]" @@ -240,6 +265,6 @@ filter { drop { } } - prune { whitelist_names => [ "GuardRecord" ] } + mutate { remove_field => ["cloudwatch_logs", "rest_data", "tags", "clientIP", "session_id","db_name", "error_description", "log", "log_level", "data1", "data2", "data3", "data4", "data5", "data6", "data7", "db_user", "year", "month", "day","time", "finalTime", "myTimestamp", "ts", "epoc", "@timestamp", "@version", "error", "sql_state", "description", "rest", "type", "sequence", "query_details", "message", "clientPort", "audit_data", "sql_full_log", "data13", "number_of_elements", "data14", "instance", "log_group", "account_id", "zone", "data12", "data15", "client_IP", "client_PORT", "db_USER", "db_NAME", "client_ip", "query", "host", "duration_ms", "duration_microseconds"] } } } \ No newline at end of file diff --git a/filter-plugin/logstash-filter-postgres-guardium/PostgresOverS3SQSPackage/AWSS3SQSProstgre.conf b/filter-plugin/logstash-filter-postgres-guardium/PostgresOverS3SQSPackage/AWSS3SQSProstgre.conf new file mode 100644 index 000000000..2e4d09892 --- /dev/null +++ b/filter-plugin/logstash-filter-postgres-guardium/PostgresOverS3SQSPackage/AWSS3SQSProstgre.conf @@ -0,0 +1,91 @@ +input { + s3_sqs { + queue_url => "" + region => "" + access_key_id => "" + secret_access_key => "" + role_arn => "" # Leave empty if not using role-based access + max_messages => + wait_time => # Must be >= 0 and <= 20, + polling_frequency => + type => "" + add_field => { + "account_id" => "" + "instance_name" => "" + } + } +} + +filter{ + + if [type] == "PostgresS3SQS" { + + json { + source => "message" + target => "parsed_message" + remove_field => ["message"] + } + + # Drop known noise patterns + if [parsed_message][message] =~ /(pg_sleep|export_postgres_logs_to_s3|cron\.job_run_details|create_foreign_table_for_log_file|logs.postgres_logs|public\.list_postgres_log_files|information_schema\.tables|aws_s3\.query_export_to_s3)/ { + drop { } + } + + # Drop ALTER/CREATE/CREATE FOREIGN TABLE statements on postgres_logs_ + if [parsed_message][message] =~ /((ALTER|CREATE(\s+FOREIGN)?)\s+TABLE)\s+postgres_logs_\d{8}_\d{4}/ { + drop { } + } + + # Sanitize multiline and quote issues + mutate { + gsub => [ + "[parsed_message][message]", "\n", "", + "[parsed_message][message]", "¶", " ", + "[parsed_message][message]", '\\"', '"' + ] + } + + # Promote nested parsed_message.message to top-level for grok + mutate { + add_field => { + "log_message" => "%{[parsed_message][message]}" + } + } + + grok { + match => { + "log_message" => [ + # Match structured AUDIT logs + 'AUDIT: %{WORD:audit_level},%{INT:session_id},%{INT:transaction_id},%{WORD:operation_type},%{DATA:command},%{DATA:object_type},%{DATA:object_name},%{GREEDYDATA:full_sql_query},%{GREEDYDATA:details}', + + # Catch-all error message fallback + '%{GREEDYDATA:ErrorMessage}' + ] + } + remove_field => ["log_message"] + } + + # Drop events that couldn't be parsed by grok + if "_grokparsefailure" in [tags] { + drop { } + } + + # Trim quotes from SQL query if it exists + mutate { + gsub => [ + "full_sql_query", '^\"', '', + "full_sql_query", '\"$', '' + ] + } + + # Call your custom Guardium plugin + s3sqs_postgresql_guardium_plugin_filter { } + + # Retain only processed GuardRecord + prune { + whitelist_names => ["GuardRecord"] + } + + } + +} \ No newline at end of file diff --git a/filter-plugin/logstash-filter-postgres-guardium/PostgresOverS3SQSPackage/AwsPostgresS3SQS_README.md b/filter-plugin/logstash-filter-postgres-guardium/PostgresOverS3SQSPackage/AwsPostgresS3SQS_README.md new file mode 100644 index 000000000..e3cbcff5f --- /dev/null +++ b/filter-plugin/logstash-filter-postgres-guardium/PostgresOverS3SQSPackage/AwsPostgresS3SQS_README.md @@ -0,0 +1,177 @@ +## Meet AWS Postgres +* Environment: AWS +* Supported inputs: S3 (pull), SQS (pull) +* Supported Guardium versions: + * Guardium Data Protection: 11.4 and above + * Guardium Data Security Center: 3.2 and above +#### Notes +This is a [Logstash](https://github.com/elastic/logstash) filter plug-in for the universal connector that is featured in IBM Security Guardium. The filter supports events sent through Cloudwatch OR SQS. + +## 1. Configuring the AWS Postgres service + +1. Go to https://console.aws.amazon.com/ +2. Click ```Services``` +3. In the Database section, click ```RDS``` +4. Select the region in the top right corner +5. In the central panel of the Amazon RDS Dashboard, click ```Create database``` +6. Choose a database creation method +7. In the Engine options, select ```PostgreSQL```, and then select the appropropriate version +8. Select an appropriate template (Production, Dev/Test, or Free Tier) +9. In the Settings section, type the database instance name and create the master account with the username and password to log in to the database +10. Select the database instance size according to your requirements +11. Select appropriate storage options (for example, you may want to enable auto-scaling) +12. Select the availability and durability options +13. Select the connectivity settings that are appropriate for your environment. To make the database accessible, set the Public access option to Publicly Accessible within Additional Configuration +14. Select the type of Authentication for the database (choose from Password Authentication, Password and IAM database authentication, and Password and Kerberos authentication) +15. Expand the Additional Configuration options: + + a. Configure the database options + + b. Select options for Backup + + c. If desired, enable Encryption on the database instances + + d. In Log exports, select the Postgresql log type to publish to Amazon CloudWatch + + e. Select the options for Deletion protection + +16. Click ```Create Database``` +17. To view the database, click ```Databases``` under Amazon RDS in the left panel +18. To authorize inbound traffic, edit the security group: + + a. In the database summary page, select the Connectivity and Security tab. Under Security, click VPC security group + + b. Click the group name that you selected while creating a database (each database has one active group) + + c. In the Inbound rule section, choose to edit the inbound rules + + d. Set this rule: + + • Type: PostgreSQL + + • Protocol: TCP + + • Port Range: 5432 + + Notes: Depending on your requirements, the source can be set to a specific IP address or it can be opened to all hosts. + + e. Click ```Add Rule ```and then click ```Save changes```. The database may need to be restarted + +## 2. Enabling the PGAudit extension + +There are different ways of auditing and logging in postgres. For this exercise, we will use PGAudit, the open +source audit logging extension for PostgreSQL 9.5+. This extension supports logging for Sessions or Objects. +Configure either Session Auditing or Object Auditing. You cannot enable both at the same time. + +### Steps to enable PGAudit + +1. Creating the database parameter group +2. Enabling Auditing using **either one** of the following: + + a. Enabling PGAudit Session Auditing + + b. Enabling PGAudit Object Auditing + +3. Associating the DB Parameter Group with the database Instance + +#### Creating the database parameter group + +When you create a database instance, it is associated with the default parameter group. Follow these +steps to create a new parameter group: + +1. Go to ```Services``` > ```Database``` > ```Parameter groups``` +2. Click Create Parameter Group in the left pane +3. Enter the parameter group details + + • Select the parameter group family. For example, aurora-postgres12. This version should match the version of the database that is created and with which this parameter group is to be associated + + • Enter the DB parameter group name + + • Enter the DB parameter group description + +4. Click ```Save```. The new group appears in the Parameter Groups section + +#### Enabling PGAudit Auditing + +Session Auditing allows you to log activities that are selected in the pgaudit.log for logging. Be cautious when you select which activities will be logged, as logged activities can affect the performance of the database instance. + +1. In the left-hand Amazon RDS panel, select Parameter Groups. +2. Select the parameter group that you created. +3. Click Edit parameters and add these settings: + + • pgaudit.log = all + (Select the options from the Allowed values list. You can specify multiple values, and separate them with ",". The values that are marked with "-" are excluded while logging.) + + • pgaudit.log_catalog = 0 + + • pgaudit.log_parameter = 0 + + • shared_preload_libraries = pgaudit,pg_cron + + • log_error_verbosity = default + + • pgaudit.role = rds_pgaudit + + • log_destination = csvlog + + • cron.database_name = `````` + +#### Associating the DB Parameter Group with the database Instance + +1. Go to ```Services``` > ```Database``` > ```RDS``` > ```Databases``` +2. Click the Postgres database instance to be updated +3. Click ```Modify``` +4. Go to the Additional Configuration ```section``` > ```database options``` > ```DB Parameter Group menu``` and select the ```newly-created group``` +5. Click ```Continue``` +6. Select the database instance in its configuration section. The state of the DB Parameter Group is pending-reboot +7. Reboot the database instance for the changes to take effect + +## 3. Viewing the PGAudit logs + +The PGAudit logs (both Session and Object logs) can be seen in log files in RDS, and also on CloudWatch: + +### Viewing the auditing details in RDS log files + +The RDS log files can be viewed, watched, and downloaded. The name of the RDS log file is modifiable and is controlled by parameter log_filename. + +1. Go to Services > Database > RDS > Databases +2. Select the database instance +3. Select the Logs & Events section +4. The end of the Logs section lists the files that contain the auditing details. The newest file is the last page + +### Viewing the logs entries on CloudWatch + +By default, each database instance has an associated log group with a name in this format: /aws/rds/instance//postgresql. You can use this log group, or you can create a new one and associate it with the database instance. + +1. On the AWS Console page, open the Services menu +2. Enter the CloudWatch string in the search box +3. Click CloudWatch to redirect to the CloudWatch dashboard +4. In the left panel, select Logs +5. Click Log Groups + +### Configuration +1. On the collector, go to ```Setup``` > ```Tools and Views``` > ```Configure Universal Connector```. +2. Enable the universal connector if it is disabled. +3. Click ```Upload File``` and select the offline [logstash-filter-s3sqs_postgresql_guardium_plugin_filter.zip](../../logstash-filter-postgres-guardium/PostgresOverS3SQS/logstash-filter-s3sqs_postgresql_guardium_plugin_filter.zip) plugin. After it is uploaded, click ```OK```. This step is not necessary for Guardium Data Protection v11.0p490 or later, v11.0p540 or later, v12.0 or later. +4. Click ```Upload File``` and select the offline [logstash-input-s3_sqs.zip](../../../input-plugin/logstash-input-s3sqs/InputS3SQSPackage/S3SQS/logstash-input-s3_sqs.zip) plugin. After it is uploaded, click ```OK```. This step is not necessary for Guardium Data Protection v11.0p490 or later, v11.0p540 or later, v12.0 or later. +4. Click the Plus sign to open the Connector Configuration dialog box. +5. Type a name in the ```Connector name``` field. +6. The audit logs are to be fetched from S3SQS directly, use the details from the [AWSS3SQSProstgre.conf](../../logstash-filter-postgres-guardium/PostgresOverS3SQSPackage/AWSS3SQSProstgre.conf) file. Update the input section to add the details from the corresponding file's input part, omitting the keyword "input{" at the beginning and its corresponding "}" at the end. More details on how to configure the relevant input plugin can be found [here](../../../input-plugin/logstash-input-s3sqs/README.md) +7. The audit logs are to be fetched from S3SQS directly, use the details from the [AWSS3SQSProstgre.conf](../../logstash-filter-postgres-guardium/PostgresOverS3SQSPackage/AWSS3SQSProstgre.conf) file. Update the filter section to add the details from the corresponding file's filter part, omitting the keyword "filter{" at the beginning and its corresponding "}" at the end. +8. The "type" fields should match in the input and the filter configuration sections. This field should be unique for every individual connector added. This is no longer required starting v12p20 and v12.1. +9. Click ```Save```. Guardium validates the new connector and displays it in the Configure Universal Connector page. +10. After the offline plug-in is installed and the configuration is uploaded and saved in the Guardium machine, restart the Universal Connector using the ```Disable/Enable``` button. + +## Note: + +### Exporting PostgreSQL or Aurora PostgreSQL Audit Logs to S3 + +You can export PostgreSQL or Aurora PostgreSQL audit logs to an S3 bucket using the following methods: + +1. **Using Extensions (`log_fdw`, `aws_s3`, and `pg_cron`)** + Refer to the [PostgresExtLogsExport](../PostgresOverS3SQSPackage/PostgresExtLogsExport.md) guide for detailed instructions. + +### Limitations: + +System-generated queries may appear in the Full SQL Report when using SQL client tools (e.g., DBeaver, DBVisualizer, pgAdmin), which can result in duplicate query entries. +Role‑based authentication using AWS IAM Role ARNs is not supported for Postgres over S3SQL at this time. \ No newline at end of file diff --git a/filter-plugin/logstash-filter-postgres-guardium/PostgresOverS3SQSPackage/PostgresExtLogsExport.md b/filter-plugin/logstash-filter-postgres-guardium/PostgresOverS3SQSPackage/PostgresExtLogsExport.md new file mode 100644 index 000000000..83fefc69c --- /dev/null +++ b/filter-plugin/logstash-filter-postgres-guardium/PostgresOverS3SQSPackage/PostgresExtLogsExport.md @@ -0,0 +1,498 @@ +## [Export logs to S3 bucket](https://aws.amazon.com/blogs/database/automate-postgresql-log-exports-to-amazon-s3-using-extensions/) + +### Note : +This implementation leverages PostgreSQL extensions such as `log_fdw`, `aws_s3`, and `pg_cron`, and closely follows the approach outlined in the official AWS blog post: [Automate PostgreSQL log exports to Amazon S3 using extensions](https://aws.amazon.com/blogs/database/automate-postgresql-log-exports-to-amazon-s3-using-extensions/). +Please note that this solution is based entirely on the methodology provided by AWS. IBM does not assume responsibility for any future updates, enhancements, or fixes that may be required due to changes in the AWS implementation or related extensions. + +### Limitation : +Client HostName is not available, will be seen as N.A. in Full SQL Report. + +### Create an IAM Role and Policy and Attach the Role to Your RDS for PostgreSQL Instance + +To allow Amazon RDS to export logs or data to Amazon S3, follow these steps: + +--- + +## 1. Create an S3 Bucket + +1. Sign in to the **AWS Management Console**. +2. Navigate to **S3** (Services → Storage → S3). +3. Click **[Create bucket]**. +4. Enter a unique **Bucket name** (e.g., `my-postgres-export-bucket`). +5. Select the same **Region** as your RDS instance. +6. Leave default settings or configure according to your requirements. +7. Click **[Create bucket]**. + +--- + +## 2. Create a Custom IAM Policy for S3 Access + +1. Go to **IAM** (Services → Security, Identity, & Compliance → IAM). +2. In the left navigation, click **Policies**. +3. Click **[Create policy]**. +4. Select the **JSON** tab and paste the following policy: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:AbortMultipartUpload", + "s3:DeleteObject", + "s3:ListMultipartUploadParts", + "s3:PutObject", + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::", + "arn:aws:s3:::/*" + ] + } + ] +} +``` + +> Replace `` with your actual bucket name. + +5. Click **[Next]**, give the policy a name (e.g., `RDSExportToS3Policy`), then click **[Create policy]**. + +--- + +## 3. Create an IAM Role for RDS + +1. In the IAM console, go to **Roles**. +2. Click **[Create role]**. +3. Select **AWS service** as the trusted entity type. +4. Choose **RDS** as the use case. +5. Click **[Next]**. +6. Skip attaching policies for now → Click **[Next]**. +7. Enter a **Role name** (e.g., `RDSExportToS3Role`), then click **[Create role]**. + +--- + +## 4. Modify the Trust Relationship + +1. Click on the newly created role (e.g., `RDSExportToS3Role`). +2. Go to the **Trust relationships** tab. +3. Click **[Edit trust policy]**. +4. Replace the contents with the following: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Service": "rds.amazonaws.com" + }, + "Action": "sts:AssumeRole" + } + ] +} +``` + +5. Click **[Update policy]**. + +--- + +## 5. Attach the Policy to the Role + +1. While on the role details page, go to the **Permissions** tab. +2. Click **[Add permissions]** → **Attach policies**. +3. Find and select the policy you created (e.g., `RDSExportToS3Policy`). +4. Click **[Add permissions]**. + +--- + +## 6. Associate the IAM Role with Your RDS Instance + +### Using AWS Console + +1. Go to **RDS** in the AWS Console. +2. Select your **PostgreSQL DB instance**. +3. Scroll to **Manage IAM roles**. +4. Under **Feature name**, choose `s3Export`. +5. Under **IAM role**, select the IAM role you created (`RDSExportToS3Role`). +6. Click **[Continue]**, then **[Modify DB instance]**. +7. Wait for the instance to return to the **Available** state. + +### Using AWS CLI + +```bash +aws rds add-role-to-db-instance \ + --db-instance-identifier \ + --feature-name s3Export \ + --role-arn arn:aws:iam:::role/RDSExportToS3Role +``` + +Replace `` with your actual RDS instance identifier and `` with your AWS account ID. + +To verify that the role has been associated correctly, you can use: + +```bash +aws rds describe-db-instances \ + --db-instance-identifier \ + --query "DBInstances[*].AssociatedRoles" +``` + +--- +For more information, follow **[Step 4 in the official AWS documentation](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/postgresql-s3-export-access-bucket.html)**. + +### Import PostgreSQL logs into the table using extension log_fdw +To use the `log_fdw` functions, we must first create the extension on the database instance. Connect to the database using psql and run the following command. +```bash + postgres=> CREATE EXTENSION log_fdw; + CREATE EXTENSION +``` +With the extension loaded, we can create a function that loads all the available PostgreSQL DB log files as a table within the database. The definition of the function is available on [GitHub](https://github.com/aws-samples/amazon-rds-and-amazon-aurora-logging-blog/blob/master/scripts/pg_log_fdw_management.sql). +```bash +-- Yaser Raja +-- AWS Professional Services +-- +-- This function uses log_fdw to load all the available RDS / Aurora PostgreSQL DB log files as a table. +-- +-- Usage: +-- 1) Create this function +-- 2) Run the following to load all the log files +-- SELECT public.load_postgres_log_files(); +-- 3) Start looking at the logs +-- SELECT * FROM logs.postgres_logs; +-- +-- Here are the key features: +-- - By default, a table named "postgres_logs" is created in schema "logs". +-- - The schema name and table name can be changed via arguments. +-- - If the table already exists, it will be DROPPED +-- - If the schema 'logs' does not exist, it will be created. +-- - Each log file is loaded as a foreign table and then made child of table logs.postgres_logs +-- - By default, CSV file format is preferred, it can be changed via argument v_prefer_csv +-- - Daily, hourly and minute-based log file name formats are supported for CSV and non-CSV output files +-- - postgresql.log.YYYY-MM-DD-HHMI +-- - postgresql.log.YYYY-MM-DD-HH +-- - postgresql.log.YYYY-MM-DD +-- - Supports the scenario where log files list consist of both the file name formats +-- - When CSV format is used, a check-constraint is added to the child table created for each log file +-- +CREATE OR REPLACE FUNCTION public.load_postgres_log_files(v_schema_name TEXT DEFAULT 'logs', v_table_name TEXT DEFAULT 'postgres_logs', v_prefer_csv BOOLEAN DEFAULT TRUE) +RETURNS TEXT +AS +$BODY$ +DECLARE + v_csv_supported INT := 0; + v_hour_pattern_used INT := 0; + v_filename TEXT; + v_dt timestamptz; + v_dt_max timestamptz; + v_partition_name TEXT; + v_ext_exists INT := 0; + v_server_exists INT := 0; + v_table_exists INT := 0; + v_server_name TEXT := 'log_server'; + v_filelist_sql TEXT; + v_enable_csv BOOLEAN := TRUE; +BEGIN + EXECUTE FORMAT('SELECT count(1) FROM pg_catalog.pg_extension WHERE extname=%L', 'log_fdw') INTO v_ext_exists; + IF v_ext_exists = 0 THEN + CREATE EXTENSION log_fdw; + END IF; + + EXECUTE 'SELECT count(1) FROM pg_catalog.pg_foreign_server WHERE srvname=$1' INTO v_server_exists USING v_server_name; + IF v_server_exists = 0 THEN + EXECUTE FORMAT('CREATE SERVER %s FOREIGN DATA WRAPPER log_fdw', v_server_name); + END IF; + + EXECUTE FORMAT('CREATE SCHEMA IF NOT EXISTS %I', v_schema_name); + + -- Set the search path to make sure the tables are created in dblogs schema + EXECUTE FORMAT('SELECT set_config(%L, %L, TRUE)', 'search_path', v_schema_name); + + -- The db log files are in UTC timezone so that date extracted from filename will also be UTC. + -- Setting timezone to get correct table constraints. + EXECUTE FORMAT('SELECT set_config(%L, %L, TRUE)', 'timezone', 'UTC'); + + -- Check the parent table exists + EXECUTE 'SELECT count(1) FROM information_schema.tables WHERE table_schema=$1 AND table_name=$2' INTO v_table_exists USING v_schema_name, v_table_name; + IF v_table_exists = 1 THEN + RAISE NOTICE 'Table % already exists. It will be dropped.', v_table_name; + EXECUTE FORMAT('SELECT set_config(%L, %L, TRUE)', 'client_min_messages', 'WARNING'); + EXECUTE FORMAT('DROP TABLE %I CASCADE', v_table_name); + EXECUTE FORMAT('SELECT set_config(%L, %L, TRUE)', 'client_min_messages', 'NOTICE'); + v_table_exists = 0; + END IF; + + -- Check the pg log format + SELECT 1 INTO v_csv_supported FROM pg_catalog.pg_settings WHERE name='log_destination' AND setting LIKE '%csvlog%'; + IF v_csv_supported = 1 AND v_prefer_csv = TRUE THEN + RAISE NOTICE 'CSV log format will be used.'; + v_filelist_sql = FORMAT('SELECT file_name FROM public.list_postgres_log_files() WHERE file_name LIKE %L ORDER BY 1 DESC', '%.csv'); + ELSE + RAISE NOTICE 'Default log format will be used.'; + v_filelist_sql = FORMAT('SELECT file_name FROM public.list_postgres_log_files() WHERE file_name NOT LIKE %L ORDER BY 1 DESC', '%.csv'); + v_enable_csv = FALSE; + END IF; + + FOR v_filename IN EXECUTE (v_filelist_sql) + LOOP + RAISE NOTICE 'Processing log file - %', v_filename; + + IF v_enable_csv = TRUE THEN + -- Dynamically checking the file name pattern so that both allowed file names patters are parsed + IF v_filename like 'postgresql.log.____-__-__-____.csv' THEN + v_dt=substring(v_filename from 'postgresql.log.#"%#"-____.csv' for '#')::timestamp + INTERVAL '1 HOUR' * (substring(v_filename from 'postgresql.log.____-__-__-#"%#"__.csv' for '#')::int); + v_dt_max = v_dt + INTERVAL '1 HOUR'; + v_dt=substring(v_filename from 'postgresql.log.#"%#"-____.csv' for '#')::timestamp + INTERVAL '1 HOUR' * (substring(v_filename from 'postgresql.log.____-__-__-#"%#"__.csv' for '#')::int) + INTERVAL '1 MINUTE' * (substring(v_filename from 'postgresql.log.____-__-__-__#"%#".csv' for '#')::int); + ELSIF v_filename like 'postgresql.log.____-__-__-__.csv' THEN + v_dt=substring(v_filename from 'postgresql.log.#"%#"-__.csv' for '#')::timestamp + INTERVAL '1 HOUR' * (substring(v_filename from 'postgresql.log.____-__-__-#"%#".csv' for '#')::int); + v_dt_max = v_dt + INTERVAL '1 HOUR'; + ELSIF v_filename like 'postgresql.log.____-__-__.csv' THEN + v_dt=substring(v_filename from 'postgresql.log.#"%#".csv' for '#')::timestamp; + v_dt_max = v_dt + INTERVAL '1 DAY'; + ELSE + RAISE NOTICE ' Skipping file'; + CONTINUE; + END IF; + ELSE + IF v_filename like 'postgresql.log.____-__-__-____' THEN + v_dt=substring(v_filename from 'postgresql.log.#"%#"-____' for '#')::timestamp + INTERVAL '1 HOUR' * (substring(v_filename from 'postgresql.log.____-__-__-#"%#"__' for '#')::int) + INTERVAL '1 MINUTE' * (substring(v_filename from 'postgresql.log.____-__-__-__#"%#"' for '#')::int); + ELSIF v_filename like 'postgresql.log.____-__-__-__' THEN + v_dt=substring(v_filename from 'postgresql.log.#"%#"-__' for '#')::timestamp + INTERVAL '1 HOUR' * (substring(v_filename from 'postgresql.log.____-__-__-#"%#"' for '#')::int); + ELSIF v_filename like 'postgresql.log.____-__-__' THEN + v_dt=substring(v_filename from 'postgresql.log.#"%#"' for '#')::timestamp; + ELSE + RAISE NOTICE ' Skipping file'; + CONTINUE; + END IF; + END IF; + v_partition_name=CONCAT(v_table_name, '_', to_char(v_dt, 'YYYYMMDD_HH24MI')); + EXECUTE FORMAT('SELECT public.create_foreign_table_for_log_file(%L, %L, %L)', v_partition_name, v_server_name, v_filename); + + IF v_table_exists = 0 THEN + EXECUTE FORMAT('CREATE TABLE %I (LIKE %I INCLUDING ALL)', v_table_name, v_partition_name); + v_table_exists = 1; + END IF; + + EXECUTE FORMAT('ALTER TABLE %I INHERIT %I', v_partition_name, v_table_name); + + IF v_enable_csv = TRUE THEN + EXECUTE FORMAT('ALTER TABLE %I ADD CONSTRAINT check_date_range CHECK (log_time>=%L and log_time < %L)', v_partition_name, v_dt, v_dt_max); + END IF; + + END LOOP; + + RETURN FORMAT('Postgres logs loaded to table %I.%I', v_schema_name, v_table_name); +END; +$BODY$ +LANGUAGE plpgsql; +``` +With the function created, we can run the function to load the PostgreSQL logs into the database. Each time we run the following command, the logs.postgres_logs table is updated with the most recent engine logs. +```bash +postgres=> SELECT public.load_postgres_log_files(); +``` +### Export PostgreSQL logs from table into Amazon S3 using aws_s3 +Now that we have a function to query for new log statements, we use aws_s3 to export the retrieved logs to Amazon S3. From the prerequisites, we should already have an S3 bucket created and we should have attached an IAM role to the DB instance that allows for writing to your S3 bucket. +Create the aws_s3 extension with the following code: +```bash +postgres=> CREATE EXTENSION aws_s3 CASCADE; +CREATE EXTENSION +``` +### Automate the log exports using extension pg_cron +Now that we have the steps to perform log uploads to Amazon S3 using the log_fdw and aws_s3 extensions, we can automate these steps using pg_cron. With pg_cron, we can write database queries to be run on a schedule of our choosing. + +As part of the prerequisites, you should have pg_cron added to the shared_preload_libraries parameter in your database instance's parameter group. After pg_cron is loaded into shared_preload_libraries, you can simply run the following command to create the extension: + +```bash +postgres=> CREATE EXTENSION pg_cron; +CREATE EXTENSION +``` +With pg_cron created, we can use the extension to perform the PostgreSQL log uploads on a cron defined schedule. To do this, we need to schedule a cron job, passing in a name, schedule, and the log export query we want to run. For example, to schedule log uploads every hour with the same query described earlier, we can run the following command: + +Create a table logs.postgres_logs_export_tracker to track last exported log timestamp with `last_exported_log_time`. +```bash + CREATE TABLE IF NOT EXISTS logs.postgres_logs_export_tracker ( + id SERIAL PRIMARY KEY, + last_exported_log_time TIMESTAMPTZ NOT NULL DEFAULT '1970-01-01 00:00:00+00' + ); + + INSERT INTO logs.postgres_logs_export_tracker (last_exported_log_time) VALUES (NOW()); +``` + +Create function `export_postgres_logs_to_s3` to export log to S3 bucket. Please replace parameter `delay_in_minutes`, `S3_bucket_name` and `region` with actual value. +```bash + CREATE OR REPLACE FUNCTION public.export_postgres_logs_to_s3() + RETURNS void LANGUAGE plpgsql SECURITY DEFINER + AS $$ + DECLARE + last_exported TIMESTAMPTZ; + new_last_exported TIMESTAMPTZ; + cutoff_time TIMESTAMPTZ; + latest_tracker_id INT; + export_query TEXT; + export_filename TEXT; + delay_interval INTERVAL := INTERVAL ''; -- Adjust delay window for example 5 minutes + BEGIN + -- 1. Read last exported timestamp from tracker + SELECT id, last_exported_log_time + INTO latest_tracker_id, last_exported + FROM logs.postgres_logs_export_tracker + ORDER BY id DESC + LIMIT 1; + + IF last_exported IS NULL THEN + last_exported := '1970-01-01 00:00:00+00'; + END IF; + + -- 2. Compute cutoff time + cutoff_time := NOW() - delay_interval; + + IF cutoff_time <= last_exported THEN + RAISE NOTICE 'Cutoff time <= last exported time (%), skipping.', last_exported; + RETURN; + END IF; + + -- 3. Load logs (your implementation) + PERFORM public.load_postgres_log_files(); + + -- 4. Build export query + export_query := format($f$ + SELECT * FROM logs.postgres_logs + WHERE message ~* '(AUDIT:)' OR sql_state_code IS DISTINCT FROM '00000' + AND log_time > %L + AND log_time <= %L + ORDER BY log_time, session_id, session_line_num + $f$, last_exported::TEXT, cutoff_time::TEXT); + + export_filename := to_char(NOW(), '"postgres-log-"YYYYMMDD_HH24MI".csv"'); + + -- 5. Export to S3 + PERFORM aws_s3.query_export_to_s3( + export_query, + '', + export_filename, + '', + options := 'format csv, header true' + ); + + -- 6. Update tracker to cutoff time + UPDATE logs.postgres_logs_export_tracker + SET last_exported_log_time = cutoff_time + WHERE id = latest_tracker_id; + + RAISE NOTICE 'Logs exported. Tracker updated to %', cutoff_time; + END; + $$; +``` +Here we have used cron job which will run on every minute, you can customise it by updating the cron job schedule expression i.e. `* * * * *` +```bash + SELECT cron.schedule( + 'postgres-s3-log-uploads-every-minute', + '* * * * *', + 'SELECT public.export_postgres_logs_to_s3();' + ); +``` +If you decide at any time that you want to cancel these automated log uploads, you can unschedule the associated cron job by passing in the job name specified previously. In the following example, the job name is `postgres-s3-log-uploads-every-minute`: +```bash +postgres=> SELECT cron.unschedule('postgres-s3-log-uploads-every-minute'); +unschedule +------------ + t +(1 row) +``` +### Creating the SQS queue +The SQS queue created in these steps will receive messages from the Event Notification (configured in the next section). +These messages, generated by monitoring the S3 bucket, will contain details of the recently added S3 log files. + + +#### Procedure +1. Go to https://console.aws.amazon.com/ +2. Click **Services** +3. Search for SQS and click on **Simple Queue Services** +4. Click **Create Queue**. +5. Select the type as **Standard**. +6. Enter the name for the queue +7. Keep the rest of the default settings + +### Creating a policy for the relevant IAM User +Perform the following steps for the IAM user who is accessing the SQS logs in Guardium: + +#### Procedure +1. Go to https://console.aws.amazon.com/ +2. Go to **IAM service** > **Policies** > **Create Policy**. +3. Select **service as SQS**. +4. Check the following checkboxes: + * **ListQueues** + * **DeleteMessage** + * **DeleteMessageBatch** + * **GetQueueAttributes** + * **GetQueueUrl** + * **ReceiveMessage** + * **ChangeMessageVisibility** + * **ChangeMessageVisibilityBatch** +5. In the resources, specify the ARN of the queue created in the above step. +6. Click **Review policy** and specify the policy name. +7. Click **Create policy**. +8. Assign the policy to the user + 1. Log in to the IAM console as an IAM user (https://console.aws.amazon.com/iam/). + 2. Go to **Users** on the console and select the relevant IAM user to whom you want to give permissions. + Click the **username**. + 3. In the **Permissions tab**, click **Add permissions**. + 4. Click **Attach existing policies directly**. + 5. Search for the policy created and check the checkbox next to it. + 6. Click **Next: Review** + 7. Click **Add permissions** + +### Creating the Event Notification +The Event Notification will get triggered when a new Object is added to S3 bucket and will send the events to the SQS queue. +Follow the steps below to configure the Event Notification + +#### Creating Access Policy to allow Notifications +Update the Access Policy of the SQS queue to allow the Notification Service to send messages to the Queue + +__*Procedure*__ +1. Go to https://console.aws.amazon.com/ +2. Go to **SQS** -> **Queues** +3. Click on the Queue that was created in the above step +4. Go to **Access Policy** +5. Click on **Edit** +6. Add the below details to the existing policy + +``` +{ + "Sid": "example-statement-ID", + "Effect": "Allow", + "Principal": { + "Service": "s3.amazonaws.com" + }, + "Action": "SQS:SendMessage", + "Resource": "", + "Condition": { + "StringEquals": { + "aws:SourceAccount": "" + }, + "ArnLike": { + "aws:SourceArn": "" + } + } +} +``` + + +7. Click on **Save** + + +#### Create the Event Notification +__*Procedure*__ +1. Go to https://console.aws.amazon.com/ +2. Go to **Services**. Search for **S3**. +3. Click on the S3 bucket that is associated with the CloudTrail. +4. Click **Properties** +5. Navigate to **Event Notifications** +6. Click on **Create event notification**. +7. Enter **Event name** +8. Enter the **Prefix** though this is optional, this can be set to capture the specific traffic. +9. In **Event Types** Select **All object create events**. +10. In **Destination** Select **SQS queue**. +11. In **Specify SQS Queue** either **Choose from your SQS queues** option select the Queue name from drop down list or **Enter SQS queue ARN** enter the Queue ARN manually. +12. Click on **Save Changes** \ No newline at end of file diff --git a/filter-plugin/logstash-filter-postgres-guardium/PostgresOverS3SQSPackage/logstash-filter-s3sqs_postgresql_guardium_plugin_filter.zip b/filter-plugin/logstash-filter-postgres-guardium/PostgresOverS3SQSPackage/logstash-filter-s3sqs_postgresql_guardium_plugin_filter.zip new file mode 100644 index 000000000..bff95d677 Binary files /dev/null and b/filter-plugin/logstash-filter-postgres-guardium/PostgresOverS3SQSPackage/logstash-filter-s3sqs_postgresql_guardium_plugin_filter.zip differ diff --git a/filter-plugin/logstash-filter-postgres-guardium/PostgresOverSQSPackage/postgreSQS.conf b/filter-plugin/logstash-filter-postgres-guardium/PostgresOverSQSPackage/postgreSQS.conf index 8a2268e64..716de12fe 100644 --- a/filter-plugin/logstash-filter-postgres-guardium/PostgresOverSQSPackage/postgreSQS.conf +++ b/filter-plugin/logstash-filter-postgres-guardium/PostgresOverSQSPackage/postgreSQS.conf @@ -4,8 +4,8 @@ #*/ input { - sqs { - + custom_sqs { + #Insert the access key and secret that has access to log group access_key_id => "" secret_access_key => "" @@ -17,6 +17,14 @@ input { add_field => {"account_id" => ""} #Insert the Instance name of the database that is to be monitored add_field => {"instance" => ""} + # Optional: Specify a custom endpoint (e.g., proxy) + # endpoint => "https://proxy.company.com" + # Set to true to use AWS's bundled CA certificates for SSL/TLS connections + use_aws_bundled_ca => false + # Optional: Provide additional settings (e.g., custom SSL certificate bundle) + # additional_settings => { + # ssl_ca_bundle => "/usr/share/logstash/third_party/" + # } } } diff --git a/filter-plugin/logstash-filter-postgres-guardium/VERSION b/filter-plugin/logstash-filter-postgres-guardium/VERSION new file mode 100644 index 000000000..21e8796a0 --- /dev/null +++ b/filter-plugin/logstash-filter-postgres-guardium/VERSION @@ -0,0 +1 @@ +1.0.3 diff --git a/filter-plugin/logstash-filter-postgres-guardium/build.gradle b/filter-plugin/logstash-filter-postgres-guardium/build.gradle new file mode 100644 index 000000000..6a38d71bc --- /dev/null +++ b/filter-plugin/logstash-filter-postgres-guardium/build.gradle @@ -0,0 +1,208 @@ +import java.nio.file.Files +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING + +apply plugin: 'java' + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + + dependencies { + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' + classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:4.0.1" + classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' + } +} + +def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + +apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" + +apply plugin: 'jacoco' +apply plugin: 'com.github.johnrengelman.shadow' + +// =========================================================================== +group = 'com.ibm.guardium.s3sqspostgresql' +version = file("VERSION").text.trim() +description = "S3SQS Postgresql-Guardium filter plugin" + +// =========================================================================== +pluginInfo.licenses = ['Apache-2.0'] +pluginInfo.longDescription = "This gem is a Logstash S3SQS postgresql filter plugin required to be installed as part of IBM Security Guardium, Guardium Universal connector configuration. This gem is not a stand-alone program." +pluginInfo.authors = ['IBM'] +pluginInfo.email = [''] +pluginInfo.homepage = "https://github.com/IBM/universal-connectors" +pluginInfo.pluginType = "filter" +pluginInfo.pluginClass = "S3SQSPostgresqlGuardiumPluginFilter" +pluginInfo.pluginName = "s3sqs_postgresql_guardium_plugin_filter" + +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 + +def jacocoVersion = '0.8.4' +def minimumCoverageStr = System.getenv("MINIMUM_COVERAGE") ?: "50.0%" +if (minimumCoverageStr.endsWith("%")) { + minimumCoverageStr = minimumCoverageStr[0..-2] +} +def minimumCoverage = Float.valueOf(minimumCoverageStr) / 100 + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + dependencies { + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' + classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:4.0.1" + } +} + +repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } +} + +dependencies { + implementation 'commons-validator:commons-validator:1.7' + implementation 'org.apache.logging.log4j:log4j-core:2.17.1' + implementation 'org.apache.commons:commons-lang3:3.7' + implementation 'com.google.code.gson:gson:2.8.9' + implementation 'commons-beanutils:commons-beanutils:1.11.0' + implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "build/libs/logstash-core.jar") + implementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "common-*.*.*.jar") + + // ✅ JUnit 4 only + testImplementation 'junit:junit:4.13.2' + testImplementation 'org.mockito:mockito-core:5.17.0' + testImplementation 'org.jruby:jruby-complete:9.2.7.0' + testImplementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "common-*.*.*.jar") +} + +test { + useJUnit() // ✅ Make sure JUnit 4 is used explicitly + testLogging { + events "passed", "skipped", "failed" + } +} + +tasks.register("vendor") { + dependsOn shadowJar + doLast { + String vendorPathPrefix = "vendor/jar-dependencies" + String projectGroupPath = project.group.replaceAll('\\.', '/') + File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") + projectJarFile.mkdirs() + Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) + validatePluginJar(projectJarFile, project.group) + } +} + +shadowJar { + archiveClassifier = null +} + +clean { + delete "${projectDir}/Gemfile" + delete "${projectDir}/${pluginInfo.pluginFullName()}.gemspec" + delete "${projectDir}/lib/" + delete "${projectDir}/vendor/" + new FileNameFinder().getFileNames(projectDir.toString(), "${pluginInfo.pluginFullName()}-*.*.*.gem").each { + delete it + } +} + +tasks.withType(JavaCompile) { + options.encoding = 'UTF-8' +} + +tasks.register("generateRubySupportFiles") { + doLast { + generateRubySupportFilesForPlugin(project.description, project.group, version) + } +} + +tasks.register("removeObsoleteJars") { + doLast { + new FileNameFinder().getFileNames( + projectDir.toString(), + "vendor/**/${pluginInfo.pluginFullName()}*.jar", + "vendor/**/${pluginInfo.pluginFullName()}-${version}.jar" + ).each { delete it } + } +} + +tasks.register("gem") { + dependsOn = [downloadAndInstallJRuby, removeObsoleteJars, vendor, generateRubySupportFiles] + doLast { + buildGem(projectDir, buildDir, "${pluginInfo.pluginFullName()}.gemspec") + } +} + +// ✅ JaCoCo Setup +jacoco { + toolVersion = jacocoVersion +} + +jacocoTestReport { + reports { + html.required = true + xml.required = true + csv.required = true + html.destination file("${buildDir}/reports/jacoco") + csv.destination file("${buildDir}/reports/jacoco/all.csv") + } + executionData.from fileTree(dir: "${buildDir}/jacoco/", includes: ['**/*.exec']) + + afterEvaluate { + classDirectories.setFrom(files(classDirectories.files.collect { + fileTree(dir: it, exclude: []) + })) + } + + doLast { + println "Report -> file://${buildDir}/reports/jacoco/index.html" + } +} + +jacocoTestCoverageVerification { + violationRules { + rule { + limit { + minimum = minimumCoverage + } + } + } + + executionData.from fileTree(dir: "${buildDir}/jacoco/", includes: ['**/*.exec']) + + afterEvaluate { + classDirectories.setFrom(files(classDirectories.files.collect { + fileTree(dir: it, exclude: []) + })) + } +} + +test.finalizedBy jacocoTestReport +check.dependsOn jacocoTestCoverageVerification, jacocoTestReport diff --git a/filter-plugin/logstash-filter-postgres-guardium/gradle/wrapper/gradle-wrapper.jar b/filter-plugin/logstash-filter-postgres-guardium/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 000000000..7454180f2 Binary files /dev/null and b/filter-plugin/logstash-filter-postgres-guardium/gradle/wrapper/gradle-wrapper.jar differ diff --git a/filter-plugin/logstash-filter-postgres-guardium/gradle/wrapper/gradle-wrapper.properties b/filter-plugin/logstash-filter-postgres-guardium/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 000000000..81aa1c044 --- /dev/null +++ b/filter-plugin/logstash-filter-postgres-guardium/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,5 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/filter-plugin/logstash-filter-postgres-guardium/gradlew b/filter-plugin/logstash-filter-postgres-guardium/gradlew new file mode 100755 index 000000000..cccdd3d51 --- /dev/null +++ b/filter-plugin/logstash-filter-postgres-guardium/gradlew @@ -0,0 +1,172 @@ +#!/usr/bin/env sh + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS="" + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin, switch paths to Windows format before running java +if $cygwin ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=$((i+1)) + done + case $i in + (0) set -- ;; + (1) set -- "$args0" ;; + (2) set -- "$args0" "$args1" ;; + (3) set -- "$args0" "$args1" "$args2" ;; + (4) set -- "$args0" "$args1" "$args2" "$args3" ;; + (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=$(save "$@") + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong +if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then + cd "$(dirname "$0")" +fi + +exec "$JAVACMD" "$@" diff --git a/filter-plugin/logstash-filter-postgres-guardium/gradlew.bat b/filter-plugin/logstash-filter-postgres-guardium/gradlew.bat new file mode 100755 index 000000000..f9553162f --- /dev/null +++ b/filter-plugin/logstash-filter-postgres-guardium/gradlew.bat @@ -0,0 +1,84 @@ +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS= + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto init + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto init + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:init +@rem Get command-line arguments, handling Windows variants + +if not "%OS%" == "Windows_NT" goto win9xME_args + +:win9xME_args +@rem Slurp the command line arguments. +set CMD_LINE_ARGS= +set _SKIP=2 + +:win9xME_args_slurp +if "x%~1" == "x" goto execute + +set CMD_LINE_ARGS=%* + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/filter-plugin/logstash-filter-postgres-guardium/mainREADME.md b/filter-plugin/logstash-filter-postgres-guardium/mainREADME.md new file mode 100644 index 000000000..a4cc153f4 --- /dev/null +++ b/filter-plugin/logstash-filter-postgres-guardium/mainREADME.md @@ -0,0 +1,14 @@ +# Postgres Universal Connector + +## Follow this link to set up and use Postgres Universal Connector Logstash Plugin + +[Postgres](./README.md) + +## Follow this link to set up and use Aurora Postgres Universal Connector over CloudWatch Connect + +[AuroraPostgresOverConnectCloudwatch](../../docs/KafkaBasedUCs/AuroraPostgresCloudwatchKafkaConnect.md) + +## Follow this link to set up and use AWS Postgres Universal Connector over CloudWatch Connect + +[AWSPostgresOverConnectCloudwatch](../../docs/KafkaBasedUCs/PostgresqlCloudwatchKafkaConnect.md) + diff --git a/filter-plugin/logstash-filter-postgres-guardium/src/main/java/com/ibm/guardium/s3sqspostgresql/Constants.java b/filter-plugin/logstash-filter-postgres-guardium/src/main/java/com/ibm/guardium/s3sqspostgresql/Constants.java new file mode 100644 index 000000000..93d38a111 --- /dev/null +++ b/filter-plugin/logstash-filter-postgres-guardium/src/main/java/com/ibm/guardium/s3sqspostgresql/Constants.java @@ -0,0 +1,91 @@ +/* +#Copyright 2020-2021 IBM Inc. All rights reserved +#SPDX-License-Identifier: Apache-2.0 +#*/ +package com.ibm.guardium.s3sqspostgresql; + +public interface Constants { + + public static final String CONNECTION_FROM = "connection_from"; + + public static final String APP_USER_NAME = "AWSService"; + + public static final String TIMESTAMP = "timestamp"; + + public static final String STATEMENT = "statement"; + + public static final String CLIENT_IP = "client_ip"; + + public static final String CLIENT_PORT = "port"; + + public static final String SUCCEEDED = "e_level"; + + public static final String DATABASE_NAME = "database_name"; + + public static final String PARSED_MESSAGE = "parsed_message"; + + public static final String USER_NAME = "user_name"; + + public static final String DB_USER = "db_user"; + + public static final String SESSION_ID = "session_id"; + + public static final String FULL_SQL_QUERY = "full_sql_query"; + + public static final String ERROR_MESSAGE = "error_message"; + + public static final String SQL_STATE_CODE = "sql_state_code"; + + public static final String SQL_STATE_CODE_SUCCESS = "00000"; + + public static final String DEFAULT_IP = "0.0.0.0"; + + public static final int DEFAULT_PORT = -1; + + public static final String UNKNOWN_STRING = ""; + + public static final String SERVER_TYPE_STRING = "POSTGRESQL"; + + public static final String DATA_PROTOCOL_STRING = "POSTGRESQL"; + + public static final String LANGUAGE = "PGRS"; + + public static final String SQL_ERROR = "SQL_ERROR"; + + public static final String LOGIN_FAILED = "LOGIN_FAILED"; + + public static final String COMM_PROTOCOL = "AWSApiCall"; + + public static final String MESSAGE = "message"; + + public static final String APPLICATION_NAME = "application_name"; + + public static final String NA = "N.A."; + + public static final String RECORDS = "records"; + + public static final String PREFIX = "pre_fix"; + + public static final String ACCOUNT_ID = "account_id"; + + public static final String ERROR_SEVERITY = "error_severity"; + + public static final String ERROR = "ERROR"; + + public static final String FATAL = "FATAL"; + + public static final String QUERY = "query"; + + public static final String LOG_GROUP = "logGroup"; + + public static final String DB_NAME = "db_name"; + + public static final String LOG_LEVEL = "log_level"; + + public static final String SERVER_HOST_NAME = "server_hostname"; + + public static final String INSTANCE_NAME = "instance_name"; + + public static final String DURATION = "duration"; + +} \ No newline at end of file diff --git a/filter-plugin/logstash-filter-postgres-guardium/src/main/java/com/ibm/guardium/s3sqspostgresql/Parser.java b/filter-plugin/logstash-filter-postgres-guardium/src/main/java/com/ibm/guardium/s3sqspostgresql/Parser.java new file mode 100644 index 000000000..0438a4ca9 --- /dev/null +++ b/filter-plugin/logstash-filter-postgres-guardium/src/main/java/com/ibm/guardium/s3sqspostgresql/Parser.java @@ -0,0 +1,371 @@ +/* +#Copyright 2020-2021 IBM Inc. All rights reserved +#SPDX-License-Identifier: Apache-2.0 +#*/ +package com.ibm.guardium.s3sqspostgresql; + +import co.elastic.logstash.api.Event; +import com.ibm.guardium.universalconnector.commons.structures.Accessor; +import com.ibm.guardium.universalconnector.commons.structures.Construct; +import com.ibm.guardium.universalconnector.commons.structures.Data; +import com.ibm.guardium.universalconnector.commons.structures.ExceptionRecord; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import com.ibm.guardium.universalconnector.commons.structures.Sentence; +import com.ibm.guardium.universalconnector.commons.structures.SentenceObject; +import com.ibm.guardium.universalconnector.commons.structures.SessionLocator; +import com.ibm.guardium.universalconnector.commons.structures.Time; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.text.ParseException; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class Parser { + + private static Logger log = LogManager.getLogger(Parser.class); + + public static Record parseRecord(final Event e) throws ParseException { + + Record record = new Record(); + + record.setAccessor(Parser.parseAccessor(e)); + + if (e.getField(Constants.PARSED_MESSAGE) != null) { + Object parsedMessageObj = e.getField(Constants.PARSED_MESSAGE); + String dbName = Constants.UNKNOWN_STRING; + if (parsedMessageObj instanceof Map) { + Map parsedMessage = (Map) parsedMessageObj; + + String accountId = getAccountId(e); + + if (!parsedMessage.isEmpty() && parsedMessage.get(Constants.DATABASE_NAME) != null + && !accountId.isEmpty()) { + dbName = parsedMessage.get(Constants.DATABASE_NAME).toString(); + } + // Set dbName for Amazon Data Firehose method + else if (null != e.getData().get(Constants.DB_NAME) + && !e.getData().get(Constants.DB_NAME).toString().isEmpty()) { + dbName = e.getData().get(Constants.DB_NAME).toString(); + } + if (dbName.isEmpty()) { + record.setDbName(Constants.NA); + } else { + record.setDbName(accountId + ":" + getInstanceName(e) + ":" + dbName); + } + + if (parsedMessage.get(Constants.CONNECTION_FROM) != null) { + + setSeesionLocator(e, parsedMessage, record, dbName); + } + // Set SessionLocator for Amazon Data Firehose method + else if (null != e.getData().get(Constants.CLIENT_IP) + && null != e.getData().get(Constants.CLIENT_PORT) + && !e.getData().get(Constants.CLIENT_IP).toString().isEmpty() + && !e.getData().get(Constants.CLIENT_PORT).toString().isEmpty()) { + String clientIP = e.getData().get(Constants.CLIENT_IP).toString(); + String clientPort = e.getData().get(Constants.CLIENT_PORT).toString(); + record.setSessionLocator(Parser.parseSessionLocator(e, clientIP, clientPort, dbName)); + + } else { + setDefaultSessionLocator(record); + } + record.setSessionId(Constants.UNKNOWN_STRING); + setOriginalSqlCommand(e, (Map) parsedMessageObj, record); + + // Extract and set execution time (duration) if available + // Duration is expected in the parsed_message as "duration" field in milliseconds + // Convert to microseconds and send as integer (to preserve decimal precision) + // Example: 1.176 ms → 1176 µs, 125.456 ms → 125456 µs + if (parsedMessage.get(Constants.DURATION) != null) { + try { + String durationStr = parsedMessage.get(Constants.DURATION).toString(); + // Parse the duration value in milliseconds + double durationMs = Double.parseDouble(durationStr); + // Convert milliseconds to microseconds by multiplying by 1000 + int executionTimeMicros = (int) Math.round(durationMs * 1000); + record.setExecutionTime(executionTimeMicros); + } catch (NumberFormatException nfe) { + log.warn("Failed to parse duration value: {}", parsedMessage.get(Constants.DURATION), nfe); + } + } + } + } + + record.setAppUserName(Constants.APP_USER_NAME); + + record.setTime(Parser.parseTimestamp(e)); + return record; + } + + private static String getAccountId(Event e) { + String accountId = ""; + if (e.getField(Constants.ACCOUNT_ID) instanceof String) { + accountId = e.getField(Constants.ACCOUNT_ID).toString(); + } else if (e.getField(Constants.ACCOUNT_ID) instanceof List) { + List rawList = (List) e.getField(Constants.ACCOUNT_ID); + List arrayList = new ArrayList<>(rawList); + + if (!arrayList.isEmpty()) { + accountId = String.valueOf(arrayList.get(0)); + } + } + return accountId; + } + + public static String getInstanceName(Event e) { + String res = ""; + if (e.getField(Constants.INSTANCE_NAME) instanceof String) { + res = e.getField(Constants.INSTANCE_NAME).toString(); + } else if (e.getField(Constants.INSTANCE_NAME) instanceof List) { + List rawList = (List) e.getField(Constants.INSTANCE_NAME); + List arrayList = new ArrayList<>(rawList); + + if (!arrayList.isEmpty()) { + res = String.valueOf(arrayList.get(0)); + } + } + return res; + } + + private static void setOriginalSqlCommand(Event e, Map parsedMessageObj, Record record) { + if (parsedMessageObj.get(Constants.SQL_STATE_CODE) != null + && !parsedMessageObj.get(Constants.SQL_STATE_CODE).toString().isEmpty() + && parsedMessageObj.get(Constants.SQL_STATE_CODE).toString().equals(Constants.SQL_STATE_CODE_SUCCESS)) { + Data data = new Data(); + setSQL(e, record, data); + } + // Set SQL for Amazon Data Firehose method + else if (null != parsedMessageObj.get(Constants.LOG_GROUP) + && null != e.getData().get(Constants.LOG_LEVEL) + && null != e.getField(Constants.FULL_SQL_QUERY) + && !parsedMessageObj.get(Constants.LOG_GROUP).toString().isEmpty() + && !e.getData().get(Constants.LOG_LEVEL).toString().isEmpty() + && !e.getField(Constants.FULL_SQL_QUERY).toString().isEmpty()) { + Data data = new Data(); + setSQL(e, record, data); + } else { + // Error message for Amazon Data Firehose method + if (null != e.getData().get(Constants.LOG_LEVEL) + && null != e.getData().get(Constants.ERROR_MESSAGE) + && !e.getData().get(Constants.LOG_LEVEL).toString().isEmpty() + && (e.getData().get(Constants.LOG_LEVEL).toString().equals(Constants.FATAL) + || !e.getData().get(Constants.ERROR_MESSAGE).toString().isEmpty())) { + + ExceptionRecord exceptionRecord = new ExceptionRecord(); + exceptionRecord.setExceptionTypeId(Constants.LOGIN_FAILED); + if (null != e.getData().get(Constants.ERROR_MESSAGE) + && e.getData().get(Constants.ERROR_MESSAGE).toString().isEmpty()) { + String message = e.getData().get(Constants.ERROR_MESSAGE).toString(); + exceptionRecord.setDescription(message); + } + exceptionRecord.setSqlString(Constants.UNKNOWN_STRING); + record.setSessionLocator(setDefaultSessionLocator(record)); + record.setException(exceptionRecord); + } else if (parsedMessageObj.get(Constants.ERROR_SEVERITY) != null && + (parsedMessageObj.get(Constants.ERROR_SEVERITY).equals(Constants.ERROR))) { + + ExceptionRecord exceptionRecord = new ExceptionRecord(); + exceptionRecord.setExceptionTypeId(Constants.SQL_ERROR); + + if (null != parsedMessageObj.get(Constants.MESSAGE) && + !parsedMessageObj.get(Constants.MESSAGE).toString().isEmpty()) { + String message = parsedMessageObj.get(Constants.MESSAGE).toString(); + message = message.replaceAll("\"(\\w+)\"", "$1"); + exceptionRecord.setDescription(message); + } + exceptionRecord.setSqlString(Constants.UNKNOWN_STRING); + record.setSessionLocator(setDefaultSessionLocator(record)); + record.setException(exceptionRecord); + } else { + ExceptionRecord exceptionRecord = new ExceptionRecord(); + exceptionRecord.setExceptionTypeId(Constants.LOGIN_FAILED); + if (null != parsedMessageObj.get(Constants.MESSAGE) && + !parsedMessageObj.get(Constants.MESSAGE).toString().isEmpty()) { + String message = parsedMessageObj.get(Constants.MESSAGE).toString(); + message = message.replaceAll("\"(\\w+)\"", "$1"); + exceptionRecord.setDescription(message); + } + exceptionRecord.setSqlString(Constants.UNKNOWN_STRING); + record.setSessionLocator(setDefaultSessionLocator(record)); + record.setException(exceptionRecord); + } + } + } + + private static void setSQL(Event e, Record record, Data data) { + if (e.getField(Constants.FULL_SQL_QUERY) != null) { + String sqlQuery = e.getField(Constants.FULL_SQL_QUERY).toString(); + if (sqlQuery.contains("\"\"")) { + sqlQuery = sqlQuery.replaceAll("\"\"([^\"]*)\"\"", "\"$1\""); + } + data.setOriginalSqlCommand(sqlQuery); + } else { + data.setOriginalSqlCommand(Constants.NA); + } + record.setData(data); + } + + // Set Default SessionLocator in case of LOGIN_FAILED or ERROR + private static SessionLocator setDefaultSessionLocator(Record record) { + String serverIp = Constants.DEFAULT_IP; + SessionLocator sessionLocator = new SessionLocator(); + sessionLocator.setClientIp(Constants.DEFAULT_IP); + sessionLocator.setClientPort(Constants.DEFAULT_PORT); + sessionLocator.setServerIp(serverIp); + sessionLocator.setServerPort(Constants.DEFAULT_PORT); + sessionLocator.setIpv6(false); + sessionLocator.setClientIpv6(Constants.UNKNOWN_STRING); + sessionLocator.setServerIpv6(Constants.UNKNOWN_STRING); + return sessionLocator; + } + + private static void setSeesionLocator(Event e, Map parsedMessage, Record record, String dbName) { + Object connectionFrom = parsedMessage.get(Constants.CONNECTION_FROM); + String[] connectionArr = connectionFrom.toString().split(":"); + + String clientIP = Constants.UNKNOWN_STRING; + String clientPort = Constants.UNKNOWN_STRING; + + if (connectionArr[0] != null && !connectionArr[0].isEmpty() && + connectionArr[1] != null && !connectionArr[1].isEmpty()) { + clientIP = connectionArr[0]; + clientPort = connectionArr[1]; + } + + record.setSessionLocator(Parser.parseSessionLocator(e, clientIP, clientPort, dbName)); + } + + public static Time parseTimestamp(final Event e) { + long millis = 0; + try { + Object field = e.getField(Constants.TIMESTAMP); + + String dateString; + + if (field instanceof Object[]) { + Object[] arr = (Object[]) field; + if (arr.length > 0) { + dateString = arr[0].toString(); + } else { + dateString = ""; + } + } else { + dateString = field.toString(); + // Handle case where string looks like "[ts1, ts2]" + if (dateString.startsWith("[") && dateString.endsWith("]")) { + String[] parts = dateString.substring(1, dateString.length() - 1).split(","); + if (parts.length > 0) { + dateString = parts[0].trim(); + } + } + } + // Try ISO first + try { + ZonedDateTime parsedTime = ZonedDateTime.parse(dateString, DateTimeFormatter.ISO_DATE_TIME); + millis = parsedTime.toInstant().toEpochMilli(); + } catch (DateTimeParseException isoEx) { + // Fallback to local date time format + log.debug("ISO parse failed, trying fallback format", isoEx); + LocalDateTime localDateTime = LocalDateTime.parse(dateString, DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")); + ZonedDateTime zonedDateTime = localDateTime.atZone(ZoneId.systemDefault()); + millis = zonedDateTime.toInstant().toEpochMilli(); + } + + } catch (Exception exe) { + log.error("parseTimestamp final failure: {}", exe); + } + return new Time(millis, 0, 0); + } + + public static SessionLocator parseSessionLocator(Event e, String clientIP, String clientPort, String dbName) { + + String accountId = getAccountId(e); + String serverIp = accountId + ":" + dbName; + + SessionLocator sessionLocator = new SessionLocator(); + sessionLocator.setClientIp(clientIP); + sessionLocator.setClientPort(Integer.parseInt(clientPort)); + sessionLocator.setServerIp(serverIp); + sessionLocator.setServerPort(Constants.DEFAULT_PORT); + sessionLocator.setIpv6(false); + sessionLocator.setClientIpv6(Constants.UNKNOWN_STRING); + sessionLocator.setServerIpv6(Constants.UNKNOWN_STRING); + return sessionLocator; + } + + + public static Accessor parseAccessor(final Event e) { + Accessor accessor = new Accessor(); + accessor.setDataType(Accessor.DATA_TYPE_GUARDIUM_SHOULD_PARSE_SQL); + accessor.setLanguage(Constants.LANGUAGE); + accessor.setClientHostName(Constants.NA); + accessor.setClientOs(Constants.UNKNOWN_STRING); + accessor.setServerOs(Constants.UNKNOWN_STRING); + + String accId = getAccountId(e); + String instanceName = getInstanceName(e); + + if (e.getField(Constants.PARSED_MESSAGE) != null) { + Object parsedMessageObj = e.getField(Constants.PARSED_MESSAGE); + if (parsedMessageObj instanceof Map) { + Map parsedMessage = (Map) parsedMessageObj; + + String dbUser = Constants.UNKNOWN_STRING; + if (!parsedMessage.isEmpty() && parsedMessage.get(Constants.USER_NAME) != null) { + dbUser = parsedMessage.get(Constants.USER_NAME).toString(); + } + // Set DBUser for Amazon Data Firehose method + else if (null != e.getData() + && null != e.getData().get(Constants.DB_USER) + && !e.getData().get(Constants.DB_USER).toString().isEmpty()) { + dbUser = e.getData().get(Constants.DB_USER).toString(); + } + if (dbUser.isEmpty()) { + accessor.setDbUser(Constants.NA); + } else { + accessor.setDbUser(dbUser); + } + + accessor.setOsUser(Constants.UNKNOWN_STRING); + String accountId = getAccountId(e); + if (parsedMessage.get(Constants.DATABASE_NAME) != null + && !accountId.isEmpty()) { + accessor.setServiceName(accountId + ":" + instanceName + ":" + parsedMessage.get(Constants.DATABASE_NAME).toString()); + } + // Set ServiceName for Amazon Data Firehose method + else if (null != e.getData() + && null != e.getData().get(Constants.DB_NAME) + && !e.getData().get(Constants.DB_NAME).toString().isEmpty()) { + accessor.setServiceName(accountId + ":" + instanceName + ":" + e.getData().get(Constants.DB_NAME).toString()); + } else { + accessor.setServiceName(Constants.NA); + } + + if (parsedMessage.get(Constants.APPLICATION_NAME) != null) { + accessor.setSourceProgram(parsedMessage.get(Constants.APPLICATION_NAME).toString()); + } else { + accessor.setSourceProgram(Constants.UNKNOWN_STRING); + } + } + } + + accessor.setServerType(Constants.SERVER_TYPE_STRING); + accessor.setCommProtocol(Constants.COMM_PROTOCOL); + accessor.setDbProtocol(Constants.DATA_PROTOCOL_STRING); + accessor.setDbProtocolVersion(Constants.UNKNOWN_STRING); + accessor.setClient_mac(Constants.UNKNOWN_STRING); + accessor.setServerDescription(Constants.UNKNOWN_STRING); + accessor.setServerHostName(accId + ":" + instanceName); + + return accessor; + } + + +} \ No newline at end of file diff --git a/filter-plugin/logstash-filter-postgres-guardium/src/main/java/com/ibm/guardium/s3sqspostgresql/S3SQSPostgresqlGuardiumPluginFilter.java b/filter-plugin/logstash-filter-postgres-guardium/src/main/java/com/ibm/guardium/s3sqspostgresql/S3SQSPostgresqlGuardiumPluginFilter.java new file mode 100644 index 000000000..55cae08bb --- /dev/null +++ b/filter-plugin/logstash-filter-postgres-guardium/src/main/java/com/ibm/guardium/s3sqspostgresql/S3SQSPostgresqlGuardiumPluginFilter.java @@ -0,0 +1,111 @@ + +/* +#Copyright 2020-2021 IBM Inc. All rights reserved +#SPDX-License-Identifier: Apache-2.0 +#*/ +package com.ibm.guardium.s3sqspostgresql; + +import java.io.File; +import java.util.Collection; +import java.util.Collections; + + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.core.LoggerContext; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.ibm.guardium.universalconnector.commons.GuardConstants; +import com.ibm.guardium.universalconnector.commons.structures.Record; + +import co.elastic.logstash.api.Configuration; +import co.elastic.logstash.api.Context; +import co.elastic.logstash.api.Event; +import co.elastic.logstash.api.Filter; +import co.elastic.logstash.api.FilterMatchListener; +import co.elastic.logstash.api.LogstashPlugin; +import co.elastic.logstash.api.PluginConfigSpec; + +//class name must match plugin name +@LogstashPlugin(name = "s3sqs_postgresql_guardium_plugin_filter") + +public class S3SQSPostgresqlGuardiumPluginFilter implements Filter { + + public static final String LOG42_CONF = "log4j2uc.properties"; + + static { + try { + String uc_etc = System.getenv("UC_ETC"); + + LoggerContext context = (LoggerContext) LogManager.getContext(false); + + File file = new File(uc_etc + File.separator + LOG42_CONF); + + context.setConfigLocation(file.toURI()); + + } catch (Exception e) { + + System.err.println("Failed to load log4j configuration " + e.getMessage()); + + e.printStackTrace(); + } + } + + private String id; + + public static final PluginConfigSpec SOURCE_CONFIG = PluginConfigSpec.stringSetting("source", "message"); + + private static Logger log = LogManager.getLogger(S3SQSPostgresqlGuardiumPluginFilter.class); + + public S3SQSPostgresqlGuardiumPluginFilter(String id, Configuration config, Context context) { + + // constructors should validate configuration options + + this.id = id; + } + + @Override + + public Collection filter(Collection events, FilterMatchListener matchListener) { + + for (Event e : events) { + if(null != e && null != e.getData()){ + try { + log.debug("Event Now: {} " + e.getData()); + + Record record = Parser.parseRecord(e); + + final GsonBuilder builder = new GsonBuilder(); + + builder.serializeNulls(); + + final Gson gson = builder.create(); + + e.setField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME, gson.toJson(record)); + + matchListener.filterMatched(e); + + } catch (Exception exception) { + log.warn("Failed event {}" , e.getData()); + } + } + + } + return events; + } + + @Override + + public Collection> configSchema() { + + // should return a list of all configuration options for this plugin + + return Collections.singletonList(SOURCE_CONFIG); + } + + @Override + public String getId() { + return this.id; + } +} diff --git a/filter-plugin/logstash-filter-postgres-guardium/src/test/java/com/ibm/guardium/test/s3sqspostgresql/S3SQSPostgresqlGuardiumPluginFilterTest.java b/filter-plugin/logstash-filter-postgres-guardium/src/test/java/com/ibm/guardium/test/s3sqspostgresql/S3SQSPostgresqlGuardiumPluginFilterTest.java new file mode 100644 index 000000000..7a6d4cd4a --- /dev/null +++ b/filter-plugin/logstash-filter-postgres-guardium/src/test/java/com/ibm/guardium/test/s3sqspostgresql/S3SQSPostgresqlGuardiumPluginFilterTest.java @@ -0,0 +1,504 @@ +package com.ibm.guardium.test.s3sqspostgresql; + +import co.elastic.logstash.api.Event; +import com.ibm.guardium.s3sqspostgresql.Constants; +import com.ibm.guardium.s3sqspostgresql.Parser; +import com.ibm.guardium.universalconnector.commons.structures.Accessor; +import com.ibm.guardium.universalconnector.commons.structures.Construct; +import com.ibm.guardium.universalconnector.commons.structures.Data; +import com.ibm.guardium.universalconnector.commons.structures.ExceptionRecord; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import com.ibm.guardium.universalconnector.commons.structures.Sentence; +import com.ibm.guardium.universalconnector.commons.structures.SentenceObject; +import com.ibm.guardium.universalconnector.commons.structures.SessionLocator; +import com.ibm.guardium.universalconnector.commons.structures.Time; +import org.junit.Test; + +import java.text.ParseException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static com.ibm.guardium.s3sqspostgresql.Parser.getInstanceName; +import static org.junit.Assert.*; +import static org.mockito.Mockito.*; + +public class S3SQSPostgresqlGuardiumPluginFilterTest { + + @Test + public void testParseRecordWithSuccessSql() throws ParseException { + Event mockEvent = mock(Event.class); + + Map parsedMsg = new HashMap<>(); + parsedMsg.put(Constants.DATABASE_NAME, "testdb"); + parsedMsg.put(Constants.USER_NAME, "user1"); + parsedMsg.put(Constants.CONNECTION_FROM, "10.0.0.1:1234"); + parsedMsg.put(Constants.SQL_STATE_CODE, Constants.SQL_STATE_CODE_SUCCESS); + + when(mockEvent.getField(Constants.PARSED_MESSAGE)).thenReturn(parsedMsg); + when(mockEvent.getField(Constants.SESSION_ID)).thenReturn("11"); + when(mockEvent.getField(Constants.FULL_SQL_QUERY)).thenReturn("SELECT * FROM table;"); + when(mockEvent.getField(Constants.TIMESTAMP)).thenReturn("2023-11-10T10:15:30Z"); + when(mockEvent.getField(Constants.ACCOUNT_ID)).thenReturn("123456"); + when(mockEvent.getField(Constants.INSTANCE_NAME)).thenReturn("postgres"); + when(mockEvent.getField(Constants.SERVER_HOST_NAME)).thenReturn("123456:postgres"); + + Record record = Parser.parseRecord(mockEvent); + + assertNotNull(record.getData()); + assertEquals("SELECT * FROM table;", record.getData().getOriginalSqlCommand()); + assertEquals("user1", record.getAccessor().getDbUser()); + assertEquals("10.0.0.1", record.getSessionLocator().getClientIp()); + assertEquals("123456:postgres", record.getAccessor().getServerHostName()); + assertEquals("123456:postgres:testdb", record.getAccessor().getServiceName()); + assertEquals("123456:postgres:testdb", record.getDbName()); + } + + @Test + public void testSQLQueryWithDoubleQuote() throws ParseException { + Event mockEvent = mock(Event.class); + + Map parsedMsg = new HashMap<>(); + parsedMsg.put(Constants.DATABASE_NAME, "testdb"); + parsedMsg.put(Constants.USER_NAME, "user1"); + parsedMsg.put(Constants.CONNECTION_FROM, "10.0.0.1:1234"); + parsedMsg.put(Constants.SQL_STATE_CODE, Constants.SQL_STATE_CODE_SUCCESS); + + when(mockEvent.getField(Constants.PARSED_MESSAGE)).thenReturn(parsedMsg); + when(mockEvent.getField(Constants.SESSION_ID)).thenReturn("11"); + when(mockEvent.getField(Constants.FULL_SQL_QUERY)).thenReturn("SELECT COUNT(*) AS \"\"RECORDS\"\" from \"table\";"); + when(mockEvent.getField(Constants.TIMESTAMP)).thenReturn("2023-11-10T10:15:30Z"); + when(mockEvent.getField(Constants.ACCOUNT_ID)).thenReturn("123456"); + when(mockEvent.getField(Constants.INSTANCE_NAME)).thenReturn("postgres"); + when(mockEvent.getField(Constants.SERVER_HOST_NAME)).thenReturn("123456:postgres"); + + Record record = Parser.parseRecord(mockEvent); + + assertNotNull(record.getData()); + assertEquals("SELECT COUNT(*) AS \"RECORDS\" from \"table\";", record.getData().getOriginalSqlCommand()); + assertEquals("user1", record.getAccessor().getDbUser()); + assertEquals("10.0.0.1", record.getSessionLocator().getClientIp()); + assertEquals("123456:postgres", record.getAccessor().getServerHostName()); + assertEquals("123456:postgres:testdb", record.getAccessor().getServiceName()); + assertEquals("123456:postgres:testdb", record.getDbName()); + } + + @Test + public void testParseTimestampValid() { + Event mockEvent = mock(Event.class); + when(mockEvent.getField(Constants.TIMESTAMP)).thenReturn("2023-11-10T10:15:30Z"); + + Time time = Parser.parseTimestamp(mockEvent); + + assertTrue(time.getTimstamp() > 0); + } + + @Test + public void testParseTimestampInvalid() { + Event mockEvent = mock(Event.class); + when(mockEvent.getField(Constants.TIMESTAMP)).thenReturn("invalid-date"); + + Time time = Parser.parseTimestamp(mockEvent); + + assertEquals(0, time.getTimstamp()); + } + + @Test + public void testParseAccessorWithDefaults() { + Event mockEvent = mock(Event.class); + when(mockEvent.getField(Constants.PARSED_MESSAGE)).thenReturn(null); + when(mockEvent.getField(Constants.ACCOUNT_ID)).thenReturn("123456"); + when(mockEvent.getField(Constants.INSTANCE_NAME)).thenReturn("postgres"); + when(mockEvent.getField(Constants.SERVER_HOST_NAME)).thenReturn("123456:postgres"); + + Accessor accessor = Parser.parseAccessor(mockEvent); + + assertEquals(Constants.NA, accessor.getClientHostName()); + assertEquals(Constants.SERVER_TYPE_STRING, accessor.getServerType()); + assertEquals("123456:postgres", accessor.getServerHostName()); + } + + @Test + public void testParseAccessorWithParsedMessage() { + Event mockEvent = mock(Event.class); + Map parsedMsg = new HashMap<>(); + parsedMsg.put(Constants.USER_NAME, "dbuser"); + parsedMsg.put(Constants.DATABASE_NAME, "service_db"); + parsedMsg.put(Constants.APPLICATION_NAME, "app1"); + + when(mockEvent.getField(Constants.PARSED_MESSAGE)).thenReturn(parsedMsg); + when(mockEvent.getField(Constants.ACCOUNT_ID)).thenReturn("123456"); + when(mockEvent.getField(Constants.INSTANCE_NAME)).thenReturn("postgres"); + when(mockEvent.getField(Constants.SERVER_HOST_NAME)).thenReturn("123456:postgres"); + + Accessor accessor = Parser.parseAccessor(mockEvent); + + assertEquals("dbuser", accessor.getDbUser()); + //assertEquals("service_db", accessor.getServiceName()); + assertEquals("app1", accessor.getSourceProgram()); + assertEquals("123456:postgres", accessor.getServerHostName()); + + } + + @Test + public void ErrorTestCaseCheck() throws Exception { + Event event = mock(Event.class); + + Map parsedMessage = new HashMap<>(); + parsedMessage.put("backend_type", "client backend"); + parsedMessage.put("database_name", "mydb"); + parsedMessage.put("internal_query", ""); + parsedMessage.put("detail", ""); + parsedMessage.put("log_time", "2025-07-21 08:22:21.726+00"); + parsedMessage.put("query_id", "0"); + parsedMessage.put("virtual_transaction_id", "5/633"); + parsedMessage.put("session_start_time", "2025-07-21 05:03:18+00"); + parsedMessage.put("process_id", "1510"); + parsedMessage.put("application_name", "DBeaver 25.0.3 - SQLEditor "); + parsedMessage.put("leader_pid", ""); + parsedMessage.put("hint", ""); + parsedMessage.put("query_pos", "1"); + parsedMessage.put("message", "syntax error at or near \"drogp\""); + parsedMessage.put("connection_from", "223.233.87.228:22408"); + parsedMessage.put("session_line_num", "458"); + parsedMessage.put("location", ""); + parsedMessage.put("transaction_id", "0"); + parsedMessage.put("user_name", "Admin123"); + parsedMessage.put("session_id", "687dca16.5e6"); + parsedMessage.put("sql_state_code", "42601"); + parsedMessage.put("error_severity", "ERROR"); + parsedMessage.put("internal_query_pos", ""); + parsedMessage.put("context", ""); + parsedMessage.put("query", "drogp table PANY"); + parsedMessage.put("command_tag", "PARSE"); + + when(event.getField("parsed_message")).thenReturn(parsedMessage); + when(event.getField("account_id")).thenReturn("123456"); + when(event.getField("timestamp")).thenReturn("2025-07-21T08:23:01.903324202Z"); + when(event.getField("session_id")).thenReturn("687dca16.5e6"); + when(event.getField("message")).thenReturn("syntax error at or near \"drogp\""); + when(event.getField("succeeded")).thenReturn("ERROR"); + when(event.getField("prefix")).thenReturn("42601"); + when(event.getField("instance_name")).thenReturn("postgres"); + when(event.getField("server_hostname")).thenReturn("123456:postgres"); + + Record record = Parser.parseRecord(event); + + assertNotNull(record); + assertEquals("123456:postgres:mydb", record.getDbName()); + + assertNotNull(record.getAccessor()); + assertEquals("Admin123", record.getAccessor().getDbUser()); + assertEquals("123456:postgres:mydb", record.getAccessor().getServiceName()); + assertEquals("DBeaver 25.0.3 - SQLEditor ", record.getAccessor().getSourceProgram()); + + assertNotNull(record.getSessionLocator()); + assertEquals(Constants.DEFAULT_IP, record.getSessionLocator().getClientIp()); + assertEquals(Constants.DEFAULT_PORT, record.getSessionLocator().getClientPort()); + assertEquals("123456:postgres", record.getAccessor().getServerHostName()); + + assertNotNull(record.getTime()); + assertTrue(record.getTime().getTimstamp() > 0); + + assertEquals("", record.getSessionId()); + + assertNotNull(record.getException()); + } + + @Test + public void testParseRecordFromSampleEventNow() throws Exception { + Event event = mock(Event.class); + + Map parsedMessage = new HashMap<>(); + parsedMessage.put(Constants.DATABASE_NAME, "mypgdb"); + parsedMessage.put(Constants.USER_NAME, "postgresadmin"); + parsedMessage.put(Constants.CONNECTION_FROM, "223.233.87.243:31637"); + parsedMessage.put(Constants.SQL_STATE_CODE, Constants.SQL_STATE_CODE_SUCCESS); + parsedMessage.put(Constants.APPLICATION_NAME, "NA"); + parsedMessage.put(Constants.MESSAGE, "INSERT INTO test123 (first_name, last_name, department, salary) VALUES ('Alice', 'Johnson', 'HR', 55000.00)"); + + when(event.getField(Constants.PARSED_MESSAGE)).thenReturn(parsedMessage); + when(event.getField(Constants.SESSION_ID)).thenReturn("1433"); + when(event.getField(Constants.FULL_SQL_QUERY)).thenReturn("INSERT INTO test123 (first_name, last_name, department, salary) VALUES ('Alice', 'Johnson', 'HR', 55000.00)"); + when(event.getField(Constants.TIMESTAMP)).thenReturn("2025-08-22T09:57:21Z"); + when(event.getField(Constants.ACCOUNT_ID)).thenReturn("123456"); + when(event.getField(Constants.INSTANCE_NAME)).thenReturn("postgres"); + when(event.getField(Constants.SERVER_HOST_NAME)).thenReturn("123456:postgres"); + + Record record = Parser.parseRecord(event); + + assertNotNull(record); + assertEquals("123456:postgres:mypgdb", record.getDbName()); + + assertNotNull(record.getAccessor()); + assertEquals("postgresadmin", record.getAccessor().getDbUser()); + assertEquals("123456:postgres:mypgdb", record.getAccessor().getServiceName()); + + assertNotNull(record.getSessionLocator()); + assertEquals("223.233.87.243", record.getSessionLocator().getClientIp()); + assertEquals(31637, record.getSessionLocator().getClientPort()); + assertEquals("123456:mypgdb", record.getSessionLocator().getServerIp()); + assertEquals("123456:postgres", record.getAccessor().getServerHostName()); + + assertNotNull(record.getTime()); + assertTrue(record.getTime().getTimstamp() > 0); + + assertEquals("", record.getSessionId()); + + assertNotNull(record.getData()); + assertEquals("INSERT INTO test123 (first_name, last_name, department, salary) VALUES ('Alice', 'Johnson', 'HR', 55000.00)", + record.getData().getOriginalSqlCommand()); + } + + @Test + public void testParseRecordFromEventNowEFG789() throws Exception { + Event event = mock(Event.class); + + Map parsedMessage = new HashMap<>(); + parsedMessage.put(Constants.DATABASE_NAME, "mypgdb"); + parsedMessage.put(Constants.USER_NAME, "postgresadmin"); + parsedMessage.put(Constants.CONNECTION_FROM, "223.233.87.243:27930"); + parsedMessage.put(Constants.SQL_STATE_CODE, Constants.SQL_STATE_CODE_SUCCESS); + parsedMessage.put(Constants.APPLICATION_NAME, "NA"); + parsedMessage.put(Constants.MESSAGE, "INSERT INTO EFG789 (first_name, last_name, department, salary) VALUES ('Bob', 'Smith', 'IT', 65000.00)"); + + when(event.getField(Constants.PARSED_MESSAGE)).thenReturn(parsedMessage); + when(event.getField(Constants.SESSION_ID)).thenReturn("23"); + when(event.getField(Constants.FULL_SQL_QUERY)).thenReturn("INSERT INTO EFG789 (first_name, last_name, department, salary) VALUES ('Bob', 'Smith', 'IT', 65000.00)"); + when(event.getField(Constants.TIMESTAMP)).thenReturn("2025-08-23T07:30:48.706Z"); + when(event.getField(Constants.ACCOUNT_ID)).thenReturn("123456"); + when(event.getField(Constants.INSTANCE_NAME)).thenReturn("postgres"); + when(event.getField(Constants.SERVER_HOST_NAME)).thenReturn("123456:postgres"); + + Record record = Parser.parseRecord(event); + + // Validate main record fields + assertNotNull(record); + assertEquals("123456:postgres:mypgdb", record.getDbName()); + assertEquals("", record.getSessionId()); + + // Validate accessor + assertNotNull(record.getAccessor()); + assertEquals("postgresadmin", record.getAccessor().getDbUser()); + assertEquals("123456:postgres:mypgdb", record.getAccessor().getServiceName()); + assertEquals("NA", record.getAccessor().getSourceProgram()); + assertEquals("123456:postgres", record.getAccessor().getServerHostName()); + + // Validate session locator + assertNotNull(record.getSessionLocator()); + assertEquals("223.233.87.243", record.getSessionLocator().getClientIp()); + assertEquals(27930, record.getSessionLocator().getClientPort()); + assertEquals("123456:mypgdb", record.getSessionLocator().getServerIp()); + + // Validate time + assertNotNull(record.getTime()); + assertTrue(record.getTime().getTimstamp() > 0); + + // Validate SQL command + assertNotNull(record.getData()); + assertEquals("INSERT INTO EFG789 (first_name, last_name, department, salary) VALUES ('Bob', 'Smith', 'IT', 65000.00)", + record.getData().getOriginalSqlCommand()); + } + + @Test + public void testParseRecordFromJHTV123InsertEvent() throws Exception { + Event event = mock(Event.class); + + // Build parsed message as per the event log + Map parsedMessage = new HashMap<>(); + parsedMessage.put(Constants.DATABASE_NAME, "mypgdb"); + parsedMessage.put(Constants.USER_NAME, "postgresadmin"); + parsedMessage.put(Constants.CONNECTION_FROM, "223.233.87.243:29370"); + parsedMessage.put(Constants.SQL_STATE_CODE, Constants.SQL_STATE_CODE_SUCCESS); + parsedMessage.put(Constants.APPLICATION_NAME, "NA"); + parsedMessage.put(Constants.MESSAGE, "INSERT INTO JHTV123 (first_name, last_name, department, salary) VALUES ('Bob', 'Smith', 'IT', 65000.00)"); + + // Mock the event fields + when(event.getField(Constants.PARSED_MESSAGE)).thenReturn(parsedMessage); + when(event.getField(Constants.SESSION_ID)).thenReturn(""); + when(event.getField(Constants.FULL_SQL_QUERY)).thenReturn("INSERT INTO JHTV123 (first_name, last_name, department, salary) VALUES ('Bob', 'Smith', 'IT', 65000.00)"); + when(event.getField(Constants.TIMESTAMP)).thenReturn("2025-08-23T08:55:48Z"); + when(event.getField(Constants.ACCOUNT_ID)).thenReturn("123456"); + when(event.getField(Constants.INSTANCE_NAME)).thenReturn("postgres"); + when(event.getField(Constants.SERVER_HOST_NAME)).thenReturn("123456:postgres"); + + // Parse the record + Record record = Parser.parseRecord(event); + + // Validate main record fields + assertNotNull(record); + assertEquals("123456:postgres:mypgdb", record.getDbName()); + assertEquals("", record.getSessionId()); + + // Validate accessor + assertNotNull(record.getAccessor()); + assertEquals("postgresadmin", record.getAccessor().getDbUser()); + assertEquals("123456:postgres:mypgdb", record.getAccessor().getServiceName()); + assertEquals("NA", record.getAccessor().getSourceProgram()); + assertEquals("123456:postgres", record.getAccessor().getServerHostName()); + + // Validate session locator + assertNotNull(record.getSessionLocator()); + assertEquals("223.233.87.243", record.getSessionLocator().getClientIp()); + assertEquals(29370, record.getSessionLocator().getClientPort()); + assertEquals("123456:mypgdb", record.getSessionLocator().getServerIp()); + + // Validate time + assertNotNull(record.getTime()); + assertTrue(record.getTime().getTimstamp() > 0); + + // Validate SQL command + assertNotNull(record.getData()); + assertEquals( + "INSERT INTO JHTV123 (first_name, last_name, department, salary) VALUES ('Bob', 'Smith', 'IT', 65000.00)", + record.getData().getOriginalSqlCommand() + ); + } + @Test + public void testParseServerHostName() throws ParseException { + Event mockEvent = mock(Event.class); + + Map parsedMsg = new HashMap<>(); + parsedMsg.put(Constants.DATABASE_NAME, "testdb"); + parsedMsg.put(Constants.USER_NAME, "user1"); + parsedMsg.put(Constants.CONNECTION_FROM, "10.0.0.1:1234"); + parsedMsg.put(Constants.SQL_STATE_CODE, Constants.SQL_STATE_CODE_SUCCESS); + + when(mockEvent.getField(Constants.PARSED_MESSAGE)).thenReturn(parsedMsg); + when(mockEvent.getField(Constants.ACCOUNT_ID)).thenReturn("123456"); + when(mockEvent.getField(Constants.INSTANCE_NAME)).thenReturn("postgres"); + + Record record = Parser.parseRecord(mockEvent); + assertEquals("123456:postgres", record.getAccessor().getServerHostName()); + } + + @Test + public void testGetInstanceName_withString() throws Exception{ + Event mockEvent = mock(Event.class); + when(mockEvent.getField(Constants.INSTANCE_NAME)).thenReturn("Postgres"); + + String result = Parser.getInstanceName(mockEvent); + assertEquals("Postgres", result); + } + + @Test + public void testGetInstanceName_withListContainingValue() throws Exception{ + Event mockEvent = mock(Event.class); + List instanceList = Arrays.asList("Postgres", "OtherValue"); + when(mockEvent.getField(Constants.INSTANCE_NAME)).thenReturn(instanceList); + + String result = Parser.getInstanceName(mockEvent); + assertEquals("Postgres", result); + } + + @Test + public void testParseRecordWithDuration() throws ParseException { + Event mockEvent = mock(Event.class); + + Map parsedMsg = new HashMap<>(); + parsedMsg.put(Constants.DATABASE_NAME, "testdb"); + parsedMsg.put(Constants.USER_NAME, "user1"); + parsedMsg.put(Constants.CONNECTION_FROM, "10.171.20.210:18514"); + parsedMsg.put(Constants.SQL_STATE_CODE, Constants.SQL_STATE_CODE_SUCCESS); + parsedMsg.put(Constants.DURATION, "0.919"); // Duration in milliseconds + + when(mockEvent.getField(Constants.PARSED_MESSAGE)).thenReturn(parsedMsg); + when(mockEvent.getField(Constants.SESSION_ID)).thenReturn("28056"); + when(mockEvent.getField(Constants.FULL_SQL_QUERY)).thenReturn("INSERT INTO users (name, email, age) VALUES ('Alice', 'alice@example.com', 25);"); + when(mockEvent.getField(Constants.TIMESTAMP)).thenReturn("2025-11-17T16:21:08Z"); + when(mockEvent.getField(Constants.ACCOUNT_ID)).thenReturn("123456"); + when(mockEvent.getField(Constants.INSTANCE_NAME)).thenReturn("postgres"); + when(mockEvent.getField(Constants.SERVER_HOST_NAME)).thenReturn("123456:postgres"); + + Record record = Parser.parseRecord(mockEvent); + + assertNotNull(record.getData()); + assertEquals("INSERT INTO users (name, email, age) VALUES ('Alice', 'alice@example.com', 25);", record.getData().getOriginalSqlCommand()); + assertEquals("user1", record.getAccessor().getDbUser()); + assertEquals("10.171.20.210", record.getSessionLocator().getClientIp()); + + // Verify execution time is set correctly (0.919 ms → 919 µs) + assertNotNull(record.getExecutionTime()); + assertEquals(Integer.valueOf(919), record.getExecutionTime()); + } + + @Test + public void testParseRecordWithLargerDuration() throws ParseException { + Event mockEvent = mock(Event.class); + + Map parsedMsg = new HashMap<>(); + parsedMsg.put(Constants.DATABASE_NAME, "testdb"); + parsedMsg.put(Constants.USER_NAME, "user1"); + parsedMsg.put(Constants.CONNECTION_FROM, "10.0.0.1:1234"); + parsedMsg.put(Constants.SQL_STATE_CODE, Constants.SQL_STATE_CODE_SUCCESS); + parsedMsg.put(Constants.DURATION, "125.456"); // Duration in milliseconds + + when(mockEvent.getField(Constants.PARSED_MESSAGE)).thenReturn(parsedMsg); + when(mockEvent.getField(Constants.SESSION_ID)).thenReturn("11"); + when(mockEvent.getField(Constants.FULL_SQL_QUERY)).thenReturn("SELECT * FROM large_table WHERE id > 1000;"); + when(mockEvent.getField(Constants.TIMESTAMP)).thenReturn("2023-11-10T10:15:30Z"); + when(mockEvent.getField(Constants.ACCOUNT_ID)).thenReturn("123456"); + when(mockEvent.getField(Constants.INSTANCE_NAME)).thenReturn("postgres"); + when(mockEvent.getField(Constants.SERVER_HOST_NAME)).thenReturn("123456:postgres"); + + Record record = Parser.parseRecord(mockEvent); + + assertNotNull(record.getData()); + assertNotNull(record.getExecutionTime()); + // 125.456 ms → 125456 µs + assertEquals(Integer.valueOf(125456), record.getExecutionTime()); + } + + @Test + public void testParseRecordWithoutDuration() throws ParseException { + Event mockEvent = mock(Event.class); + + Map parsedMsg = new HashMap<>(); + parsedMsg.put(Constants.DATABASE_NAME, "testdb"); + parsedMsg.put(Constants.USER_NAME, "user1"); + parsedMsg.put(Constants.CONNECTION_FROM, "10.0.0.1:1234"); + parsedMsg.put(Constants.SQL_STATE_CODE, Constants.SQL_STATE_CODE_SUCCESS); + // No duration field + + when(mockEvent.getField(Constants.PARSED_MESSAGE)).thenReturn(parsedMsg); + when(mockEvent.getField(Constants.SESSION_ID)).thenReturn("11"); + when(mockEvent.getField(Constants.FULL_SQL_QUERY)).thenReturn("SELECT * FROM table;"); + when(mockEvent.getField(Constants.TIMESTAMP)).thenReturn("2023-11-10T10:15:30Z"); + when(mockEvent.getField(Constants.ACCOUNT_ID)).thenReturn("123456"); + when(mockEvent.getField(Constants.INSTANCE_NAME)).thenReturn("postgres"); + when(mockEvent.getField(Constants.SERVER_HOST_NAME)).thenReturn("123456:postgres"); + + Record record = Parser.parseRecord(mockEvent); + + assertNotNull(record.getData()); + // Execution time should be null when duration is not present + assertNull(record.getExecutionTime()); + } + + @Test + public void testParseRecordWithInvalidDuration() throws ParseException { + Event mockEvent = mock(Event.class); + + Map parsedMsg = new HashMap<>(); + parsedMsg.put(Constants.DATABASE_NAME, "testdb"); + parsedMsg.put(Constants.USER_NAME, "user1"); + parsedMsg.put(Constants.CONNECTION_FROM, "10.0.0.1:1234"); + parsedMsg.put(Constants.SQL_STATE_CODE, Constants.SQL_STATE_CODE_SUCCESS); + parsedMsg.put(Constants.DURATION, "invalid_value"); // Invalid duration + + when(mockEvent.getField(Constants.PARSED_MESSAGE)).thenReturn(parsedMsg); + when(mockEvent.getField(Constants.SESSION_ID)).thenReturn("11"); + when(mockEvent.getField(Constants.FULL_SQL_QUERY)).thenReturn("SELECT * FROM table;"); + when(mockEvent.getField(Constants.TIMESTAMP)).thenReturn("2023-11-10T10:15:30Z"); + when(mockEvent.getField(Constants.ACCOUNT_ID)).thenReturn("123456"); + when(mockEvent.getField(Constants.INSTANCE_NAME)).thenReturn("postgres"); + when(mockEvent.getField(Constants.SERVER_HOST_NAME)).thenReturn("123456:postgres"); + + Record record = Parser.parseRecord(mockEvent); + + assertNotNull(record.getData()); + // Execution time should be null when duration parsing fails + assertNull(record.getExecutionTime()); + } +} diff --git a/filter-plugin/logstash-filter-postgres-guardium/versions.yml b/filter-plugin/logstash-filter-postgres-guardium/versions.yml new file mode 100644 index 000000000..a23c8e1db --- /dev/null +++ b/filter-plugin/logstash-filter-postgres-guardium/versions.yml @@ -0,0 +1,18 @@ +--- +dependencies: + commonsValidator: 1.7 + log4jCore: 2.22.0 + log4jApi: 2.17.2 + commonsLang: 3.7 + gson: 2.8.9 + junit: 4.12 + jrubyComplete: 9.2.7.0 + junitJupiter: 5.7.1 + mockitoAll: 2.0.2-beta + json: 20231013 + parboiledJava: 1.1.8 + javaxJson: 1.1.4 + guava: 32.1.3-jre + commonsText: 1.10.0 + tinkergraphGremlin: 3.6.4 + rdf4jQueryparserSparql: 4.2.4 \ No newline at end of file diff --git a/filter-plugin/logstash-filter-postgres-ibmcloud-guardium/build.gradle b/filter-plugin/logstash-filter-postgres-ibmcloud-guardium/build.gradle index f91e321d3..6790344db 100644 --- a/filter-plugin/logstash-filter-postgres-ibmcloud-guardium/build.gradle +++ b/filter-plugin/logstash-filter-postgres-ibmcloud-guardium/build.gradle @@ -2,6 +2,29 @@ import java.nio.file.Files import static java.nio.file.StandardCopyOption.REPLACE_EXISTING apply plugin: 'java' + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + + dependencies { + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' + classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' + } +} + +def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + apply plugin: 'jacoco' apply plugin: 'eclipse' apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" @@ -22,34 +45,25 @@ pluginInfo.pluginType = "filter" pluginInfo.pluginClass = "ICDPostgresqlGuardiumFilter" pluginInfo.pluginName = "icd_postgresql_guardium_filter" // must match the @LogstashPlugin annotation in the main plugin class // =========================================================================== -sourceCompatibility = 1.8 -targetCompatibility = 1.8 -buildscript { - repositories { - mavenCentral() - jcenter() - maven { - url "https://plugins.gradle.org/m2/" - } - } - - dependencies { - classpath 'com.github.jengelman.gradle.plugins:shadow:6.1.0' - classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' - } -} -def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); -def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 repositories { - mavenCentral() + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } } apply plugin: "com.github.johnrengelman.shadow" //apply plugin: 'org.owasp.dependencycheck' shadowJar { - classifier = null + archiveClassifier = null transform(com.github.jengelman.gradle.plugins.shadow.transformers.Log4j2PluginsCacheFileTransformer) } @@ -59,6 +73,7 @@ dependencies { implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: versions.dependencies.log4jCore implementation 'org.apache.commons:commons-lang3:' + versions.dependencies.commonsLang implementation 'com.google.code.gson:gson:' + versions.dependencies.gson + implementation group: 'commons-beanutils', name: 'commons-beanutils', version: versions.dependencies.commonsBeanutils testImplementation group: 'org.mockito', name: 'mockito-all', version: versions.dependencies.mockitoAll testImplementation 'org.junit.jupiter:junit-jupiter:' + versions.dependencies.junitJupiter implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "build/libs/logstash-core-*.*.*.jar") @@ -80,7 +95,6 @@ tasks.withType(JavaCompile) { /*spotbugs { ignoreFailures = true showStackTraces = false - reportsDir = file("$buildDir/spotbugs") } spotbugsMain { @@ -95,8 +109,8 @@ spotbugsMain { } tasks.withType(com.github.spotbugs.snom.SpotBugsTask) { reports { - xml.enabled = false - html.enabled = true + xml.required = false + html.required = true } }*/ @@ -117,7 +131,6 @@ task vendor(dependsOn: shadowJar) { File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") projectJarFile.mkdirs() Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) - validatePluginJar(projectJarFile, project.group) } } @@ -148,8 +161,8 @@ tasks.register("gem"){ jacocoTestReport { reports { - xml.enabled true - html.enabled true + xml.required = true + html.required = true } afterEvaluate { // (optional) : to exclude classes / packages from coverage diff --git a/filter-plugin/logstash-filter-postgres-ibmcloud-guardium/gradle/wrapper/gradle-wrapper.properties b/filter-plugin/logstash-filter-postgres-ibmcloud-guardium/gradle/wrapper/gradle-wrapper.properties index aa991fcea..81aa1c044 100644 --- a/filter-plugin/logstash-filter-postgres-ibmcloud-guardium/gradle/wrapper/gradle-wrapper.properties +++ b/filter-plugin/logstash-filter-postgres-ibmcloud-guardium/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/filter-plugin/logstash-filter-progressdb-guardium/build.gradle b/filter-plugin/logstash-filter-progressdb-guardium/build.gradle index 569b1af9f..f1e2072df 100644 --- a/filter-plugin/logstash-filter-progressdb-guardium/build.gradle +++ b/filter-plugin/logstash-filter-progressdb-guardium/build.gradle @@ -2,6 +2,30 @@ import java.nio.file.Files import static java.nio.file.StandardCopyOption.REPLACE_EXISTING apply plugin: 'java' + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + + dependencies { + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' + classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.1.0" + classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' + } +} + +def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" // =========================================================================== @@ -20,10 +44,10 @@ pluginInfo.pluginClass = "ProgressGuardiumPluginFilter" pluginInfo.pluginName = "progress_guardium_plugin_filter" // must match the @LogstashPlugin annotation in the main plugin class // =========================================================================== -sourceCompatibility = 1.8 -targetCompatibility = 1.8 +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 -def jacocoVersion = '0.8.4' +def jacocoVersion = '0.8.11' // minimumCoverage can be set by Travis ENV def minimumCoverageStr = System.getenv("MINIMUM_COVERAGE") ?: "50.0%" if (minimumCoverageStr.endsWith("%")) { @@ -31,28 +55,17 @@ if (minimumCoverageStr.endsWith("%")) { } def minimumCoverage = Float.valueOf(minimumCoverageStr) / 100 -buildscript { - repositories { - maven { - url "https://plugins.gradle.org/m2/" - } - mavenCentral() - jcenter() - } - - dependencies { - classpath 'com.github.jengelman.gradle.plugins:shadow:4.0.4' - classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.0.0-RC2" - classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' - } -} - -def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); -def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) repositories { - mavenCentral() + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } } tasks.register("vendor"){ @@ -63,14 +76,13 @@ tasks.register("vendor"){ File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") projectJarFile.mkdirs() Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) - validatePluginJar(projectJarFile, project.group) } } apply plugin: 'com.github.johnrengelman.shadow' shadowJar { - classifier = null + archiveClassifier = null } dependencies { @@ -78,6 +90,7 @@ dependencies { implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: versions.dependencies.log4jCore implementation 'org.apache.commons:commons-lang3:' + versions.dependencies.commonsLang implementation 'com.google.code.gson:gson:' + versions.dependencies.gson + implementation group: 'commons-beanutils', name: 'commons-beanutils', version: versions.dependencies.commonsBeanutils implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "build/libs/logstash-core-*.*.*.jar") implementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "common-*.*.*.jar") @@ -101,6 +114,17 @@ tasks.withType(JavaCompile) { options.encoding = 'UTF-8' } +test { + jvmArgs = [ + '--add-opens=java.base/java.lang=ALL-UNNAMED', + '--add-opens=java.base/java.util=ALL-UNNAMED', + '--add-opens=java.base/java.text=ALL-UNNAMED', + '--add-opens=java.base/java.lang.reflect=ALL-UNNAMED', + '--add-opens=java.base/sun.util.resources=ALL-UNNAMED', + '--add-opens=java.base/sun.util.cldr=ALL-UNNAMED' + ] +} + tasks.register("generateRubySupportFiles") { @@ -135,17 +159,16 @@ apply plugin: "org.barfuin.gradle.jacocolog" // ------------------------------------ jacoco { toolVersion = "${jacocoVersion}" - reportsDir = file("$buildDir/reports/jacoco") } jacocoTestReport { // You will see "Report -> file://...." at the end of a JaCoCo build // If no output, run this first: ./gradlew test reports { - html.enabled true - xml.enabled true - csv.enabled true - html.destination file("${buildDir}/reports/jacoco") - csv.destination file("${buildDir}/reports/jacoco/all.csv") + html.required = true + xml.required = true + csv.required = true + html.outputLocation = file("${buildDir}/reports/jacoco") + csv.outputLocation = file("${buildDir}/reports/jacoco/all.csv") } executionData.from fileTree(dir: "${buildDir}/jacoco/", includes: [ '**/*.exec' diff --git a/filter-plugin/logstash-filter-progressdb-guardium/gradle/wrapper/gradle-wrapper.properties b/filter-plugin/logstash-filter-progressdb-guardium/gradle/wrapper/gradle-wrapper.properties index aa991fcea..81aa1c044 100644 --- a/filter-plugin/logstash-filter-progressdb-guardium/gradle/wrapper/gradle-wrapper.properties +++ b/filter-plugin/logstash-filter-progressdb-guardium/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/filter-plugin/logstash-filter-progressdb-guardium/src/main/java/com/ibm/guardium/progress/Parser.java b/filter-plugin/logstash-filter-progressdb-guardium/src/main/java/com/ibm/guardium/progress/Parser.java index 49c4cd2c4..3442fa9af 100644 --- a/filter-plugin/logstash-filter-progressdb-guardium/src/main/java/com/ibm/guardium/progress/Parser.java +++ b/filter-plugin/logstash-filter-progressdb-guardium/src/main/java/com/ibm/guardium/progress/Parser.java @@ -5,7 +5,15 @@ package com.ibm.guardium.progress; import com.google.gson.JsonObject; -import com.ibm.guardium.universalconnector.commons.structures.*; +import com.ibm.guardium.universalconnector.commons.structures.Accessor; +import com.ibm.guardium.universalconnector.commons.structures.Construct; +import com.ibm.guardium.universalconnector.commons.structures.Data; +import com.ibm.guardium.universalconnector.commons.structures.ExceptionRecord; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import com.ibm.guardium.universalconnector.commons.structures.Sentence; +import com.ibm.guardium.universalconnector.commons.structures.SentenceObject; +import com.ibm.guardium.universalconnector.commons.structures.SessionLocator; +import com.ibm.guardium.universalconnector.commons.structures.Time; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.text.ParseException; diff --git a/filter-plugin/logstash-filter-progressdb-guardium/src/test/java/com/ibm/guardium/progress/ParserTest.java b/filter-plugin/logstash-filter-progressdb-guardium/src/test/java/com/ibm/guardium/progress/ParserTest.java index 66add037e..783326ff1 100644 --- a/filter-plugin/logstash-filter-progressdb-guardium/src/test/java/com/ibm/guardium/progress/ParserTest.java +++ b/filter-plugin/logstash-filter-progressdb-guardium/src/test/java/com/ibm/guardium/progress/ParserTest.java @@ -5,7 +5,15 @@ package com.ibm.guardium.progress; import com.google.gson.JsonObject; -import com.ibm.guardium.universalconnector.commons.structures.*; +import com.ibm.guardium.universalconnector.commons.structures.Accessor; +import com.ibm.guardium.universalconnector.commons.structures.Construct; +import com.ibm.guardium.universalconnector.commons.structures.Data; +import com.ibm.guardium.universalconnector.commons.structures.ExceptionRecord; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import com.ibm.guardium.universalconnector.commons.structures.Sentence; +import com.ibm.guardium.universalconnector.commons.structures.SentenceObject; +import com.ibm.guardium.universalconnector.commons.structures.SessionLocator; +import com.ibm.guardium.universalconnector.commons.structures.Time; import org.junit.Test; import java.text.ParseException; import static com.ibm.guardium.progress.Constants.minOffset; diff --git a/filter-plugin/logstash-filter-pubsub-apachesolr-guardium/build.gradle b/filter-plugin/logstash-filter-pubsub-apachesolr-guardium/build.gradle index b3f6c8b7a..9a8c6037d 100644 --- a/filter-plugin/logstash-filter-pubsub-apachesolr-guardium/build.gradle +++ b/filter-plugin/logstash-filter-pubsub-apachesolr-guardium/build.gradle @@ -2,6 +2,30 @@ import java.nio.file.Files import static java.nio.file.StandardCopyOption.REPLACE_EXISTING apply plugin: 'java' + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + + dependencies { + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' + classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.1.0" + classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' + } +} + +def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" @@ -21,10 +45,10 @@ pluginInfo.pluginClass = "ApacheSolrGcpConnector" pluginInfo.pluginName = "apache_solr_gcp_connector" // must match the @LogstashPlugin annotation in the main plugin class // =========================================================================== -sourceCompatibility = 1.8 -targetCompatibility = 1.8 +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 -def jacocoVersion = '0.8.4' +def jacocoVersion = '0.8.11' // minimumCoverage can be set by Travis ENV def minimumCoverageStr = System.getenv("MINIMUM_COVERAGE") ?: "50.0%" if (minimumCoverageStr.endsWith("%")) { @@ -32,28 +56,17 @@ if (minimumCoverageStr.endsWith("%")) { } def minimumCoverage = Float.valueOf(minimumCoverageStr) / 100 -buildscript { - repositories { - maven { - url "https://plugins.gradle.org/m2/" - } - mavenCentral() - jcenter() - } - - dependencies { - classpath 'com.github.jengelman.gradle.plugins:shadow:4.0.4' - classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.0.0-RC2" - classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' - } -} - -def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); -def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) repositories { - mavenCentral() + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } } tasks.register("vendor"){ @@ -64,14 +77,13 @@ tasks.register("vendor"){ File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") projectJarFile.mkdirs() Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) - validatePluginJar(projectJarFile, project.group) } } apply plugin: 'com.github.johnrengelman.shadow' shadowJar { - classifier = null + archiveClassifier = null } dependencies { @@ -79,6 +91,7 @@ dependencies { implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: versions.dependencies.log4jCore implementation 'com.google.code.gson:gson:' + versions.dependencies.gson implementation 'org.apache.commons:commons-lang3:' + versions.dependencies.commonsLang + implementation group: 'commons-beanutils', name: 'commons-beanutils', version: versions.dependencies.commonsBeanutils testImplementation group: 'org.mockito', name: 'mockito-all', version: versions.dependencies.mockitoAll testImplementation 'org.junit.jupiter:junit-jupiter:' + versions.dependencies.junitJupiter @@ -107,6 +120,17 @@ tasks.withType(JavaCompile) { options.encoding = 'UTF-8' } +test { + jvmArgs = [ + '--add-opens=java.base/java.lang=ALL-UNNAMED', + '--add-opens=java.base/java.util=ALL-UNNAMED', + '--add-opens=java.base/java.text=ALL-UNNAMED', + '--add-opens=java.base/java.lang.reflect=ALL-UNNAMED', + '--add-opens=java.base/sun.util.resources=ALL-UNNAMED', + '--add-opens=java.base/sun.util.cldr=ALL-UNNAMED' + ] +} + tasks.register("generateRubySupportFiles") { @@ -149,17 +173,16 @@ apply plugin: "org.barfuin.gradle.jacocolog" // ------------------------------------ jacoco { toolVersion = "${jacocoVersion}" - reportsDir = file("$buildDir/reports/jacoco") } jacocoTestReport { // You will see "Report -> file://...." at the end of a JaCoCo build // If no output, run this first: ./gradlew test reports { - html.enabled true - xml.enabled true - csv.enabled true - html.destination file("${buildDir}/reports/jacoco") - csv.destination file("${buildDir}/reports/jacoco/all.csv") + html.required = true + xml.required = true + csv.required = true + html.outputLocation = file("${buildDir}/reports/jacoco") + csv.outputLocation = file("${buildDir}/reports/jacoco/all.csv") } executionData.from fileTree(dir: "${buildDir}/jacoco/", includes: [ '**/*.exec' diff --git a/filter-plugin/logstash-filter-pubsub-apachesolr-guardium/gradle/wrapper/gradle-wrapper.properties b/filter-plugin/logstash-filter-pubsub-apachesolr-guardium/gradle/wrapper/gradle-wrapper.properties index aa991fcea..81aa1c044 100644 --- a/filter-plugin/logstash-filter-pubsub-apachesolr-guardium/gradle/wrapper/gradle-wrapper.properties +++ b/filter-plugin/logstash-filter-pubsub-apachesolr-guardium/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/filter-plugin/logstash-filter-pubsub-apachesolr-guardium/mainREADME.md b/filter-plugin/logstash-filter-pubsub-apachesolr-guardium/mainREADME.md new file mode 100644 index 000000000..c99a970b6 --- /dev/null +++ b/filter-plugin/logstash-filter-pubsub-apachesolr-guardium/mainREADME.md @@ -0,0 +1,10 @@ +# ApacheSolr Universal Connector + +## Follow this link to set up and use ApacheSolr Universal Connector over PubSub Logstash Plugin + +[ApacheSolrOverPubSub](./README.md) + +## Follow this link to set up and use ApacheSolr Universal Connector over PubSub Connect + +[ApacheSolrOverConnectPubSub](../../docs/KafkaBasedUCs/ApachesolrPubsubkafkaConnect.md) + diff --git a/filter-plugin/logstash-filter-pubsub-bigquery-guardium/README.md b/filter-plugin/logstash-filter-pubsub-bigquery-guardium/README.md index 186e84f9d..45ee40ced 100644 --- a/filter-plugin/logstash-filter-pubsub-bigquery-guardium/README.md +++ b/filter-plugin/logstash-filter-pubsub-bigquery-guardium/README.md @@ -139,6 +139,7 @@ project-id_bigquery.googleapis.com. (protoPayload.metadata.jobChange.job.jobStatus.jobState = DONE AND -protoPayload.metadata.jobChange.job.jobConfig.queryConfig.statementType = "SCRIPT"))" 13. The parser does not support queries in which a keyword is used as a table name or column name, or in scenarios of nested parameters inside functions. 14. The BigQuery audit log doesn’t include login failed logs, so these will not appear in the guardium LOGIN_FAILED report. +15. Syntactically correct SQL queries that fail on Database will be captured only in SQL_Error report. ## Configuring the BigQuery filter in Guardium The Guardium universal connector is the Guardium entry point for native audit/data_access logs. The Guardium universal connector identifies and parses the received events, and converts them to a standard Guardium format. The output of the Guardium universal connector is forwarded to the Guardium sniffer on the collector, for policy and auditing enforcements. Configure Guardium to read the native audit/data_access logs by customizing the BigQuery template. diff --git a/filter-plugin/logstash-filter-pubsub-bigquery-guardium/build.gradle b/filter-plugin/logstash-filter-pubsub-bigquery-guardium/build.gradle index 480466a05..29840a482 100644 --- a/filter-plugin/logstash-filter-pubsub-bigquery-guardium/build.gradle +++ b/filter-plugin/logstash-filter-pubsub-bigquery-guardium/build.gradle @@ -2,6 +2,30 @@ import java.nio.file.Files import static java.nio.file.StandardCopyOption.REPLACE_EXISTING apply plugin: 'java' + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + + dependencies { + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' + classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.1.0" + classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' + } +} + +def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + apply plugin: 'jacoco' apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" apply plugin: "com.github.johnrengelman.shadow" @@ -24,10 +48,10 @@ pluginInfo.pluginClass = "BigQueryGuardiumFilter" pluginInfo.pluginName = "big_query_guardium_filter" // must match the @LogstashPlugin annotation in the main plugin class // =========================================================================== -sourceCompatibility = 1.8 -targetCompatibility = 1.8 +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 -def jacocoVersion = '0.8.4' +def jacocoVersion = '0.8.11' // minimumCoverage can be set by Travis ENV def minimumCoverageStr = System.getenv("MINIMUM_COVERAGE") ?: "50.0%" if (minimumCoverageStr.endsWith("%")) { @@ -35,28 +59,17 @@ if (minimumCoverageStr.endsWith("%")) { } def minimumCoverage = Float.valueOf(minimumCoverageStr) / 100 -buildscript { - repositories { - maven { - url "https://plugins.gradle.org/m2/" - } - mavenCentral() - jcenter() - } - - dependencies { - classpath 'com.github.jengelman.gradle.plugins:shadow:4.0.4' - classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.0.0-RC2" - classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' - } -} - -def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); -def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) repositories { - mavenCentral() + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } } tasks.register("vendor"){ @@ -67,14 +80,13 @@ tasks.register("vendor"){ File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") projectJarFile.mkdirs() Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) - validatePluginJar(projectJarFile, project.group) } } apply plugin: 'com.github.johnrengelman.shadow' shadowJar { - classifier = null + archiveClassifier = null } @@ -82,12 +94,13 @@ dependencies { implementation 'com.google.code.gson:gson:' + versions.dependencies.gson implementation 'org.apache.commons:commons-lang3:' + versions.dependencies.commonsLang implementation 'commons-validator:commons-validator:' + versions.dependencies.commonsValidator + implementation group: 'commons-beanutils', name: 'commons-beanutils', version: versions.dependencies.commonsBeanutils testImplementation group: 'org.mockito', name: 'mockito-all', version: versions.dependencies.mockitoAll testImplementation 'org.junit.jupiter:junit-jupiter:' + versions.dependencies.junitJupiter testImplementation 'org.jruby:jruby-complete:' + versions.dependencies.jrubyComplete implementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "common-*.*.*.jar") - implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "build/libs/logstash-core-*.*.*.jar") + implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "build/libs/logstash-core.jar") implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: versions.dependencies.log4jCore implementation group: 'org.json', name: 'json', version: versions.dependencies.json implementation group: 'org.parboiled', name: 'parboiled-java', version: versions.dependencies.parboiledJava @@ -110,6 +123,15 @@ tasks.withType(JavaCompile) { } test { useJUnitPlatform() + jvmArgs = [ + '--add-opens=java.base/java.lang=ALL-UNNAMED', + '--add-opens=java.base/java.util=ALL-UNNAMED', + '--add-opens=java.base/java.text=ALL-UNNAMED', + '--add-opens=java.base/java.lang.reflect=ALL-UNNAMED', + '--add-opens=java.base/sun.util.resources=ALL-UNNAMED', + '--add-opens=java.base/sun.util.cldr=ALL-UNNAMED', + '--add-opens=java.base/jdk.internal.reflect=ALL-UNNAMED' + ] } tasks.register("generateRubySupportFiles") { doLast { @@ -151,17 +173,16 @@ apply plugin: "org.barfuin.gradle.jacocolog" // ------------------------------------ jacoco { toolVersion = "${jacocoVersion}" - reportsDir = file("$buildDir/reports/jacoco") } jacocoTestReport { // You will see "Report -> file://...." at the end of a JaCoCo build // If no output, run this first: ./gradlew test reports { - html.enabled true - xml.enabled true - csv.enabled true - html.destination file("${buildDir}/reports/jacoco") - csv.destination file("${buildDir}/reports/jacoco/all.csv") + html.required = true + xml.required = true + csv.required = true + html.outputLocation = file("${buildDir}/reports/jacoco") + csv.outputLocation = file("${buildDir}/reports/jacoco/all.csv") } executionData.from fileTree(dir: "${buildDir}/jacoco/", includes: [ '**/*.exec' diff --git a/filter-plugin/logstash-filter-pubsub-bigquery-guardium/gradle/wrapper/gradle-wrapper.properties b/filter-plugin/logstash-filter-pubsub-bigquery-guardium/gradle/wrapper/gradle-wrapper.properties index aa991fcea..81aa1c044 100644 --- a/filter-plugin/logstash-filter-pubsub-bigquery-guardium/gradle/wrapper/gradle-wrapper.properties +++ b/filter-plugin/logstash-filter-pubsub-bigquery-guardium/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/filter-plugin/logstash-filter-pubsub-bigquery-guardium/mainREADME.md b/filter-plugin/logstash-filter-pubsub-bigquery-guardium/mainREADME.md new file mode 100644 index 000000000..e14fe9bb8 --- /dev/null +++ b/filter-plugin/logstash-filter-pubsub-bigquery-guardium/mainREADME.md @@ -0,0 +1,10 @@ +# BigQuery Universal Connector + +## Follow this link to set up and use BigQuery Universal Connector over PubSub Logstash Plugin + +[BigQueryOverPubSub](./README.md) + +## Follow this link to set up and use BigQuery Universal Connector over PubSub Connect + +[BigQueryOverConnectPubSub](../../docs/KafkaBasedUCs/BigqueryPubsubKafkaConnect.md) + diff --git a/filter-plugin/logstash-filter-pubsub-bigtable-guardium/build.gradle b/filter-plugin/logstash-filter-pubsub-bigtable-guardium/build.gradle index 0793da3de..94078d7a1 100644 --- a/filter-plugin/logstash-filter-pubsub-bigtable-guardium/build.gradle +++ b/filter-plugin/logstash-filter-pubsub-bigtable-guardium/build.gradle @@ -2,6 +2,30 @@ import java.nio.file.Files import static java.nio.file.StandardCopyOption.REPLACE_EXISTING apply plugin: 'java' + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + + dependencies { + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' + classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.1.0" + classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' + } +} + +def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + apply plugin: 'jacoco' apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" apply plugin: "com.github.johnrengelman.shadow" @@ -24,10 +48,10 @@ pluginInfo.pluginClass = "BigTableGuardiumFilter" pluginInfo.pluginName = "big_table_guardium_filter" // must match the @LogstashPlugin annotation in the main plugin class // =========================================================================== -sourceCompatibility = 11 -targetCompatibility = 11 +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 -def jacocoVersion = '0.8.4' +def jacocoVersion = '0.8.11' // minimumCoverage can be set by Travis ENV def minimumCoverageStr = System.getenv("MINIMUM_COVERAGE") ?: "50.0%" if (minimumCoverageStr.endsWith("%")) { @@ -35,28 +59,17 @@ if (minimumCoverageStr.endsWith("%")) { } def minimumCoverage = Float.valueOf(minimumCoverageStr) / 100 -buildscript { - repositories { - maven { - url "https://plugins.gradle.org/m2/" - } - mavenCentral() - jcenter() - } - - dependencies { - classpath 'com.github.jengelman.gradle.plugins:shadow:4.0.4' - classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.0.0-RC2" - classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' - } -} - -def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); -def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) repositories { - mavenCentral() + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } } tasks.register("vendor"){ @@ -67,14 +80,13 @@ tasks.register("vendor"){ File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") projectJarFile.mkdirs() Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) - validatePluginJar(projectJarFile, project.group) } } apply plugin: 'com.github.johnrengelman.shadow' shadowJar { - classifier = null + archiveClassifier = null } @@ -82,6 +94,7 @@ dependencies { implementation 'com.google.code.gson:gson:' + versions.dependencies.gson implementation 'org.apache.commons:commons-lang3:' + versions.dependencies.commonsLang implementation 'commons-validator:commons-validator:' + versions.dependencies.commonsValidator + implementation group: 'commons-beanutils', name: 'commons-beanutils', version: versions.dependencies.commonsBeanutils testImplementation group: 'org.mockito', name: 'mockito-all', version: versions.dependencies.mockitoAll testImplementation 'org.junit.jupiter:junit-jupiter:' + versions.dependencies.junitJupiter @@ -151,17 +164,16 @@ apply plugin: "org.barfuin.gradle.jacocolog" // ------------------------------------ jacoco { toolVersion = "${jacocoVersion}" - reportsDir = file("$buildDir/reports/jacoco") } jacocoTestReport { // You will see "Report -> file://...." at the end of a JaCoCo build // If no output, run this first: ./gradlew test reports { - html.enabled true - xml.enabled true - csv.enabled true - html.destination file("${buildDir}/reports/jacoco") - csv.destination file("${buildDir}/reports/jacoco/all.csv") + html.required = true + xml.required = true + csv.required = true + html.outputLocation = file("${buildDir}/reports/jacoco") + csv.outputLocation = file("${buildDir}/reports/jacoco/all.csv") } executionData.from fileTree(dir: "${buildDir}/jacoco/", includes: [ '**/*.exec' diff --git a/filter-plugin/logstash-filter-pubsub-bigtable-guardium/gradle/wrapper/gradle-wrapper.properties b/filter-plugin/logstash-filter-pubsub-bigtable-guardium/gradle/wrapper/gradle-wrapper.properties index aa991fcea..81aa1c044 100644 --- a/filter-plugin/logstash-filter-pubsub-bigtable-guardium/gradle/wrapper/gradle-wrapper.properties +++ b/filter-plugin/logstash-filter-pubsub-bigtable-guardium/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/filter-plugin/logstash-filter-pubsub-bigtable-guardium/mainREADME.md b/filter-plugin/logstash-filter-pubsub-bigtable-guardium/mainREADME.md new file mode 100644 index 000000000..45208c4f5 --- /dev/null +++ b/filter-plugin/logstash-filter-pubsub-bigtable-guardium/mainREADME.md @@ -0,0 +1,10 @@ +# BigTable Universal Connector + +## Follow this link to set up and use BigTable Universal Connector over PubSub Logstash Plugin + +[BigTableOverPubSub](./README.md) + +## Follow this link to set up and use BigTable Universal Connector over PubSub Connect + +[BigTableOverConnectPubSub](../../docs/KafkaBasedUCs/BigtablePubsubKafkaConnect.md) + diff --git a/filter-plugin/logstash-filter-pubsub-bigtable-guardium/src/main/java/com/ibm/guardium/bigtable/Parser.java b/filter-plugin/logstash-filter-pubsub-bigtable-guardium/src/main/java/com/ibm/guardium/bigtable/Parser.java index c78801847..7fb0723ee 100644 --- a/filter-plugin/logstash-filter-pubsub-bigtable-guardium/src/main/java/com/ibm/guardium/bigtable/Parser.java +++ b/filter-plugin/logstash-filter-pubsub-bigtable-guardium/src/main/java/com/ibm/guardium/bigtable/Parser.java @@ -8,7 +8,15 @@ import com.google.gson.JsonArray; import com.google.gson.JsonObject; import com.ibm.guardium.bigtable.errorcode.BigTableErrorCodes; -import com.ibm.guardium.universalconnector.commons.structures.*; +import com.ibm.guardium.universalconnector.commons.structures.Accessor; +import com.ibm.guardium.universalconnector.commons.structures.Construct; +import com.ibm.guardium.universalconnector.commons.structures.Data; +import com.ibm.guardium.universalconnector.commons.structures.ExceptionRecord; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import com.ibm.guardium.universalconnector.commons.structures.Sentence; +import com.ibm.guardium.universalconnector.commons.structures.SentenceObject; +import com.ibm.guardium.universalconnector.commons.structures.SessionLocator; +import com.ibm.guardium.universalconnector.commons.structures.Time; import org.apache.commons.lang3.StringUtils; import org.apache.commons.validator.routines.InetAddressValidator; diff --git a/filter-plugin/logstash-filter-pubsub-bigtable-guardium/src/test/java/com/ibm/guardium/bigtable/ParserTest.java b/filter-plugin/logstash-filter-pubsub-bigtable-guardium/src/test/java/com/ibm/guardium/bigtable/ParserTest.java index 9cb9501ee..738c911d3 100644 --- a/filter-plugin/logstash-filter-pubsub-bigtable-guardium/src/test/java/com/ibm/guardium/bigtable/ParserTest.java +++ b/filter-plugin/logstash-filter-pubsub-bigtable-guardium/src/test/java/com/ibm/guardium/bigtable/ParserTest.java @@ -3,7 +3,15 @@ import com.google.gson.Gson; import com.google.gson.JsonObject; import com.ibm.guardium.bigtable.errorcode.BigTableErrorCodes; -import com.ibm.guardium.universalconnector.commons.structures.*; +import com.ibm.guardium.universalconnector.commons.structures.Accessor; +import com.ibm.guardium.universalconnector.commons.structures.Construct; +import com.ibm.guardium.universalconnector.commons.structures.Data; +import com.ibm.guardium.universalconnector.commons.structures.ExceptionRecord; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import com.ibm.guardium.universalconnector.commons.structures.Sentence; +import com.ibm.guardium.universalconnector.commons.structures.SentenceObject; +import com.ibm.guardium.universalconnector.commons.structures.SessionLocator; +import com.ibm.guardium.universalconnector.commons.structures.Time; import org.junit.jupiter.api.Test; import java.io.IOException; diff --git a/filter-plugin/logstash-filter-pubsub-firebase-realtime-guardium/README.md b/filter-plugin/logstash-filter-pubsub-firebase-realtime-guardium/README.md index 766fafbb6..4a8b0d195 100644 --- a/filter-plugin/logstash-filter-pubsub-firebase-realtime-guardium/README.md +++ b/filter-plugin/logstash-filter-pubsub-firebase-realtime-guardium/README.md @@ -167,3 +167,4 @@ The Guardium universal connector is the Guardium entry point for native audit/da - Firebase does not generate logs for GetDatabaseInstance event. - While using Firebase CLI(third-party tool), the GetDatabaseInstance event populates all data operations (GET/PUSH/UPDATE/REMOVE) inside the logs. - The "logging.googleapis.com" service is a general logging service that logs various activities, such as switching between databases or clicking on different tabs. The service is available on the Firebase UI, and when updating JSON files using the UI. Logs for the Firebase UI and the JSON file updates on the UI will generate the same logs with no new information. +- Error queries are not supported in GCP for Firebase. diff --git a/filter-plugin/logstash-filter-pubsub-firebase-realtime-guardium/build.gradle b/filter-plugin/logstash-filter-pubsub-firebase-realtime-guardium/build.gradle index c0e9855a9..248f2e8a5 100644 --- a/filter-plugin/logstash-filter-pubsub-firebase-realtime-guardium/build.gradle +++ b/filter-plugin/logstash-filter-pubsub-firebase-realtime-guardium/build.gradle @@ -2,6 +2,30 @@ import java.nio.file.Files import static java.nio.file.StandardCopyOption.REPLACE_EXISTING apply plugin: 'java' + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + + dependencies { + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' + classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.1.0" + classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' + } +} + +def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" // =========================================================================== @@ -20,10 +44,10 @@ pluginInfo.pluginClass = "FireBaseGuardiumFilter" pluginInfo.pluginName = "fire_base_guardium_filter" // must match the @LogstashPlugin annotation in the main plugin class // =========================================================================== -sourceCompatibility = 1.8 -targetCompatibility = 1.8 +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 -def jacocoVersion = '0.8.4' +def jacocoVersion = '0.8.11' // minimumCoverage can be set by Travis ENV def minimumCoverageStr = System.getenv("MINIMUM_COVERAGE") ?: "50.0%" if (minimumCoverageStr.endsWith("%")) { @@ -31,28 +55,17 @@ if (minimumCoverageStr.endsWith("%")) { } def minimumCoverage = Float.valueOf(minimumCoverageStr) / 100 -buildscript { - repositories { - maven { - url "https://plugins.gradle.org/m2/" - } - mavenCentral() - jcenter() - } - - dependencies { - classpath 'com.github.jengelman.gradle.plugins:shadow:4.0.4' - classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.0.0-RC2" - classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' - } -} - -def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); -def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) repositories { - mavenCentral() + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } } tasks.register("vendor"){ @@ -63,20 +76,20 @@ tasks.register("vendor"){ File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") projectJarFile.mkdirs() Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) - validatePluginJar(projectJarFile, project.group) } } apply plugin: 'com.github.johnrengelman.shadow' shadowJar { - classifier = null + archiveClassifier = null } dependencies { implementation 'com.google.code.gson:gson:' + versions.dependencies.gson implementation 'org.apache.commons:commons-lang3:' + versions.dependencies.commonsLang + implementation group: 'commons-beanutils', name: 'commons-beanutils', version: versions.dependencies.commonsBeanutils testImplementation group: 'org.mockito', name: 'mockito-all', version: versions.dependencies.mockitoAll @@ -109,6 +122,17 @@ tasks.withType(JavaCompile) { options.encoding = 'UTF-8' } +test { + jvmArgs = [ + '--add-opens=java.base/java.lang=ALL-UNNAMED', + '--add-opens=java.base/java.util=ALL-UNNAMED', + '--add-opens=java.base/java.text=ALL-UNNAMED', + '--add-opens=java.base/java.lang.reflect=ALL-UNNAMED', + '--add-opens=java.base/sun.util.resources=ALL-UNNAMED', + '--add-opens=java.base/sun.util.cldr=ALL-UNNAMED' + ] +} + tasks.register("generateRubySupportFiles") { @@ -150,17 +174,16 @@ apply plugin: "org.barfuin.gradle.jacocolog" // ------------------------------------ jacoco { toolVersion = "${jacocoVersion}" - reportsDir = file("$buildDir/reports/jacoco") } jacocoTestReport { // You will see "Report -> file://...." at the end of a JaCoCo build // If no output, run this first: ./gradlew test reports { - html.enabled true - xml.enabled true - csv.enabled true - html.destination file("${buildDir}/reports/jacoco") - csv.destination file("${buildDir}/reports/jacoco/all.csv") + html.required = true + xml.required = true + csv.required = true + html.outputLocation = file("${buildDir}/reports/jacoco") + csv.outputLocation = file("${buildDir}/reports/jacoco/all.csv") } executionData.from fileTree(dir: "${buildDir}/jacoco/", includes: [ '**/*.exec' diff --git a/filter-plugin/logstash-filter-pubsub-firebase-realtime-guardium/gradle/wrapper/gradle-wrapper.properties b/filter-plugin/logstash-filter-pubsub-firebase-realtime-guardium/gradle/wrapper/gradle-wrapper.properties index aa991fcea..81aa1c044 100644 --- a/filter-plugin/logstash-filter-pubsub-firebase-realtime-guardium/gradle/wrapper/gradle-wrapper.properties +++ b/filter-plugin/logstash-filter-pubsub-firebase-realtime-guardium/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/filter-plugin/logstash-filter-pubsub-firestore-guardium/README.md b/filter-plugin/logstash-filter-pubsub-firestore-guardium/README.md index 8afd4c842..336f42bb0 100644 --- a/filter-plugin/logstash-filter-pubsub-firestore-guardium/README.md +++ b/filter-plugin/logstash-filter-pubsub-firestore-guardium/README.md @@ -142,3 +142,4 @@ The Guardium universal connector is the Guardium entry point for native audit/da - Source program : Not available with logs
- OS User : Not available with logs
- Client HostName : Not available with logs +- Error queries are not supported in GCP for Firestore. \ No newline at end of file diff --git a/filter-plugin/logstash-filter-pubsub-firestore-guardium/build.gradle b/filter-plugin/logstash-filter-pubsub-firestore-guardium/build.gradle index 1fcf0a8cb..481e10a84 100644 --- a/filter-plugin/logstash-filter-pubsub-firestore-guardium/build.gradle +++ b/filter-plugin/logstash-filter-pubsub-firestore-guardium/build.gradle @@ -2,6 +2,30 @@ import java.nio.file.Files import static java.nio.file.StandardCopyOption.REPLACE_EXISTING apply plugin: 'java' + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + + dependencies { + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' + classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.1.0" + classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' + } +} + +def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" // =========================================================================== @@ -19,10 +43,10 @@ pluginInfo.pluginType = "filter" pluginInfo.pluginClass = "FireStoreGuardiumFilter" pluginInfo.pluginName = "fire_store_guardium_filter" // must match the @LogstashPlugin annotation in the main plugin class // =========================================================================== -sourceCompatibility = 1.8 -targetCompatibility = 1.8 +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 -def jacocoVersion = '0.8.4' +def jacocoVersion = '0.8.11' // minimumCoverage can be set by Travis ENV def minimumCoverageStr = System.getenv("MINIMUM_COVERAGE") ?: "50.0%" if (minimumCoverageStr.endsWith("%")) { @@ -30,28 +54,17 @@ if (minimumCoverageStr.endsWith("%")) { } def minimumCoverage = Float.valueOf(minimumCoverageStr) / 100 -buildscript { - repositories { - maven { - url "https://plugins.gradle.org/m2/" - } - mavenCentral() - jcenter() - } - - dependencies { - classpath 'com.github.jengelman.gradle.plugins:shadow:4.0.4' - classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.0.0-RC2" - classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' - } -} - -def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); -def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) repositories { - mavenCentral() + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } } tasks.register("vendor"){ @@ -62,19 +75,19 @@ tasks.register("vendor"){ File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") projectJarFile.mkdirs() Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) - validatePluginJar(projectJarFile, project.group) } } apply plugin: 'com.github.johnrengelman.shadow' shadowJar { - classifier = null + archiveClassifier = null } dependencies { implementation 'com.google.code.gson:gson:' + versions.dependencies.gson implementation 'org.apache.commons:commons-lang3:' + versions.dependencies.commonsLang + implementation group: 'commons-beanutils', name: 'commons-beanutils', version: versions.dependencies.commonsBeanutils testImplementation group: 'org.mockito', name: 'mockito-all', version: versions.dependencies.mockitoAll @@ -105,6 +118,17 @@ tasks.withType(JavaCompile) { options.encoding = 'UTF-8' } +test { + jvmArgs = [ + '--add-opens=java.base/java.lang=ALL-UNNAMED', + '--add-opens=java.base/java.util=ALL-UNNAMED', + '--add-opens=java.base/java.text=ALL-UNNAMED', + '--add-opens=java.base/java.lang.reflect=ALL-UNNAMED', + '--add-opens=java.base/sun.util.resources=ALL-UNNAMED', + '--add-opens=java.base/sun.util.cldr=ALL-UNNAMED' + ] +} + tasks.register("generateRubySupportFiles") { @@ -147,17 +171,16 @@ apply plugin: "org.barfuin.gradle.jacocolog" // ------------------------------------ jacoco { toolVersion = "${jacocoVersion}" - reportsDir = file("$buildDir/reports/jacoco") } jacocoTestReport { // You will see "Report -> file://...." at the end of a JaCoCo build // If no output, run this first: ./gradlew test reports { - html.enabled true - xml.enabled true - csv.enabled true - html.destination file("${buildDir}/reports/jacoco") - csv.destination file("${buildDir}/reports/jacoco/all.csv") + html.required = true + xml.required = true + csv.required = true + html.outputLocation = file("${buildDir}/reports/jacoco") + csv.outputLocation = file("${buildDir}/reports/jacoco/all.csv") } executionData.from fileTree(dir: "${buildDir}/jacoco/", includes: [ '**/*.exec' diff --git a/filter-plugin/logstash-filter-pubsub-firestore-guardium/gradle/wrapper/gradle-wrapper.properties b/filter-plugin/logstash-filter-pubsub-firestore-guardium/gradle/wrapper/gradle-wrapper.properties index aa991fcea..81aa1c044 100644 --- a/filter-plugin/logstash-filter-pubsub-firestore-guardium/gradle/wrapper/gradle-wrapper.properties +++ b/filter-plugin/logstash-filter-pubsub-firestore-guardium/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/filter-plugin/logstash-filter-pubsub-mysql-guardium/PubSubMySQLPackage/logstash-filter-pubsub-mysql-guardium-7.16.4.zip b/filter-plugin/logstash-filter-pubsub-mysql-guardium/PubSubMySQLPackage/logstash-filter-pubsub-mysql-guardium-7.16.4.zip index 5395f6d15..1db4d2a6f 100644 Binary files a/filter-plugin/logstash-filter-pubsub-mysql-guardium/PubSubMySQLPackage/logstash-filter-pubsub-mysql-guardium-7.16.4.zip and b/filter-plugin/logstash-filter-pubsub-mysql-guardium/PubSubMySQLPackage/logstash-filter-pubsub-mysql-guardium-7.16.4.zip differ diff --git a/filter-plugin/logstash-filter-pubsub-mysql-guardium/PubSubMySQLPackage/logstash-filter-pubsub-mysql-guardium-8.3.4.zip b/filter-plugin/logstash-filter-pubsub-mysql-guardium/PubSubMySQLPackage/logstash-filter-pubsub-mysql-guardium-8.3.4.zip index 5395f6d15..1db4d2a6f 100644 Binary files a/filter-plugin/logstash-filter-pubsub-mysql-guardium/PubSubMySQLPackage/logstash-filter-pubsub-mysql-guardium-8.3.4.zip and b/filter-plugin/logstash-filter-pubsub-mysql-guardium/PubSubMySQLPackage/logstash-filter-pubsub-mysql-guardium-8.3.4.zip differ diff --git a/filter-plugin/logstash-filter-pubsub-mysql-guardium/lib/logstash/filters/pubsub-mysql-guardium.rb b/filter-plugin/logstash-filter-pubsub-mysql-guardium/lib/logstash/filters/pubsub-mysql-guardium.rb index 3441ceade..951ad1eb6 100644 --- a/filter-plugin/logstash-filter-pubsub-mysql-guardium/lib/logstash/filters/pubsub-mysql-guardium.rb +++ b/filter-plugin/logstash-filter-pubsub-mysql-guardium/lib/logstash/filters/pubsub-mysql-guardium.rb @@ -137,7 +137,7 @@ def filter(event) event.set('[GuardRecord][sessionLocator][clientIpv6]', nil) event.set('[GuardRecord][sessionLocator][serverIpv6]', nil) event.set('[GuardRecord][sessionLocator][clientIp]', '127.0.0.1') - event.set('[GuardRecord][sessionLocator][serverPort]', parse['session_id'].to_s.empty? ? -1 : 3306) + event.set('[GuardRecord][sessionLocator][serverPort]', 3306) event.set('[GuardRecord][sessionLocator][isIpv6]', false) event.set('[GuardRecord][accessor][serverType]', 'MySQL') diff --git a/filter-plugin/logstash-filter-pubsub-postgresql-guardium/CHANGELOG.md b/filter-plugin/logstash-filter-pubsub-postgresql-guardium/CHANGELOG.md index 701260621..8a94e8b26 100644 --- a/filter-plugin/logstash-filter-pubsub-postgresql-guardium/CHANGELOG.md +++ b/filter-plugin/logstash-filter-pubsub-postgresql-guardium/CHANGELOG.md @@ -1,3 +1,5 @@ +## 1.0.5 + - Added new parse message to avoid ruby hash convert issue ## 1.0.4 - Assign default values to the database user and database name. ## 1.0.3 diff --git a/filter-plugin/logstash-filter-pubsub-postgresql-guardium/PubSubPostgreSQLPackage/PubSubPostgreSQL/filter.conf b/filter-plugin/logstash-filter-pubsub-postgresql-guardium/PubSubPostgreSQLPackage/PubSubPostgreSQL/filter.conf index 3ea53d6a2..cce4c641f 100644 --- a/filter-plugin/logstash-filter-pubsub-postgresql-guardium/PubSubPostgreSQLPackage/PubSubPostgreSQL/filter.conf +++ b/filter-plugin/logstash-filter-pubsub-postgresql-guardium/PubSubPostgreSQLPackage/PubSubPostgreSQL/filter.conf @@ -1,5 +1,5 @@ filter { if [type] == "PubSubPostgreSQL" { - pubsub-postgresql-guardium{} + pubsub-postgresql-guardium{} } } \ No newline at end of file diff --git a/filter-plugin/logstash-filter-pubsub-postgresql-guardium/lib/logstash/filters/pubsub-postgresql-guardium.rb b/filter-plugin/logstash-filter-pubsub-postgresql-guardium/lib/logstash/filters/pubsub-postgresql-guardium.rb index 91865bb6b..c289a6dbe 100644 --- a/filter-plugin/logstash-filter-pubsub-postgresql-guardium/lib/logstash/filters/pubsub-postgresql-guardium.rb +++ b/filter-plugin/logstash-filter-pubsub-postgresql-guardium/lib/logstash/filters/pubsub-postgresql-guardium.rb @@ -25,17 +25,16 @@ def register # auxilary functions for parsing def parsePostgresLog(event) begin - parseEvent = event.get('message') - parse = JSON.parse(parseEvent) - textPayload = parse["textPayload"] - @logger.debug("Start parsing postgres log. payload: #{textPayload}") - message = textPayload.match(/(?(\d*-){2}(\d*)\s(\d*:){2}(\d*.\d*))(\s)UTC(\s)\[(?\d*)\].*db=(?\S*),user=(?\S*)\s(?[A-Z]*):(?.*)/) + a = event.get('textPayload') + @logger.debug("Start parsing postgres log. payload: #{a}") + message = event.get('textPayload').match(/(?(\d*-){2}(\d*)\s(\d*:){2}(\d*.\d*))(\s)UTC(\s)\[(?\d*)\].*db=(?\S*),user=(?\S*)\s(?[A-Z]*):(?.*)/) + msg = message['msg'] severity = message['severity'] - db_name = message['db_name'].to_s.empty? ? "NA" : message['db_name'] + db_name = message['db_name'].to_s.empty? ? "N.A." : message['db_name'] session_id = message['session_id'].to_s.empty? ? "" : message['session_id'] - uname = message['uname'].to_s.empty? ? "NA" : message['uname'] + uname = message['uname'].to_s.empty? ? "N.A." : message['uname'] timestamp = message['ts'] event.set('[GuardRecord][dbName]', db_name) @@ -84,17 +83,19 @@ def parsePostgresLog(event) def parsePgAudit(event) begin - parseEvent = event.get('message') - parse = JSON.parse(parseEvent) - protoPayload = parse["protoPayload"] - @logger.debug("Start parsing pg audit log. payload: #{protoPayload}") + + a = event.get('protoPayload') + @logger.debug("Start parsing pg audit log. payload: #{a}") + protoPayload = event.get('protoPayload') + request = protoPayload['request'] original_sql = request['statement'] session_id = request['databaseSessionId'].to_s.empty? ? "" : request['databaseSessionId'] - db_name = request['database'].to_s.empty? ? "NA" : request['database'] - uname = request['user'].to_s.empty? ? "NA" : request['user'] - timestamp = parse["timestamp"] + db_name = request['database'].to_s.empty? ? "N.A." : request['database'] + uname = request['user'].to_s.empty? ? "N.A." : request['user'] + timestamp = event.get('timestamp') + #timestamp = parse["timestamp"] event.set('[GuardRecord][data][originalSqlCommand]', original_sql) event.set('[GuardRecord][data][construct]', nil) @@ -119,25 +120,32 @@ def filter(event) matched = false begin - @logger.debug("Start processing new event: #{event}") - message = event.get('message') - @logger.debug(" Debug Message : #{message}") - parse = JSON.parse(message) - @logger.debug(" Parse message : #{parse}") + pmessage = { + "resource" => event.get("resource"), + "logName" => event.get("logName"), + "severity" => event.get("severity"), + "host" => event.get("host") + } + + @logger.debug("Start processing new event: #{pmessage}") - log_name_for_debug = parse["logName"]; + @logger.debug(">>> message class: #{pmessage.class}") + + @logger.debug(" Debug Message : #{pmessage}") + + log_name_for_debug = pmessage['logName'] @logger.debug("log name: #{log_name_for_debug}") - log_name = log_name_for_debug.match(/.*%2F(?.*)/) + log_name = pmessage['logName'].match(/.*%2F(?.*)/) log_type = log_name['log_type'] @logger.debug("log type: #{log_type}") - - resource = parse["resource"] + resource = pmessage['resource'] labels = resource['labels'] database_id = labels['database_id'] server_hostname = "#{labels["region"]}:#{database_id}" - severity = parse["severity"] - client_hostname = parse["host"] + severity = pmessage['severity'] + client_hostname = pmessage['host'] + @logger.debug('Parsing by log type') case log_type @@ -194,6 +202,7 @@ def filter(event) event.remove('insertId') event.remove('severity') event.remove('labels') + event.remove('pmessage') event.set('GuardRecord', event.get('GuardRecord').to_json) diff --git a/filter-plugin/logstash-filter-pubsub-postgresql-guardium/mainREADME.md b/filter-plugin/logstash-filter-pubsub-postgresql-guardium/mainREADME.md new file mode 100644 index 000000000..d6b8f74ac --- /dev/null +++ b/filter-plugin/logstash-filter-pubsub-postgresql-guardium/mainREADME.md @@ -0,0 +1,10 @@ +# Postgres Universal Connector + +## Follow this link to set up and use Postgres Universal Connector over PubSub Logstash Plugin + +[PostgresOverPubSub](./README.md) + +## Follow this link to set up and use Postgres Universal Connector over PubSub Connect + +[PostgresOverConnectPubSub](../../docs/KafkaBasedUCs/PostgresqlPubsubKafkaConnect.md) + diff --git a/filter-plugin/logstash-filter-pubsub-postgresql-guardium/postgresqlGooglePubsub.conf b/filter-plugin/logstash-filter-pubsub-postgresql-guardium/postgresqlGooglePubsub.conf index eb536e03f..ebefccf41 100644 --- a/filter-plugin/logstash-filter-pubsub-postgresql-guardium/postgresqlGooglePubsub.conf +++ b/filter-plugin/logstash-filter-pubsub-postgresql-guardium/postgresqlGooglePubsub.conf @@ -17,6 +17,9 @@ input { include_metadata => true codec => "json" + + #max_message is an optional parameter. Value of max_message needs to be adjusted according to traffic. + #max_message => 100 # Type should be populated with a unique ID per connector to provide data source isolation # can be any string, preferably a meaningful one diff --git a/filter-plugin/logstash-filter-pubsub-spanner-guardium/build.gradle b/filter-plugin/logstash-filter-pubsub-spanner-guardium/build.gradle index 2728956a4..f8fef0daa 100644 --- a/filter-plugin/logstash-filter-pubsub-spanner-guardium/build.gradle +++ b/filter-plugin/logstash-filter-pubsub-spanner-guardium/build.gradle @@ -2,6 +2,30 @@ import java.nio.file.Files import static java.nio.file.StandardCopyOption.REPLACE_EXISTING apply plugin: 'java' + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + + dependencies { + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' + classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.1.0" + classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' + } +} + +def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" // =========================================================================== @@ -20,10 +44,10 @@ pluginInfo.pluginClass = "SpannerDBGuardiumFilter" pluginInfo.pluginName = "spanner_db_guardium_filter" // must match the @LogstashPlugin annotation in the main plugin class // =========================================================================== -sourceCompatibility = 1.8 -targetCompatibility = 1.8 +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 -def jacocoVersion = '0.8.4' +def jacocoVersion = '0.8.11' // minimumCoverage can be set by Travis ENV def minimumCoverageStr = System.getenv("MINIMUM_COVERAGE") ?: "50.0%" if (minimumCoverageStr.endsWith("%")) { @@ -31,28 +55,17 @@ if (minimumCoverageStr.endsWith("%")) { } def minimumCoverage = Float.valueOf(minimumCoverageStr) / 100 -buildscript { - repositories { - maven { - url "https://plugins.gradle.org/m2/" - } - mavenCentral() - jcenter() - } - - dependencies { - classpath 'com.github.jengelman.gradle.plugins:shadow:4.0.4' - classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.0.0-RC2" - classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' - } -} - -def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); -def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) repositories { - mavenCentral() + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } } tasks.register("vendor"){ @@ -63,20 +76,20 @@ tasks.register("vendor"){ File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") projectJarFile.mkdirs() Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) - validatePluginJar(projectJarFile, project.group) } } apply plugin: 'com.github.johnrengelman.shadow' shadowJar { - classifier = null + archiveClassifier = null } dependencies { implementation 'com.google.code.gson:gson:' + versions.dependencies.gson implementation 'org.apache.commons:commons-lang3:' + versions.dependencies.commonsLang implementation 'commons-validator:commons-validator:' + versions.dependencies.commonsValidator + implementation group: 'commons-beanutils', name: 'commons-beanutils', version: versions.dependencies.commonsBeanutils testImplementation group: 'org.mockito', name: 'mockito-all', version: versions.dependencies.mockitoAll testImplementation 'org.junit.jupiter:junit-jupiter:' + versions.dependencies.junitJupiter @@ -108,6 +121,17 @@ tasks.withType(JavaCompile) { options.encoding = 'UTF-8' } +test { + jvmArgs = [ + '--add-opens=java.base/java.lang=ALL-UNNAMED', + '--add-opens=java.base/java.util=ALL-UNNAMED', + '--add-opens=java.base/java.text=ALL-UNNAMED', + '--add-opens=java.base/java.lang.reflect=ALL-UNNAMED', + '--add-opens=java.base/sun.util.resources=ALL-UNNAMED', + '--add-opens=java.base/sun.util.cldr=ALL-UNNAMED' + ] +} + tasks.register("generateRubySupportFiles") { @@ -150,17 +174,16 @@ apply plugin: "org.barfuin.gradle.jacocolog" // ------------------------------------ jacoco { toolVersion = "${jacocoVersion}" - reportsDir = file("$buildDir/reports/jacoco") } jacocoTestReport { // You will see "Report -> file://...." at the end of a JaCoCo build // If no output, run this first: ./gradlew test reports { - html.enabled true - xml.enabled true - csv.enabled true - html.destination file("${buildDir}/reports/jacoco") - csv.destination file("${buildDir}/reports/jacoco/all.csv") + html.required = true + xml.required = true + csv.required = true + html.outputLocation = file("${buildDir}/reports/jacoco") + csv.outputLocation = file("${buildDir}/reports/jacoco/all.csv") } executionData.from fileTree(dir: "${buildDir}/jacoco/", includes: [ '**/*.exec' diff --git a/filter-plugin/logstash-filter-pubsub-spanner-guardium/gradle/wrapper/gradle-wrapper.properties b/filter-plugin/logstash-filter-pubsub-spanner-guardium/gradle/wrapper/gradle-wrapper.properties index aa991fcea..81aa1c044 100644 --- a/filter-plugin/logstash-filter-pubsub-spanner-guardium/gradle/wrapper/gradle-wrapper.properties +++ b/filter-plugin/logstash-filter-pubsub-spanner-guardium/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/filter-plugin/logstash-filter-pubsub-spanner-guardium/mainREADME.md b/filter-plugin/logstash-filter-pubsub-spanner-guardium/mainREADME.md new file mode 100644 index 000000000..10b6e33e6 --- /dev/null +++ b/filter-plugin/logstash-filter-pubsub-spanner-guardium/mainREADME.md @@ -0,0 +1,10 @@ +# Spanner Universal Connector + +## Follow this link to set up and use Spanner Universal Connector over PubSub Logstash Plugin + +[SpannerOverPubSub](./README.md) + +## Follow this link to set up and use Spanner Universal Connector over PubSub Connect + +[SpannerOverConnectPubSub](../../docs/KafkaBasedUCs/SpannerPubsubKafkaConnect.md) + diff --git a/filter-plugin/logstash-filter-redshift-aws-guardium/build.gradle b/filter-plugin/logstash-filter-redshift-aws-guardium/build.gradle index b38356506..014525d9d 100644 --- a/filter-plugin/logstash-filter-redshift-aws-guardium/build.gradle +++ b/filter-plugin/logstash-filter-redshift-aws-guardium/build.gradle @@ -2,6 +2,30 @@ import java.nio.file.Files import static java.nio.file.StandardCopyOption.REPLACE_EXISTING apply plugin: 'java' + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + + dependencies { + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' + classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.1.0" + classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' + } +} + +def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" // =========================================================================== @@ -20,10 +44,10 @@ pluginInfo.pluginClass = "RedShiftGuardiumConnector" pluginInfo.pluginName = "redshift_guardium_connector" // must match the @LogstashPlugin annotation in the main plugin class // =========================================================================== -sourceCompatibility = 1.8 -targetCompatibility = 1.8 +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 -def jacocoVersion = '0.8.4' +def jacocoVersion = '0.8.11' // minimumCoverage can be set by Travis ENV def minimumCoverageStr = System.getenv("MINIMUM_COVERAGE") ?: "50.0%" if (minimumCoverageStr.endsWith("%")) { @@ -31,27 +55,16 @@ if (minimumCoverageStr.endsWith("%")) { } def minimumCoverage = Float.valueOf(minimumCoverageStr) / 100 -buildscript { - repositories { - maven { - url "https://plugins.gradle.org/m2/" - } - mavenCentral() - jcenter() - } - - dependencies { - classpath 'com.github.jengelman.gradle.plugins:shadow:4.0.4' - classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.0.0-RC2" - classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' - } -} - -def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); -def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) repositories { - mavenCentral() + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } } tasks.register("vendor"){ @@ -62,14 +75,13 @@ tasks.register("vendor"){ File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") projectJarFile.mkdirs() Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) - validatePluginJar(projectJarFile, project.group) } } apply plugin: 'com.github.johnrengelman.shadow' shadowJar { - classifier = null + archiveClassifier = null } dependencies { @@ -77,6 +89,7 @@ dependencies { implementation 'org.apache.commons:commons-lang3:' + versions.dependencies.commonsLang implementation 'com.google.code.gson:gson:' + versions.dependencies.gson implementation group: 'org.json', name: 'json', version: versions.dependencies.json + implementation group: 'commons-beanutils', name: 'commons-beanutils', version: versions.dependencies.commonsBeanutils implementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "common-*.*.*.jar") implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "build/libs/logstash-core-*.*.*.jar") @@ -102,6 +115,17 @@ tasks.withType(JavaCompile) { options.encoding = 'UTF-8' } +test { + jvmArgs = [ + '--add-opens=java.base/java.lang=ALL-UNNAMED', + '--add-opens=java.base/java.util=ALL-UNNAMED', + '--add-opens=java.base/java.text=ALL-UNNAMED', + '--add-opens=java.base/java.lang.reflect=ALL-UNNAMED', + '--add-opens=java.base/sun.util.resources=ALL-UNNAMED', + '--add-opens=java.base/sun.util.cldr=ALL-UNNAMED' + ] +} + tasks.register("generateRubySupportFiles") { @@ -144,17 +168,16 @@ apply plugin: "org.barfuin.gradle.jacocolog" // ------------------------------------ jacoco { toolVersion = "${jacocoVersion}" - reportsDir = file("$buildDir/reports/jacoco") } jacocoTestReport { // You will see "Report -> file://...." at the end of a JaCoCo build // If no output, run this first: ./gradlew test reports { - html.enabled true - xml.enabled true - csv.enabled true - html.destination file("${buildDir}/reports/jacoco") - csv.destination file("${buildDir}/reports/jacoco/all.csv") + html.required = true + xml.required = true + csv.required = true + html.outputLocation = file("${buildDir}/reports/jacoco") + csv.outputLocation = file("${buildDir}/reports/jacoco/all.csv") } executionData.from fileTree(dir: "${buildDir}/jacoco/", includes: [ '**/*.exec' diff --git a/filter-plugin/logstash-filter-redshift-aws-guardium/gradle/wrapper/gradle-wrapper.properties b/filter-plugin/logstash-filter-redshift-aws-guardium/gradle/wrapper/gradle-wrapper.properties index aa991fcea..81aa1c044 100644 --- a/filter-plugin/logstash-filter-redshift-aws-guardium/gradle/wrapper/gradle-wrapper.properties +++ b/filter-plugin/logstash-filter-redshift-aws-guardium/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/filter-plugin/logstash-filter-redshift-aws-guardium/logstash-filter-redshift_guardium_connector.zip b/filter-plugin/logstash-filter-redshift-aws-guardium/logstash-filter-redshift_guardium_connector.zip new file mode 100644 index 000000000..ab1a0ed8a Binary files /dev/null and b/filter-plugin/logstash-filter-redshift-aws-guardium/logstash-filter-redshift_guardium_connector.zip differ diff --git a/filter-plugin/logstash-filter-redshift-aws-guardium/mainREADME.md b/filter-plugin/logstash-filter-redshift-aws-guardium/mainREADME.md new file mode 100644 index 000000000..4325fca3a --- /dev/null +++ b/filter-plugin/logstash-filter-redshift-aws-guardium/mainREADME.md @@ -0,0 +1,10 @@ +# Redshift Universal Connector + +## Follow this link to set up and use Redshift Universal Connector over CloudWatch Logstash Plugin + +[RedshiftOverCloudwatch](./README.md) + +## Follow this link to set up and use Redshift Universal Connector over CloudWatch Connect + +[RedshiftOverConnectCloudwatch](../../docs/KafkaBasedUCs/RedshiftCloudwatchKafkaConnect.md) + diff --git a/filter-plugin/logstash-filter-redshift-aws-guardium/src/main/java/com/ibm/guardium/redshift/Parser.java b/filter-plugin/logstash-filter-redshift-aws-guardium/src/main/java/com/ibm/guardium/redshift/Parser.java index 681363ced..910bf7857 100644 --- a/filter-plugin/logstash-filter-redshift-aws-guardium/src/main/java/com/ibm/guardium/redshift/Parser.java +++ b/filter-plugin/logstash-filter-redshift-aws-guardium/src/main/java/com/ibm/guardium/redshift/Parser.java @@ -19,6 +19,7 @@ import java.util.ArrayList; import java.util.Date; import java.util.List; +import java.util.Locale; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -27,32 +28,32 @@ /** * Parser class is responsible to parse data of events object and set to guard * object. - * + * * @author Ankita Pawar */ public class Parser { - private static final DateTimeFormatterBuilder dateTimeFormatterBuilder = new DateTimeFormatterBuilder() + private final DateTimeFormatterBuilder dateTimeFormatterBuilder = new DateTimeFormatterBuilder() .append(DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSS[[XXX][X]]")); - private static final DateTimeFormatter DATE_TIME_FORMATTER = dateTimeFormatterBuilder.toFormatter(); - private static Logger log = LogManager.getLogger(Parser.class); + private final DateTimeFormatter DATE_TIME_FORMATTER = dateTimeFormatterBuilder.toFormatter(); + private Logger log = LogManager.getLogger(Parser.class); final static String regex = "(\\w+\\s+(?i)External\\s+(?i)table)\\s+['\\\"\\`]?([a-zA-Z_.]+)*|((?i)show\\s+(?i)table)\\s+['\\\"\\`]?([a-zA-Z0-9_.]+)*|((?i)alter\\s+(?i)table)\\s+['\\\"\\`]?([a-zA-Z0-9_.]+)*|((?i)create\\s+(?i)table)\\s+['\\\"\\`]?([a-zA-Z0-9_]+)*|(\\w+\\s+(?i)PROCEDURE)\\s+['\\\"\\`]?([a-zA-Z0-9_.]+)*|(\\w+\\s+(?i)DATASHARE)\\s+['\\\"\\`]?(\\w+)*|(\\w+\\s+(?i)DATASHARES)\\s+['\\\"\\`]?(?i)like\\s+'([a-zA-Z_%-0-9]+)'*|(\\w+\\s+(?i)DATASHARES)|(\\w+\\s+(?i)library)\\s+['\\\"\\`]?([a-zA-Z0-9_.]+)*|(\\w+\\s+(?i)model)\\s+['\\\"\\`]?(\\w+)*|(\\w+\\s+(?i)Identity\\s+(?i)provider)\\s+['\\\"\\`]?([a-zA-Z0-9_.]+)*|(\\w+\\s+(?i)EXTERNAL\\s+(?i)FUNCTION)\\s+['\\\"\\`]?([a-zA-Z0-9_.]+)*|(\\w+\\s+(?i)EXTERNAL\\s+(?i)SCHEMA)\\s+['\\\"\\`]?([a-zA-Z0-9_.]+)*|(\\w+\\s+(?i)view)\\s+['\\\"\\`]?([a-zA-Z0-9_.]+)*|(\\w+\\s+(?i)compression)\\s+['\\\"\\`]?([a-zA-Z0-9_.]+)*|((?i)copy)\\s+['\\\"\\`]?([a-zA-Z0-9_.]+)*|((?i)vacuum)\\s+((?i)sort|(?i)delete)\\s+(?i)only\\s+['\\\"\\`]?(\\w+)|((?i)vacuum)\\s+(?i)reindex\\s+['\\\"\\`]?(\\w+)|((?i)vacuum)\\s*['\\\"\\`]?(\\w+)*|((?i)cancel)|((?i)unload)\\s+\\('([^']+)'\\)\\s+to\\s+'([^']+)'|(\\w+\\s+(?i)database)\\s+['\\\"\\`]?(\\w+)*|(\\w+\\s+(?i)materialized\\s+(?i)view)\\s+((?i)if\\s+(?i)exists\\s+)?['\\\"\\`]?(\\w+)*|(\\w+\\s+(?i)role)\\s+['\\\"\\`]?(\\w+)*|((?i)create\\s+(?i)or\\s+(?i)replace\\s+(?i)function)\\s+['\\\"\\`]?(\\w+)*|(\\w+\\s+(?i)function)\\s+['\\\"\\`]?(\\w+)*"; final static Pattern pattern = Pattern.compile(regex, Pattern.MULTILINE); - static List lst = new ArrayList<>(); + List lst = new ArrayList<>(); /** - * + * * @param event * @return record */ - public static Record parseRecord(final Event event) throws ParseException { + public Record parseRecord(final Event event) throws ParseException { Record record = new Record(); - String sessionId = RedShiftTags.NA_STRING; String dbName = RedShiftTags.UNKNOWN_STRING; record.setSessionId( - event.getField(RedShiftTags.ID) != null ? event.getField(RedShiftTags.ID).toString() : sessionId); + event.getField(RedShiftTags.ID) != null ? event.getField(RedShiftTags.ID).toString() : RedShiftTags.NA_STRING); + // Set dbName if present if (event.getField(RedShiftTags.DB) != null && event.getField(RedShiftTags.DBPREFIX) != null) { dbName = event.getField(RedShiftTags.DBPREFIX).toString() + ":" @@ -92,11 +93,11 @@ public static Record parseRecord(final Event event) throws ParseException { /** * Populating SessionLocator from event. - * + * * @param event * @return SessionLocator */ - public static SessionLocator parseSessionLocator(final Event event) { + public SessionLocator parseSessionLocator(final Event event) { SessionLocator sessionLocator = new SessionLocator(); sessionLocator.setIpv6(false); if (event.getField(RedShiftTags.REMOTEHOST) != null && event.getField(RedShiftTags.REMOTEPORT) != null) { @@ -124,11 +125,11 @@ public static SessionLocator parseSessionLocator(final Event event) { /** * Populating Accessor from event. - * + * * @param event * @return Accessor */ - public static Accessor parseAccessor(final Event event, final Record record) { + public Accessor parseAccessor(final Event event, final Record record) { Accessor accessor = new Accessor(); if (event.getField(RedShiftTags.U_IDENTIFIER) != null) { accessor.setDbUser(event.getField(RedShiftTags.U_IDENTIFIER).toString()); @@ -168,6 +169,8 @@ public static Accessor parseAccessor(final Event event, final Record record) { lst.add(matcher.group(i)); } } + } + if(!lst.isEmpty()) { accessor.setLanguage(Accessor.LANGUAGE_FREE_TEXT_STRING); accessor.setDataType(Accessor.DATA_TYPE_GUARDIUM_SHOULD_NOT_PARSE_SQL); } else { @@ -180,11 +183,11 @@ public static Accessor parseAccessor(final Event event, final Record record) { /** * Populating data object. - * + * * @param event * @return Data */ - public static Data parseData(final Event event) { + public Data parseData(final Event event) { Data data = new Data(); data.setOriginalSqlCommand(regexCustomReplace(event)); return data; @@ -192,17 +195,17 @@ public static Data parseData(final Event event) { /** * Populating data object. - * + * * @param event * @return Data */ - private static Data parseDataRegex(final Event event) { + private Data parseDataRegex(final Event event) { Data data = new Data(); data.setConstruct(parseAsConstruct(event)); return data; } - private static Construct parseAsConstruct(final Event event) { + private Construct parseAsConstruct(final Event event) { final Construct construct = new Construct(); final Sentence sentence = parseSentence(event); // String test=regexCustomReplace(event); @@ -214,7 +217,7 @@ private static Construct parseAsConstruct(final Event event) { } - private static Sentence parseSentence(final Event event) { + private Sentence parseSentence(final Event event) { Sentence sentence = null; sentence = new Sentence(lst.get(0)); if (lst.size() == 1) { @@ -231,11 +234,11 @@ private static Sentence parseSentence(final Event event) { * Using this to perform operation on input, convert String core into * sentenceObject Object and then return the value as response * - * @param String core + * @param message * @return sentenceobject * */ - private static SentenceObject parseSentenceObject(String message) { + private SentenceObject parseSentenceObject(String message) { SentenceObject sentenceObject = null; sentenceObject = new SentenceObject(message); sentenceObject.setName(message); @@ -243,7 +246,7 @@ private static SentenceObject parseSentenceObject(String message) { return sentenceObject; } - private static String regexCustomReplace(final Event event) { + private String regexCustomReplace(final Event event) { String query = RedShiftTags.UNKNOWN_STRING; if (event.getField(RedShiftTags.SQLQUERY) != null) { query = event.getField(RedShiftTags.SQLQUERY).toString(); @@ -256,35 +259,39 @@ private static String regexCustomReplace(final Event event) { } /** - * + * * @param event * @return Time */ - public static Time getConnTime(final Event event) { - String day = event.getField("day").toString(); - String month = event.getField("month").toString(); - String year = event.getField("year").toString(); - String md = event.getField("md").toString(); - String rdtime = event.getField("time").toString(); - String concatStr = day + ", " + md + " " + month + " " + year + " " + rdtime; - Date date = null; - SimpleDateFormat formatter = new SimpleDateFormat("E, dd MMM yyyy HH:mm:ss:SSS"); + public Time getConnTime(final Event event) { try { - date = formatter.parse(concatStr); + String day = event.getField("day").toString(); + String md = event.getField("md").toString(); + String month = event.getField("month").toString(); + String year = event.getField("year").toString(); + String rdtime = event.getField("time").toString(); + + String concatStr = day + ", " + md + " " + month + " " + year + " " + rdtime; + + SimpleDateFormat formatter = + new SimpleDateFormat("E, dd MMM yyyy HH:mm:ss:SSS", Locale.ENGLISH); + Date date = formatter.parse(concatStr); + long millis = date.getTime(); + + return new Time(millis, date.getTimezoneOffset(), 0); } catch (Exception ex) { - ex.printStackTrace(); + log.error("An error occurred during parsing the time for event: {}", event, ex); + return new Time(0, 0, 0); } - long millis = date.getTime(); - return new Time(millis, date.getTimezoneOffset(), 0); } /** - * + * * @param event * @return Time */ - public static Time getTime(Event event) { + public Time getTime(Event event) { String dateString = event.getField(RedShiftTags.TIMESTAMP).toString(); ZonedDateTime date = ZonedDateTime.parse(dateString); long millis = date.toInstant().toEpochMilli(); @@ -293,11 +300,11 @@ public static Time getTime(Event event) { } /** - * + * * @param event * @return Exception */ - public static ExceptionRecord parseException(final Event event) { + public ExceptionRecord parseException(final Event event) { ExceptionRecord exceptionRecord = new ExceptionRecord(); exceptionRecord.setExceptionTypeId(RedShiftTags.EXCEPTION_TYPE_AUTHENTICATION_STRING); exceptionRecord.setDescription("LOGIN_FAILED"); @@ -305,7 +312,7 @@ public static ExceptionRecord parseException(final Event event) { return exceptionRecord; } - public static boolean isIpv6(final String address) { + public boolean isIpv6(final String address) { return address.contains(":"); } diff --git a/filter-plugin/logstash-filter-redshift-aws-guardium/src/main/java/com/ibm/guardium/redshift/RedShiftGuardiumConnector.java b/filter-plugin/logstash-filter-redshift-aws-guardium/src/main/java/com/ibm/guardium/redshift/RedShiftGuardiumConnector.java index 7b776ca1d..aa5378667 100644 --- a/filter-plugin/logstash-filter-redshift-aws-guardium/src/main/java/com/ibm/guardium/redshift/RedShiftGuardiumConnector.java +++ b/filter-plugin/logstash-filter-redshift-aws-guardium/src/main/java/com/ibm/guardium/redshift/RedShiftGuardiumConnector.java @@ -44,24 +44,29 @@ public String getSourceField() { /** * This filter method invoked by the Logstash execution engine. - * + * * @param events , matchListener * @return events */ @Override public Collection filter(Collection events, FilterMatchListener matchListener) { ArrayList skippedEvents = new ArrayList<>(); + Parser parser = new Parser(); for (Event e : events) { + if (log.isDebugEnabled()) { + log.debug("Event Now: {}", e.getData()); + } if ((e.getField(RedShiftTags.SQLQUERY) != null && e.getField(RedShiftTags.U_IDENTIFIER) != null && e.getField(RedShiftTags.SQLQUERY).toString() instanceof String && e.getField(RedShiftTags.SQLQUERY).toString().contains("LOG") && !(e.getField(RedShiftTags.U_IDENTIFIER).toString().equalsIgnoreCase("RDSDB"))) || (e.getField(RedShiftTags.USER_NAME) != null - && !e.getField(RedShiftTags.USER_NAME).toString().equalsIgnoreCase("RDSDB") - && e.getField(RedShiftTags.STATUS).toString().equalsIgnoreCase("authentication failure"))) { + && !e.getField(RedShiftTags.USER_NAME).toString().equalsIgnoreCase("RDSDB") + && e.getField(RedShiftTags.STATUS).toString().equalsIgnoreCase("authentication failure"))) { try { - Record record = Parser.parseRecord(e); + + Record record = parser.parseRecord(e); final GsonBuilder builder = new GsonBuilder(); builder.serializeNulls(); final Gson gson = builder.create(); diff --git a/filter-plugin/logstash-filter-redshift-aws-guardium/src/test/java/com/ibm/guardium/redshift/ParserTest.java b/filter-plugin/logstash-filter-redshift-aws-guardium/src/test/java/com/ibm/guardium/redshift/ParserTest.java index 2d434f7bf..038f99f63 100644 --- a/filter-plugin/logstash-filter-redshift-aws-guardium/src/test/java/com/ibm/guardium/redshift/ParserTest.java +++ b/filter-plugin/logstash-filter-redshift-aws-guardium/src/test/java/com/ibm/guardium/redshift/ParserTest.java @@ -17,6 +17,7 @@ import co.elastic.logstash.api.Event; class ParserTest { + Parser parser = new Parser(); @Test public void testparseAccessor() { Record record = new Record(); @@ -24,7 +25,7 @@ public void testparseAccessor() { Event e = new org.logstash.Event(); e.setField("username", "awsuser"); e.setField("os_version", "Linux 4.14.262-200.489.amzn2.x86_64 amd64"); - final Accessor accessor = Parser.parseAccessor(e, record); + final Accessor accessor = parser.parseAccessor(e, record); assertNotNull(accessor); assertEquals("awsuser", accessor.getDbUser()); assertEquals("Linux 4.14.262-200.489.amzn2.x86_64 amd64", accessor.getOsUser()); @@ -35,7 +36,7 @@ public void testparsesessionLocator1() { Event e = new org.logstash.Event(); e.setField(RedShiftTags.REMOTEHOST, "223.233.72.13"); e.setField(RedShiftTags.REMOTEPORT, "31370"); - final SessionLocator sessionLocator = Parser.parseSessionLocator(e); + final SessionLocator sessionLocator = parser.parseSessionLocator(e); assertNotNull(sessionLocator); assertEquals("223.233.72.13", sessionLocator.getClientIp()); assertEquals(31370, sessionLocator.getClientPort()); @@ -46,7 +47,7 @@ public void testparsesessionLocator2() { Event e = new org.logstash.Event(); e.setField("remotehost", "::1"); e.setField("remoteport", "31370"); - final SessionLocator sessionLocator = Parser.parseSessionLocator(e); + final SessionLocator sessionLocator = parser.parseSessionLocator(e); assertNotNull(sessionLocator); assertEquals("::1", sessionLocator.getClientIpv6()); assertEquals(31370, sessionLocator.getClientPort()); @@ -56,7 +57,7 @@ public void testparsesessionLocator2() { public void testParseData() { Event e = new org.logstash.Event(); e.setField(RedShiftTags.SQLQUERY, "SELECT d.datname as Name"); - final Data data = Parser.parseData(e); + final Data data = parser.parseData(e); assertNotNull(data); assertEquals("SELECT d.datname as Name", data.getOriginalSqlCommand()); } @@ -69,16 +70,16 @@ public void testparsegetConnTime() { e.setField("md", "16"); e.setField("year", "2022"); e.setField("time", "04:58:51:641"); - final Time contime = Parser.getConnTime(e); + final Time contime = parser.getConnTime(e); assertNotNull(contime); - assertEquals(1647386931641L, contime.getTimstamp()); + assertEquals(1647421131641L, contime.getTimstamp()); } @Test public void testparsegetTime() { Event e = new org.logstash.Event(); e.setField("timestamp", "2022-03-17T07:34:29.118Z"); - final Time time = Parser.getTime(e); + final Time time = parser.getTime(e); assertNotNull(time); assertEquals(1647502469118L, time.getTimstamp()); } @@ -87,7 +88,7 @@ public void testparsegetTime() { public void testparseException() { Event e = new org.logstash.Event(); e.setField("action", "authentication failure"); - final ExceptionRecord exceptionRecord = Parser.parseException(e); + final ExceptionRecord exceptionRecord = parser.parseException(e); assertNotNull(exceptionRecord); assertEquals("LOGIN_FAILED", exceptionRecord.getExceptionTypeId()); assertEquals("LOGIN_FAILED", exceptionRecord.getDescription()); @@ -107,12 +108,12 @@ public void testparseRecord_ConnectionLog() throws ParseException { e.setField("year", "2022"); e.setField("time", "04:58:51:641"); e.setField("action", "authentication failure"); - final Record record = Parser.parseRecord(e); + final Record record = parser.parseRecord(e); assertNotNull(record); assertEquals("796", record.getSessionId()); assertEquals("979326520502_guardiumredshift:dev", record.getDbName()); assertEquals("awsuser", record.getAppUserName()); - assertEquals(1647386931641L, record.getTime().getTimstamp()); + assertEquals(1647421131641L, record.getTime().getTimstamp()); assertEquals("LOGIN_FAILED", record.getException().getExceptionTypeId()); assertEquals("LOGIN_FAILED", record.getException().getDescription()); @@ -127,7 +128,7 @@ public void testparseRecord_UserActivityLog() throws ParseException { e.setField("user", "awsuser"); e.setField("timestamp", "2022-03-17T07:34:29.118Z"); e.setField("sql_query", "SELECT d.datname as Name"); - final Record record = Parser.parseRecord(e); + final Record record = parser.parseRecord(e); assertNotNull(record); assertEquals("1073979586", record.getSessionId()); assertEquals("979326520502_guardiumredshift:dev", record.getDbName()); @@ -142,7 +143,7 @@ public void testparseRecord_NA() throws ParseException { e.setField("pid", "1073979586"); e.setField("timestamp", "2022-03-17T07:34:29.118Z"); e.setField("sql_query", "SELECT d.datname as Name"); - final Record record = Parser.parseRecord(e); + final Record record = parser.parseRecord(e); assertNotNull(record); assertEquals("1073979586", record.getSessionId()); assertEquals(1647502469118L, record.getTime().getTimstamp()); @@ -154,10 +155,10 @@ public void testIsipv6() throws ParseException { Event e = new org.logstash.Event(); e.setField("remotehost", "::ffff:136.226.255.29"); e.setField("remoteport", "31370"); - final SessionLocator sessionLocator = Parser.parseSessionLocator(e); + final SessionLocator sessionLocator = parser.parseSessionLocator(e); assertNotNull(sessionLocator); assertEquals("::ffff:136.226.255.29", sessionLocator.getClientIpv6()); assertEquals(31370, sessionLocator.getClientPort()); } -} +} \ No newline at end of file diff --git a/filter-plugin/logstash-filter-redshift-aws-guardium/src/test/java/com/ibm/guardium/redshift/RedShiftGuardiumConnectorTest.java b/filter-plugin/logstash-filter-redshift-aws-guardium/src/test/java/com/ibm/guardium/redshift/RedShiftGuardiumConnectorTest.java index db1ede4ae..8e8e45b24 100644 --- a/filter-plugin/logstash-filter-redshift-aws-guardium/src/test/java/com/ibm/guardium/redshift/RedShiftGuardiumConnectorTest.java +++ b/filter-plugin/logstash-filter-redshift-aws-guardium/src/test/java/com/ibm/guardium/redshift/RedShiftGuardiumConnectorTest.java @@ -6,6 +6,7 @@ import java.util.List; import java.util.concurrent.atomic.AtomicInteger; +import com.ibm.guardium.universalconnector.commons.structures.Record; import org.junit.Assert; import org.junit.jupiter.api.Test; import org.logstash.plugins.ContextImpl; @@ -17,6 +18,8 @@ import co.elastic.logstash.api.FilterMatchListener; import co.elastic.logstash.api.PluginConfigSpec; +import static org.junit.jupiter.api.Assertions.assertEquals; + public class RedShiftGuardiumConnectorTest { final static Context context = new ContextImpl(null, null); @@ -43,184 +46,184 @@ public void testgetSourceField() { } @Test public void testparseDDLRecord() { - Context context = new ContextImpl(null, null); - RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); - Event e = new org.logstash.Event(); - TestMatchListener matchListener = new TestMatchListener(); - e.setField("pid", "796"); - e.setField("db", "dev"); - e.setField("user", "awsuser"); - e.setField("timestamp", "2021-08-23T16:28:14Z"); - e.setField("dbprefix", "979326520502_guardiumredshift"); - e.setField("serverHostnamePrefix", "979326520502_guardiumredshift"); - e.setField("sql_query", - "LOG: ALTER DATASHARE salesshare ADD TABLE public.tickit_sales_redshift;"); - Collection results = filter.filter(Collections.singletonList(e), matchListener); - Assert.assertEquals(1, results.size()); - Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); - Assert.assertEquals(1, matchListener.getMatchCount()); + Context context = new ContextImpl(null, null); + RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); + Event e = new org.logstash.Event(); + TestMatchListener matchListener = new TestMatchListener(); + e.setField("pid", "796"); + e.setField("db", "dev"); + e.setField("user", "awsuser"); + e.setField("timestamp", "2021-08-23T16:28:14Z"); + e.setField("dbprefix", "979326520502_guardiumredshift"); + e.setField("serverHostnamePrefix", "979326520502_guardiumredshift"); + e.setField("sql_query", + "LOG: ALTER DATASHARE salesshare ADD TABLE public.tickit_sales_redshift;"); + Collection results = filter.filter(Collections.singletonList(e), matchListener); + Assert.assertEquals(1, results.size()); + Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); + Assert.assertEquals(1, matchListener.getMatchCount()); } @Test public void testparselibrary() { - Context context = new ContextImpl(null, null); - RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); - Event e = new org.logstash.Event(); - TestMatchListener matchListener = new TestMatchListener(); - e.setField("pid", "796"); - e.setField("db", "dev"); - e.setField("user", "awsuser"); - e.setField("timestamp", "2021-08-23T16:28:14Z"); - e.setField("dbprefix", "979326520502_guardiumredshift"); - e.setField("serverHostnamePrefix", "979326520502_guardiumredshift"); - e.setField("sql_query", - "LOG: create library f_urlparse1\r\n" - + "language plpythonu\r\n" - + "from 's3://guardiumredshift2/geomatry.zip'\r\n" - + "credentials 'aws_iam_role=arn:aws:iam::979326520502:role/myspectrum_role'\r\n" - + "region as 'us-east-1';"); - Collection results = filter.filter(Collections.singletonList(e), matchListener); - Assert.assertEquals(1, results.size()); - Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); - Assert.assertEquals(1, matchListener.getMatchCount()); - } - + Context context = new ContextImpl(null, null); + RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); + Event e = new org.logstash.Event(); + TestMatchListener matchListener = new TestMatchListener(); + e.setField("pid", "796"); + e.setField("db", "dev"); + e.setField("user", "awsuser"); + e.setField("timestamp", "2021-08-23T16:28:14Z"); + e.setField("dbprefix", "979326520502_guardiumredshift"); + e.setField("serverHostnamePrefix", "979326520502_guardiumredshift"); + e.setField("sql_query", + "LOG: create library f_urlparse1\r\n" + + "language plpythonu\r\n" + + "from 's3://guardiumredshift2/geomatry.zip'\r\n" + + "credentials 'aws_iam_role=arn:aws:iam::979326520502:role/myspectrum_role'\r\n" + + "region as 'us-east-1';"); + Collection results = filter.filter(Collections.singletonList(e), matchListener); + Assert.assertEquals(1, results.size()); + Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); + Assert.assertEquals(1, matchListener.getMatchCount()); + } + @Test public void testparseCreate() { - Context context = new ContextImpl(null, null); - RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); - Event e = new org.logstash.Event(); - TestMatchListener matchListener = new TestMatchListener(); - e.setField("pid", "796"); - e.setField("db", "dev"); - e.setField("user", "awsuser"); - e.setField("timestamp", "2021-08-23T16:28:14Z"); - e.setField("dbprefix", "979326520502_guardiumredshift"); - e.setField("serverHostnamePrefix", "979326520502_guardiumredshift"); - e.setField("sql_query", - "LOG: create external table spectrum_schema.spectrum_table(c1 int) stored as parque location 's3://guardiumredshift/myfolder/';"); - Collection results = filter.filter(Collections.singletonList(e), matchListener); - Assert.assertEquals(1, results.size()); - Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); - Assert.assertEquals(1, matchListener.getMatchCount()); + Context context = new ContextImpl(null, null); + RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); + Event e = new org.logstash.Event(); + TestMatchListener matchListener = new TestMatchListener(); + e.setField("pid", "796"); + e.setField("db", "dev"); + e.setField("user", "awsuser"); + e.setField("timestamp", "2021-08-23T16:28:14Z"); + e.setField("dbprefix", "979326520502_guardiumredshift"); + e.setField("serverHostnamePrefix", "979326520502_guardiumredshift"); + e.setField("sql_query", + "LOG: create external table spectrum_schema.spectrum_table(c1 int) stored as parque location 's3://guardiumredshift/myfolder/';"); + Collection results = filter.filter(Collections.singletonList(e), matchListener); + Assert.assertEquals(1, results.size()); + Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); + Assert.assertEquals(1, matchListener.getMatchCount()); } @Test public void testparseMINUS() { - Context context = new ContextImpl(null, null); - RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); - Event e = new org.logstash.Event(); - TestMatchListener matchListener = new TestMatchListener(); - e.setField("pid", "796"); - e.setField("db", "dev"); - e.setField("user", "awsuser"); - e.setField("timestamp", "2021-08-23T16:28:14Z"); - e.setField("dbprefix", "979326520502_guardiumredshift"); - e.setField("serverHostnamePrefix", "979326520502_guardiumredshift"); - e.setField("sql_query", - "LOG: select * from sales1\r\n " - + "MINUs\r\n " - + "select * from sales2;\r\n"); - Collection results = filter.filter(Collections.singletonList(e), matchListener); - Assert.assertEquals(1, results.size()); - Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); - Assert.assertEquals(1, matchListener.getMatchCount()); + Context context = new ContextImpl(null, null); + RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); + Event e = new org.logstash.Event(); + TestMatchListener matchListener = new TestMatchListener(); + e.setField("pid", "796"); + e.setField("db", "dev"); + e.setField("user", "awsuser"); + e.setField("timestamp", "2021-08-23T16:28:14Z"); + e.setField("dbprefix", "979326520502_guardiumredshift"); + e.setField("serverHostnamePrefix", "979326520502_guardiumredshift"); + e.setField("sql_query", + "LOG: select * from sales1\r\n " + + "MINUs\r\n " + + "select * from sales2;\r\n"); + Collection results = filter.filter(Collections.singletonList(e), matchListener); + Assert.assertEquals(1, results.size()); + Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); + Assert.assertEquals(1, matchListener.getMatchCount()); } @Test public void testUnload() { - Context context = new ContextImpl(null, null); - RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); - Event e = new org.logstash.Event(); - TestMatchListener matchListener = new TestMatchListener(); - e.setField("pid", "796"); - e.setField("db", "dev"); - e.setField("user", "awsuser"); - e.setField("timestamp", "2021-08-23T16:28:14Z"); - e.setField("dbprefix", "979326520502_guardiumredshift"); - e.setField("serverHostnamePrefix", "979326520502_guardiumredshift"); - e.setField("sql_query", - "LOG: unload ('select * from venue') to 's3://mybucket/unload/' iam_role 'arn:aws:iam::0123456789012:role/MyRedshiftRole';"); - Collection results = filter.filter(Collections.singletonList(e), matchListener); - Assert.assertEquals(1, results.size()); - Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); - Assert.assertEquals(1, matchListener.getMatchCount()); + Context context = new ContextImpl(null, null); + RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); + Event e = new org.logstash.Event(); + TestMatchListener matchListener = new TestMatchListener(); + e.setField("pid", "796"); + e.setField("db", "dev"); + e.setField("user", "awsuser"); + e.setField("timestamp", "2021-08-23T16:28:14Z"); + e.setField("dbprefix", "979326520502_guardiumredshift"); + e.setField("serverHostnamePrefix", "979326520502_guardiumredshift"); + e.setField("sql_query", + "LOG: unload ('select * from venue') to 's3://mybucket/unload/' iam_role 'arn:aws:iam::0123456789012:role/MyRedshiftRole';"); + Collection results = filter.filter(Collections.singletonList(e), matchListener); + Assert.assertEquals(1, results.size()); + Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); + Assert.assertEquals(1, matchListener.getMatchCount()); } @Test public void testparseVacuum() { - Context context = new ContextImpl(null, null); - RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); - Event e = new org.logstash.Event(); - TestMatchListener matchListener = new TestMatchListener(); - e.setField("pid", "796"); - e.setField("db", "dev"); - e.setField("user", "awsuser"); - e.setField("timestamp", "2021-08-23T16:28:14Z"); - e.setField("dbprefix", "979326520502_guardiumredshift"); - e.setField("serverHostnamePrefix", "979326520502_guardiumredshift"); - e.setField("sql_query", - "LOG: vacuum sort only sales to 75 percent"); - Collection results = filter.filter(Collections.singletonList(e), matchListener); - Assert.assertEquals(1, results.size()); - Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); - Assert.assertEquals(1, matchListener.getMatchCount()); + Context context = new ContextImpl(null, null); + RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); + Event e = new org.logstash.Event(); + TestMatchListener matchListener = new TestMatchListener(); + e.setField("pid", "796"); + e.setField("db", "dev"); + e.setField("user", "awsuser"); + e.setField("timestamp", "2021-08-23T16:28:14Z"); + e.setField("dbprefix", "979326520502_guardiumredshift"); + e.setField("serverHostnamePrefix", "979326520502_guardiumredshift"); + e.setField("sql_query", + "LOG: vacuum sort only sales to 75 percent"); + Collection results = filter.filter(Collections.singletonList(e), matchListener); + Assert.assertEquals(1, results.size()); + Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); + Assert.assertEquals(1, matchListener.getMatchCount()); } @Test public void testCopy() { - Context context = new ContextImpl(null, null); - RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); - Event e = new org.logstash.Event(); - TestMatchListener matchListener = new TestMatchListener(); - e.setField("pid", "796"); - e.setField("db", "dev"); - e.setField("user", "awsuser"); - e.setField("timestamp", "2021-08-23T16:28:14Z"); - e.setField("dbprefix", "979326520502_guardiumredshift"); - e.setField("serverHostnamePrefix", "979326520502_guardiumredshift"); - e.setField("sql_query", - "LOG: copy category " - + "from 's3://mybucket/custdata' \r\n" - + "iam_role 'arn:aws:iam::0123456789012:role/MyRedshiftRole'"); - Collection results = filter.filter(Collections.singletonList(e), matchListener); - Assert.assertEquals(1, results.size()); - Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); - Assert.assertEquals(1, matchListener.getMatchCount()); + Context context = new ContextImpl(null, null); + RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); + Event e = new org.logstash.Event(); + TestMatchListener matchListener = new TestMatchListener(); + e.setField("pid", "796"); + e.setField("db", "dev"); + e.setField("user", "awsuser"); + e.setField("timestamp", "2021-08-23T16:28:14Z"); + e.setField("dbprefix", "979326520502_guardiumredshift"); + e.setField("serverHostnamePrefix", "979326520502_guardiumredshift"); + e.setField("sql_query", + "LOG: copy category " + + "from 's3://mybucket/custdata' \r\n" + + "iam_role 'arn:aws:iam::0123456789012:role/MyRedshiftRole'"); + Collection results = filter.filter(Collections.singletonList(e), matchListener); + Assert.assertEquals(1, results.size()); + Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); + Assert.assertEquals(1, matchListener.getMatchCount()); } @Test public void testSelectTop() { - Context context = new ContextImpl(null, null); - RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); - Event e = new org.logstash.Event(); - TestMatchListener matchListener = new TestMatchListener(); - e.setField("pid", "796"); - e.setField("db", "dev"); - e.setField("user", "awsuser"); - e.setField("timestamp", "2021-08-23T16:28:14Z"); - e.setField("dbprefix", "979326520502_guardiumredshift"); - e.setField("serverHostnamePrefix", "979326520502_guardiumredshift"); - e.setField("sql_query", - "LOG: select top 10 * from sales;"); - Collection results = filter.filter(Collections.singletonList(e), matchListener); - Assert.assertEquals(1, results.size()); - Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); - Assert.assertEquals(1, matchListener.getMatchCount()); + Context context = new ContextImpl(null, null); + RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); + Event e = new org.logstash.Event(); + TestMatchListener matchListener = new TestMatchListener(); + e.setField("pid", "796"); + e.setField("db", "dev"); + e.setField("user", "awsuser"); + e.setField("timestamp", "2021-08-23T16:28:14Z"); + e.setField("dbprefix", "979326520502_guardiumredshift"); + e.setField("serverHostnamePrefix", "979326520502_guardiumredshift"); + e.setField("sql_query", + "LOG: select top 10 * from sales;"); + Collection results = filter.filter(Collections.singletonList(e), matchListener); + Assert.assertEquals(1, results.size()); + Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); + Assert.assertEquals(1, matchListener.getMatchCount()); } @Test public void testCancel() { - Context context = new ContextImpl(null, null); - RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); - Event e = new org.logstash.Event(); - TestMatchListener matchListener = new TestMatchListener(); - e.setField("pid", "796"); - e.setField("db", "dev"); - e.setField("user", "awsuser"); - e.setField("timestamp", "2021-08-23T16:28:14Z"); - e.setField("dbprefix", "979326520502_guardiumredshift"); - e.setField("serverHostnamePrefix", "979326520502_guardiumredshift"); - e.setField("sql_query", - "LOG: cancel 802"); - Collection results = filter.filter(Collections.singletonList(e), matchListener); - Assert.assertEquals(1, results.size()); - Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); - Assert.assertEquals(1, matchListener.getMatchCount()); + Context context = new ContextImpl(null, null); + RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); + Event e = new org.logstash.Event(); + TestMatchListener matchListener = new TestMatchListener(); + e.setField("pid", "796"); + e.setField("db", "dev"); + e.setField("user", "awsuser"); + e.setField("timestamp", "2021-08-23T16:28:14Z"); + e.setField("dbprefix", "979326520502_guardiumredshift"); + e.setField("serverHostnamePrefix", "979326520502_guardiumredshift"); + e.setField("sql_query", + "LOG: cancel 802"); + Collection results = filter.filter(Collections.singletonList(e), matchListener); + Assert.assertEquals(1, results.size()); + Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); + Assert.assertEquals(1, matchListener.getMatchCount()); } @Test public void testparse_connectionLog() { @@ -338,7 +341,7 @@ public void testparseRecord1() { e.setField("dbprefix", "979326520502_guardiumredshift"); e.setField("serverHostnamePrefix", "979326520502_guardiumredshift"); e.setField("sql_query", - "LOG: CREATE TABLE T1 ( col1 Varchar(20) distkey sortkey );"); + "LOG: CREATE TABLE T1 ( ��col1 Varchar(20) distkey sortkey );"); Collection results = filter.filter(Collections.singletonList(e), matchListener); Assert.assertEquals(1, results.size()); Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); @@ -371,6 +374,7 @@ public void testparseSessionID_Invalid() { RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); Event e = new org.logstash.Event(); TestMatchListener matchListener = new TestMatchListener(); + e.setField("pid", "796"); e.setField("dbname", "dev"); e.setField("dbprefix", "979326520502_guardiumredshift"); e.setField("user", "awsuser"); @@ -459,503 +463,503 @@ public void testIsipv6() { } @Test public void testSelectPIVOT() { - Context context = new ContextImpl(null, null); - RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); - Event e = new org.logstash.Event(); - TestMatchListener matchListener = new TestMatchListener(); - e.setField("pid", "796"); - e.setField("dbname", "dev"); - e.setField("user", "awsuser"); - e.setField("timestamp", "2021-08-23T16:28:14Z"); - e.setField("dbprefix", "979326520502_guardiumredshift"); - e.setField("serverHostnamePrefix", "979326520502_guardiumredshift"); - e.setField("sql_query", - "LOG: SELECT *\r\n" - + "FROM (SELECT partname, price FROM part) PIVOT (\r\n" - + " AVG(price) FOR partname IN ('P1', 'P2', 'P3')\r\n" - + ");"); - Collection results = filter.filter(Collections.singletonList(e), matchListener); - Assert.assertEquals(1, results.size()); - Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); - Assert.assertEquals(1, matchListener.getMatchCount()); + Context context = new ContextImpl(null, null); + RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); + Event e = new org.logstash.Event(); + TestMatchListener matchListener = new TestMatchListener(); + e.setField("pid", "796"); + e.setField("dbname", "dev"); + e.setField("user", "awsuser"); + e.setField("timestamp", "2021-08-23T16:28:14Z"); + e.setField("dbprefix", "979326520502_guardiumredshift"); + e.setField("serverHostnamePrefix", "979326520502_guardiumredshift"); + e.setField("sql_query", + "LOG: SELECT *\r\n" + + "FROM (SELECT partname, price FROM part) PIVOT (\r\n" + + " AVG(price) FOR partname IN ('P1', 'P2', 'P3')\r\n" + + ");"); + Collection results = filter.filter(Collections.singletonList(e), matchListener); + Assert.assertEquals(1, results.size()); + Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); + Assert.assertEquals(1, matchListener.getMatchCount()); } @Test public void testSelectUNPIVOT() { - Context context = new ContextImpl(null, null); - RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); - Event e = new org.logstash.Event(); - TestMatchListener matchListener = new TestMatchListener(); - e.setField("pid", "796"); - e.setField("dbname", "dev"); - e.setField("user", "awsuser"); - e.setField("timestamp", "2021-08-23T16:28:14Z"); - e.setField("dbprefix", "979326520502_guardiumredshift"); - e.setField("serverHostnamePrefix", "979326520502_guardiumredshift"); - e.setField("sql_query", - "LOG: SELECT * \r\n " - + "FROM (SELECT red, green, blue FROM count_by_color) UNPIVOT (\r\n" - + " cnt FOR color IN (red, green, blue)\r\n" - + ");"); - Collection results = filter.filter(Collections.singletonList(e), matchListener); - Assert.assertEquals(1, results.size()); - Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); - Assert.assertEquals(1, matchListener.getMatchCount()); + Context context = new ContextImpl(null, null); + RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); + Event e = new org.logstash.Event(); + TestMatchListener matchListener = new TestMatchListener(); + e.setField("pid", "796"); + e.setField("dbname", "dev"); + e.setField("user", "awsuser"); + e.setField("timestamp", "2021-08-23T16:28:14Z"); + e.setField("dbprefix", "979326520502_guardiumredshift"); + e.setField("serverHostnamePrefix", "979326520502_guardiumredshift"); + e.setField("sql_query", + "LOG: SELECT * \r\n " + + "FROM (SELECT red, green, blue FROM count_by_color) UNPIVOT (\r\n" + + " cnt FOR color IN (red, green, blue)\r\n" + + ");"); + Collection results = filter.filter(Collections.singletonList(e), matchListener); + Assert.assertEquals(1, results.size()); + Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); + Assert.assertEquals(1, matchListener.getMatchCount()); } @Test public void testifExists() { - Context context = new ContextImpl(null, null); - RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); - Event e = new org.logstash.Event(); - TestMatchListener matchListener = new TestMatchListener(); - e.setField("pid", "796"); - e.setField("db", "dev"); - e.setField("user", "awsuser"); - e.setField("timestamp", "2021-08-23T16:28:14Z"); - e.setField("dbprefix", "979326520502_guardiumredshift"); - e.setField("serverHostnamePrefix", "979326520502_guardiumredshift"); - e.setField("sql_query", - "LOG: create table if exists cities( cityid integer not null, city varchar(100) not null,\r\n" - + "state char(2) not null);"); - Collection results = filter.filter(Collections.singletonList(e), matchListener); - Assert.assertEquals(1, results.size()); - Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); - Assert.assertEquals(1, matchListener.getMatchCount()); + Context context = new ContextImpl(null, null); + RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); + Event e = new org.logstash.Event(); + TestMatchListener matchListener = new TestMatchListener(); + e.setField("pid", "796"); + e.setField("db", "dev"); + e.setField("user", "awsuser"); + e.setField("timestamp", "2021-08-23T16:28:14Z"); + e.setField("dbprefix", "979326520502_guardiumredshift"); + e.setField("serverHostnamePrefix", "979326520502_guardiumredshift"); + e.setField("sql_query", + "LOG: create table if exists cities( cityid integer not null, city varchar(100) not null,\r\n" + + "state char(2) not null);"); + Collection results = filter.filter(Collections.singletonList(e), matchListener); + Assert.assertEquals(1, results.size()); + Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); + Assert.assertEquals(1, matchListener.getMatchCount()); } @Test public void testifExists1() { - Context context = new ContextImpl(null, null); - RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); - Event e = new org.logstash.Event(); - TestMatchListener matchListener = new TestMatchListener(); - e.setField("pid", "796"); - e.setField("db", "dev"); - e.setField("user", "awsuser"); - e.setField("timestamp", "2021-08-23T16:28:14Z"); - e.setField("dbprefix", "979326520502_guardiumredshift"); - e.setField("serverHostnamePrefix", "979326520502_guardiumredshift"); - e.setField("sql_query", - "LOG: DROP MODEL IF EXISTS remote_customer_churn;"); - Collection results = filter.filter(Collections.singletonList(e), matchListener); - Assert.assertEquals(1, results.size()); - Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); - Assert.assertEquals(1, matchListener.getMatchCount()); + Context context = new ContextImpl(null, null); + RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); + Event e = new org.logstash.Event(); + TestMatchListener matchListener = new TestMatchListener(); + e.setField("pid", "796"); + e.setField("db", "dev"); + e.setField("user", "awsuser"); + e.setField("timestamp", "2021-08-23T16:28:14Z"); + e.setField("dbprefix", "979326520502_guardiumredshift"); + e.setField("serverHostnamePrefix", "979326520502_guardiumredshift"); + e.setField("sql_query", + "LOG: DROP MODEL IF EXISTS remote_customer_churn;"); + Collection results = filter.filter(Collections.singletonList(e), matchListener); + Assert.assertEquals(1, results.size()); + Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); + Assert.assertEquals(1, matchListener.getMatchCount()); } @Test public void testCreateFunction() { - Context context = new ContextImpl(null, null); - RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); - Event e = new org.logstash.Event(); - TestMatchListener matchListener = new TestMatchListener(); - e.setField("pid", "796"); - e.setField("db", "dev"); - e.setField("user", "awsuser"); - e.setField("timestamp", "2021-08-23T16:28:14Z"); - e.setField("dbprefix", "979326520502_guardiumredshift"); - e.setField("serverHostnamePrefix", "979326520502_guardiumredshift"); - e.setField("sql_query", - "LOG: create function f_sqlTest (a int, b int )\r\n" - + "returns int\r\n" - + "stable\r\n" - + "as $$\r\n" - + "select 1\r\n" - + "$$ language sql"); - Collection results = filter.filter(Collections.singletonList(e), matchListener); - Assert.assertEquals(1, results.size()); - Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); - Assert.assertEquals(1, matchListener.getMatchCount()); + Context context = new ContextImpl(null, null); + RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); + Event e = new org.logstash.Event(); + TestMatchListener matchListener = new TestMatchListener(); + e.setField("pid", "796"); + e.setField("db", "dev"); + e.setField("user", "awsuser"); + e.setField("timestamp", "2021-08-23T16:28:14Z"); + e.setField("dbprefix", "979326520502_guardiumredshift"); + e.setField("serverHostnamePrefix", "979326520502_guardiumredshift"); + e.setField("sql_query", + "LOG: create function f_sqlTest (a int, b int )\r\n" + + "returns int\r\n" + + "stable\r\n" + + "as $$\r\n" + + "select 1\r\n" + + "$$ language sql"); + Collection results = filter.filter(Collections.singletonList(e), matchListener); + Assert.assertEquals(1, results.size()); + Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); + Assert.assertEquals(1, matchListener.getMatchCount()); } @Test public void testCreateExternalSchema() { - Context context = new ContextImpl(null, null); - RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); - Event e = new org.logstash.Event(); - TestMatchListener matchListener = new TestMatchListener(); - e.setField("pid", "796"); - e.setField("db", "dev"); - e.setField("user", "awsuser"); - e.setField("timestamp", "2021-08-23T16:28:14Z"); - e.setField("dbprefix", "979326520502_guardiumredshift"); - e.setField("serverHostnamePrefix", "979326520502_guardiumredshift"); - e.setField("sql_query", - "LOG: CREATE EXTERNAL SCHEMA example_schema FROM" - + "DATA CATALOG\r\n " - + "DATABASE 'dev'\r\n" - + "REGION 'us-east-1'\r\n" - + "IAM_ROLE 'arn:aws:iam::979326520502:role/myspectrum_role'\r\n" - + "CREATE EXTERNAL DATABASE IF NOT EXISTS;"); - Collection results = filter.filter(Collections.singletonList(e), matchListener); - Assert.assertEquals(1, results.size()); - Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); - Assert.assertEquals(1, matchListener.getMatchCount()); + Context context = new ContextImpl(null, null); + RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); + Event e = new org.logstash.Event(); + TestMatchListener matchListener = new TestMatchListener(); + e.setField("pid", "796"); + e.setField("db", "dev"); + e.setField("user", "awsuser"); + e.setField("timestamp", "2021-08-23T16:28:14Z"); + e.setField("dbprefix", "979326520502_guardiumredshift"); + e.setField("serverHostnamePrefix", "979326520502_guardiumredshift"); + e.setField("sql_query", + "LOG: CREATE EXTERNAL SCHEMA example_schema FROM" + + "DATA CATALOG\r\n " + + "DATABASE 'dev'\r\n" + + "REGION 'us-east-1'\r\n" + + "IAM_ROLE 'arn:aws:iam::979326520502:role/myspectrum_role'\r\n" + + "CREATE EXTERNAL DATABASE IF NOT EXISTS;"); + Collection results = filter.filter(Collections.singletonList(e), matchListener); + Assert.assertEquals(1, results.size()); + Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); + Assert.assertEquals(1, matchListener.getMatchCount()); } @Test public void testCreateExternalSchema1() { - Context context = new ContextImpl(null, null); - RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); - Event e = new org.logstash.Event(); - TestMatchListener matchListener = new TestMatchListener(); - e.setField("pid", "796"); - e.setField("db", "dev"); - e.setField("user", "awsuser"); - e.setField("timestamp", "2021-08-23T16:28:14Z"); - e.setField("dbprefix", "979326520502_guardiumredshift"); - e.setField("serverHostnamePrefix", "979326520502_guardiumredshift"); - e.setField("sql_query", - "LOG: CREATE EXTERNAL TABLE external_schema.external_table (\r\n" - + " col1 SMALLINT,\r\n" - + " col2 CHAR(1)\r\n" - + ")"); - Collection results = filter.filter(Collections.singletonList(e), matchListener); - Assert.assertEquals(1, results.size()); - Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); - Assert.assertEquals(1, matchListener.getMatchCount()); + Context context = new ContextImpl(null, null); + RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); + Event e = new org.logstash.Event(); + TestMatchListener matchListener = new TestMatchListener(); + e.setField("pid", "796"); + e.setField("db", "dev"); + e.setField("user", "awsuser"); + e.setField("timestamp", "2021-08-23T16:28:14Z"); + e.setField("dbprefix", "979326520502_guardiumredshift"); + e.setField("serverHostnamePrefix", "979326520502_guardiumredshift"); + e.setField("sql_query", + "LOG: CREATE EXTERNAL TABLE external_schema.external_table (\r\n" + + " col1 SMALLINT,\r\n" + + " col2 CHAR(1)\r\n" + + ")"); + Collection results = filter.filter(Collections.singletonList(e), matchListener); + Assert.assertEquals(1, results.size()); + Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); + Assert.assertEquals(1, matchListener.getMatchCount()); } @Test public void testAlterDatabase() { - Context context = new ContextImpl(null, null); - RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); - Event e = new org.logstash.Event(); - TestMatchListener matchListener = new TestMatchListener(); - e.setField("pid", "796"); - e.setField("db", "dev"); - e.setField("user", "awsuser"); - e.setField("timestamp", "2021-08-23T16:28:14Z"); - e.setField("dbprefix", "979326520502:guardiumredshift"); - e.setField("serverHostnamePrefix", "979326520502-guardiumredshift"); - e.setField("sql_query", - "LOG: alter database tickit_sandbox rename to tickit_test;"); - Collection results = filter.filter(Collections.singletonList(e), matchListener); - Assert.assertEquals(1, results.size()); - Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); - Assert.assertEquals(1, matchListener.getMatchCount()); + Context context = new ContextImpl(null, null); + RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); + Event e = new org.logstash.Event(); + TestMatchListener matchListener = new TestMatchListener(); + e.setField("pid", "796"); + e.setField("db", "dev"); + e.setField("user", "awsuser"); + e.setField("timestamp", "2021-08-23T16:28:14Z"); + e.setField("dbprefix", "979326520502:guardiumredshift"); + e.setField("serverHostnamePrefix", "979326520502-guardiumredshift"); + e.setField("sql_query", + "LOG: alter database tickit_sandbox rename to tickit_test;"); + Collection results = filter.filter(Collections.singletonList(e), matchListener); + Assert.assertEquals(1, results.size()); + Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); + Assert.assertEquals(1, matchListener.getMatchCount()); } @Test public void testAlterDatabase2() { - Context context = new ContextImpl(null, null); - RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); - Event e = new org.logstash.Event(); - TestMatchListener matchListener = new TestMatchListener(); - e.setField("pid", "796"); - e.setField("db", "dev"); - e.setField("user", "awsuser"); - e.setField("timestamp", "2021-08-23T16:28:14Z"); - e.setField("dbprefix", "979326520502:guardiumredshift"); - e.setField("serverHostnamePrefix", "979326520502-guardiumredshift"); - e.setField("sql_query", - "LOG: ALTER DATABASE sampledb ISOLATION LEVEL SNAPSHOT;"); - Collection results = filter.filter(Collections.singletonList(e), matchListener); - Assert.assertEquals(1, results.size()); - Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); - Assert.assertEquals(1, matchListener.getMatchCount()); + Context context = new ContextImpl(null, null); + RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); + Event e = new org.logstash.Event(); + TestMatchListener matchListener = new TestMatchListener(); + e.setField("pid", "796"); + e.setField("db", "dev"); + e.setField("user", "awsuser"); + e.setField("timestamp", "2021-08-23T16:28:14Z"); + e.setField("dbprefix", "979326520502:guardiumredshift"); + e.setField("serverHostnamePrefix", "979326520502-guardiumredshift"); + e.setField("sql_query", + "LOG: ALTER DATABASE sampledb ISOLATION LEVEL SNAPSHOT;"); + Collection results = filter.filter(Collections.singletonList(e), matchListener); + Assert.assertEquals(1, results.size()); + Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); + Assert.assertEquals(1, matchListener.getMatchCount()); } @Test public void testCreateMaterializedView() { - Context context = new ContextImpl(null, null); - RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); - Event e = new org.logstash.Event(); - TestMatchListener matchListener = new TestMatchListener(); - e.setField("pid", "796"); - e.setField("db", "dev"); - e.setField("user", "awsuser"); - e.setField("timestamp", "2021-08-23T16:28:14Z"); - e.setField("dbprefix", "979326520502:guardiumredshift"); - e.setField("serverHostnamePrefix", "979326520502-guardiumredshift"); - e.setField("sql_query", - "LOG: CREATE MATERIALIZED VIEW tickets_mv AS\r\n" - + " select catgroup,\r\n" - + " sum(qtysold) as sold\r\n" - + " from category c, event e, sales s\r\n" - + " where c.catid = e.catid\r\n" - + " and e.eventid = s.eventid\r\n" - + " group by catgroup;"); - Collection results = filter.filter(Collections.singletonList(e), matchListener); - Assert.assertEquals(1, results.size()); - Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); - Assert.assertEquals(1, matchListener.getMatchCount()); + Context context = new ContextImpl(null, null); + RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); + Event e = new org.logstash.Event(); + TestMatchListener matchListener = new TestMatchListener(); + e.setField("pid", "796"); + e.setField("db", "dev"); + e.setField("user", "awsuser"); + e.setField("timestamp", "2021-08-23T16:28:14Z"); + e.setField("dbprefix", "979326520502:guardiumredshift"); + e.setField("serverHostnamePrefix", "979326520502-guardiumredshift"); + e.setField("sql_query", + "LOG: CREATE MATERIALIZED VIEW tickets_mv AS\r\n" + + " select catgroup,\r\n" + + " sum(qtysold) as sold\r\n" + + " from category c, event e, sales s\r\n" + + " where c.catid = e.catid\r\n" + + " and e.eventid = s.eventid\r\n" + + " group by catgroup;"); + Collection results = filter.filter(Collections.singletonList(e), matchListener); + Assert.assertEquals(1, results.size()); + Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); + Assert.assertEquals(1, matchListener.getMatchCount()); } @Test public void testCreateMaterializedView2() { - Context context = new ContextImpl(null, null); - RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); - Event e = new org.logstash.Event(); - TestMatchListener matchListener = new TestMatchListener(); - e.setField("pid", "796"); - e.setField("db", "dev"); - e.setField("user", "awsuser"); - e.setField("timestamp", "2021-08-23T16:28:14Z"); - e.setField("dbprefix", "979326520502:guardiumredshift"); - e.setField("serverHostnamePrefix", "979326520502-guardiumredshift"); - e.setField("sql_query", - "LOG: CREATE MATERIALIZED VIEW mv_sales_vw as\r\n" - + "select salesid, qtysold, pricepaid, commission, saletime from public.sales\r\n" - + "union all\r\n" - + "select salesid, qtysold, pricepaid, commission, saletime from spectrum.sales;"); - Collection results = filter.filter(Collections.singletonList(e), matchListener); - Assert.assertEquals(1, results.size()); - Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); - Assert.assertEquals(1, matchListener.getMatchCount()); + Context context = new ContextImpl(null, null); + RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); + Event e = new org.logstash.Event(); + TestMatchListener matchListener = new TestMatchListener(); + e.setField("pid", "796"); + e.setField("db", "dev"); + e.setField("user", "awsuser"); + e.setField("timestamp", "2021-08-23T16:28:14Z"); + e.setField("dbprefix", "979326520502:guardiumredshift"); + e.setField("serverHostnamePrefix", "979326520502-guardiumredshift"); + e.setField("sql_query", + "LOG: CREATE MATERIALIZED VIEW mv_sales_vw as\r\n" + + "select salesid, qtysold, pricepaid, commission, saletime from public.sales\r\n" + + "union all\r\n" + + "select salesid, qtysold, pricepaid, commission, saletime from spectrum.sales;"); + Collection results = filter.filter(Collections.singletonList(e), matchListener); + Assert.assertEquals(1, results.size()); + Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); + Assert.assertEquals(1, matchListener.getMatchCount()); } @Test public void testCreateMaterializedView3() { - Context context = new ContextImpl(null, null); - RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); - Event e = new org.logstash.Event(); - TestMatchListener matchListener = new TestMatchListener(); - e.setField("pid", "796"); - e.setField("db", "dev"); - e.setField("user", "awsuser"); - e.setField("timestamp", "2021-08-23T16:28:14Z"); - e.setField("dbprefix", "979326520502:guardiumredshift"); - e.setField("serverHostnamePrefix", "979326520502-guardiumredshift"); - e.setField("sql_query", - "LOG: create materialized view mv_sales_vw as select a from t;"); - Collection results = filter.filter(Collections.singletonList(e), matchListener); - Assert.assertEquals(1, results.size()); - Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); - Assert.assertEquals(1, matchListener.getMatchCount()); + Context context = new ContextImpl(null, null); + RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); + Event e = new org.logstash.Event(); + TestMatchListener matchListener = new TestMatchListener(); + e.setField("pid", "796"); + e.setField("db", "dev"); + e.setField("user", "awsuser"); + e.setField("timestamp", "2021-08-23T16:28:14Z"); + e.setField("dbprefix", "979326520502:guardiumredshift"); + e.setField("serverHostnamePrefix", "979326520502-guardiumredshift"); + e.setField("sql_query", + "LOG: create materialized view mv_sales_vw as select a from t;"); + Collection results = filter.filter(Collections.singletonList(e), matchListener); + Assert.assertEquals(1, results.size()); + Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); + Assert.assertEquals(1, matchListener.getMatchCount()); } @Test public void testDropMaterializedView() { - Context context = new ContextImpl(null, null); - RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); - Event e = new org.logstash.Event(); - TestMatchListener matchListener = new TestMatchListener(); - e.setField("pid", "796"); - e.setField("db", "dev"); - e.setField("user", "awsuser"); - e.setField("timestamp", "2021-08-23T16:28:14Z"); - e.setField("dbprefix", "979326520502:guardiumredshift"); - e.setField("serverHostnamePrefix", "979326520502-guardiumredshift"); - e.setField("sql_query", - "LOG: DROP MATERIALIZED VIEW IF EXISTS mv_name;"); - Collection results = filter.filter(Collections.singletonList(e), matchListener); - Assert.assertEquals(1, results.size()); - Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); - Assert.assertEquals(1, matchListener.getMatchCount()); + Context context = new ContextImpl(null, null); + RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); + Event e = new org.logstash.Event(); + TestMatchListener matchListener = new TestMatchListener(); + e.setField("pid", "796"); + e.setField("db", "dev"); + e.setField("user", "awsuser"); + e.setField("timestamp", "2021-08-23T16:28:14Z"); + e.setField("dbprefix", "979326520502:guardiumredshift"); + e.setField("serverHostnamePrefix", "979326520502-guardiumredshift"); + e.setField("sql_query", + "LOG: DROP MATERIALIZED VIEW IF EXISTS mv_name;"); + Collection results = filter.filter(Collections.singletonList(e), matchListener); + Assert.assertEquals(1, results.size()); + Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); + Assert.assertEquals(1, matchListener.getMatchCount()); } @Test public void testRefreshMaterializedView() { - Context context = new ContextImpl(null, null); - RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); - Event e = new org.logstash.Event(); - TestMatchListener matchListener = new TestMatchListener(); - e.setField("pid", "796"); - e.setField("db", "dev"); - e.setField("user", "awsuser"); - e.setField("timestamp", "2021-08-23T16:28:14Z"); - e.setField("dbprefix", "979326520502:guardiumredshift"); - e.setField("serverHostnamePrefix", "979326520502-guardiumredshift"); - e.setField("sql_query", - "LOG: REFRESH MATERIALIZED VIEW tickets_mv;"); - Collection results = filter.filter(Collections.singletonList(e), matchListener); - Assert.assertEquals(1, results.size()); - Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); - Assert.assertEquals(1, matchListener.getMatchCount()); + Context context = new ContextImpl(null, null); + RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); + Event e = new org.logstash.Event(); + TestMatchListener matchListener = new TestMatchListener(); + e.setField("pid", "796"); + e.setField("db", "dev"); + e.setField("user", "awsuser"); + e.setField("timestamp", "2021-08-23T16:28:14Z"); + e.setField("dbprefix", "979326520502:guardiumredshift"); + e.setField("serverHostnamePrefix", "979326520502-guardiumredshift"); + e.setField("sql_query", + "LOG: REFRESH MATERIALIZED VIEW tickets_mv;"); + Collection results = filter.filter(Collections.singletonList(e), matchListener); + Assert.assertEquals(1, results.size()); + Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); + Assert.assertEquals(1, matchListener.getMatchCount()); } @Test public void testAlterMaterializedView() { - Context context = new ContextImpl(null, null); - RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); - Event e = new org.logstash.Event(); - TestMatchListener matchListener = new TestMatchListener(); - e.setField("pid", "796"); - e.setField("db", "dev"); - e.setField("user", "awsuser"); - e.setField("timestamp", "2021-08-23T16:28:14Z"); - e.setField("dbprefix", "979326520502:guardiumredshift"); - e.setField("serverHostnamePrefix", "979326520502-guardiumredshift"); - e.setField("sql_query", - "LOG: ALTER MATERIALIZED VIEW tickets_mv AUTO REFRESH YES"); - Collection results = filter.filter(Collections.singletonList(e), matchListener); - Assert.assertEquals(1, results.size()); - Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); - Assert.assertEquals(1, matchListener.getMatchCount()); + Context context = new ContextImpl(null, null); + RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); + Event e = new org.logstash.Event(); + TestMatchListener matchListener = new TestMatchListener(); + e.setField("pid", "796"); + e.setField("db", "dev"); + e.setField("user", "awsuser"); + e.setField("timestamp", "2021-08-23T16:28:14Z"); + e.setField("dbprefix", "979326520502:guardiumredshift"); + e.setField("serverHostnamePrefix", "979326520502-guardiumredshift"); + e.setField("sql_query", + "LOG: ALTER MATERIALIZED VIEW tickets_mv AUTO REFRESH YES"); + Collection results = filter.filter(Collections.singletonList(e), matchListener); + Assert.assertEquals(1, results.size()); + Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); + Assert.assertEquals(1, matchListener.getMatchCount()); } @Test public void testCreateRole() { - Context context = new ContextImpl(null, null); - RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); - Event e = new org.logstash.Event(); - TestMatchListener matchListener = new TestMatchListener(); - e.setField("pid", "796"); - e.setField("db", "dev"); - e.setField("user", "awsuser"); - e.setField("timestamp", "2021-08-23T16:28:14Z"); - e.setField("dbprefix", "979326520502:guardiumredshift"); - e.setField("serverHostnamePrefix", "979326520502-guardiumredshift"); - e.setField("sql_query", - "LOG: CREATE ROLE sample_role1;"); - Collection results = filter.filter(Collections.singletonList(e), matchListener); - Assert.assertEquals(1, results.size()); - Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); - Assert.assertEquals(1, matchListener.getMatchCount()); + Context context = new ContextImpl(null, null); + RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); + Event e = new org.logstash.Event(); + TestMatchListener matchListener = new TestMatchListener(); + e.setField("pid", "796"); + e.setField("db", "dev"); + e.setField("user", "awsuser"); + e.setField("timestamp", "2021-08-23T16:28:14Z"); + e.setField("dbprefix", "979326520502:guardiumredshift"); + e.setField("serverHostnamePrefix", "979326520502-guardiumredshift"); + e.setField("sql_query", + "LOG: CREATE ROLE sample_role1;"); + Collection results = filter.filter(Collections.singletonList(e), matchListener); + Assert.assertEquals(1, results.size()); + Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); + Assert.assertEquals(1, matchListener.getMatchCount()); } @Test public void testCreateRole2() { - Context context = new ContextImpl(null, null); - RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); - Event e = new org.logstash.Event(); - TestMatchListener matchListener = new TestMatchListener(); - e.setField("pid", "796"); - e.setField("db", "dev"); - e.setField("user", "awsuser"); - e.setField("timestamp", "2021-08-23T16:28:14Z"); - e.setField("dbprefix", "979326520502:guardiumredshift"); - e.setField("serverHostnamePrefix", "979326520502-guardiumredshift"); - e.setField("sql_query", - "LOG: CREATE ROLE sample_role1 EXTERNALID \"ABC123\";"); - Collection results = filter.filter(Collections.singletonList(e), matchListener); - Assert.assertEquals(1, results.size()); - Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); - Assert.assertEquals(1, matchListener.getMatchCount()); + Context context = new ContextImpl(null, null); + RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); + Event e = new org.logstash.Event(); + TestMatchListener matchListener = new TestMatchListener(); + e.setField("pid", "796"); + e.setField("db", "dev"); + e.setField("user", "awsuser"); + e.setField("timestamp", "2021-08-23T16:28:14Z"); + e.setField("dbprefix", "979326520502:guardiumredshift"); + e.setField("serverHostnamePrefix", "979326520502-guardiumredshift"); + e.setField("sql_query", + "LOG: CREATE ROLE sample_role1 EXTERNALID \"ABC123\";"); + Collection results = filter.filter(Collections.singletonList(e), matchListener); + Assert.assertEquals(1, results.size()); + Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); + Assert.assertEquals(1, matchListener.getMatchCount()); } @Test public void testDropRole() { - Context context = new ContextImpl(null, null); - RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); - Event e = new org.logstash.Event(); - TestMatchListener matchListener = new TestMatchListener(); - e.setField("pid", "796"); - e.setField("db", "dev"); - e.setField("user", "awsuser"); - e.setField("timestamp", "2021-08-23T16:28:14Z"); - e.setField("dbprefix", "979326520502:guardiumredshift"); - e.setField("serverHostnamePrefix", "979326520502-guardiumredshift"); - e.setField("sql_query", - "LOG: DROP ROLE sample_role FORCE;"); - Collection results = filter.filter(Collections.singletonList(e), matchListener); - Assert.assertEquals(1, results.size()); - Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); - Assert.assertEquals(1, matchListener.getMatchCount()); + Context context = new ContextImpl(null, null); + RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); + Event e = new org.logstash.Event(); + TestMatchListener matchListener = new TestMatchListener(); + e.setField("pid", "796"); + e.setField("db", "dev"); + e.setField("user", "awsuser"); + e.setField("timestamp", "2021-08-23T16:28:14Z"); + e.setField("dbprefix", "979326520502:guardiumredshift"); + e.setField("serverHostnamePrefix", "979326520502-guardiumredshift"); + e.setField("sql_query", + "LOG: DROP ROLE sample_role FORCE;"); + Collection results = filter.filter(Collections.singletonList(e), matchListener); + Assert.assertEquals(1, results.size()); + Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); + Assert.assertEquals(1, matchListener.getMatchCount()); } @Test public void testAlterRole1() { - Context context = new ContextImpl(null, null); - RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); - Event e = new org.logstash.Event(); - TestMatchListener matchListener = new TestMatchListener(); - e.setField("pid", "796"); - e.setField("db", "dev"); - e.setField("user", "awsuser"); - e.setField("timestamp", "2021-08-23T16:28:14Z"); - e.setField("dbprefix", "979326520502:guardiumredshift"); - e.setField("serverHostnamePrefix", "979326520502-guardiumredshift"); - e.setField("sql_query", - "LOG: ALTER ROLE sample_role1 WITH RENAME TO sample_role2;"); - Collection results = filter.filter(Collections.singletonList(e), matchListener); - Assert.assertEquals(1, results.size()); - Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); - Assert.assertEquals(1, matchListener.getMatchCount()); + Context context = new ContextImpl(null, null); + RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); + Event e = new org.logstash.Event(); + TestMatchListener matchListener = new TestMatchListener(); + e.setField("pid", "796"); + e.setField("db", "dev"); + e.setField("user", "awsuser"); + e.setField("timestamp", "2021-08-23T16:28:14Z"); + e.setField("dbprefix", "979326520502:guardiumredshift"); + e.setField("serverHostnamePrefix", "979326520502-guardiumredshift"); + e.setField("sql_query", + "LOG: ALTER ROLE sample_role1 WITH RENAME TO sample_role2;"); + Collection results = filter.filter(Collections.singletonList(e), matchListener); + Assert.assertEquals(1, results.size()); + Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); + Assert.assertEquals(1, matchListener.getMatchCount()); } @Test public void testAlterRole2() { - Context context = new ContextImpl(null, null); - RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); - Event e = new org.logstash.Event(); - TestMatchListener matchListener = new TestMatchListener(); - e.setField("pid", "796"); - e.setField("db", "dev"); - e.setField("user", "awsuser"); - e.setField("timestamp", "2021-08-23T16:28:14Z"); - e.setField("dbprefix", "979326520502:guardiumredshift"); - e.setField("serverHostnamePrefix", "979326520502-guardiumredshift"); - e.setField("sql_query", - "LOG: ALTER ROLE sample_role1 EXTERNALID TO \"XYZ456\";"); - Collection results = filter.filter(Collections.singletonList(e), matchListener); - Assert.assertEquals(1, results.size()); - Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); - Assert.assertEquals(1, matchListener.getMatchCount()); + Context context = new ContextImpl(null, null); + RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); + Event e = new org.logstash.Event(); + TestMatchListener matchListener = new TestMatchListener(); + e.setField("pid", "796"); + e.setField("db", "dev"); + e.setField("user", "awsuser"); + e.setField("timestamp", "2021-08-23T16:28:14Z"); + e.setField("dbprefix", "979326520502:guardiumredshift"); + e.setField("serverHostnamePrefix", "979326520502-guardiumredshift"); + e.setField("sql_query", + "LOG: ALTER ROLE sample_role1 EXTERNALID TO \"XYZ456\";"); + Collection results = filter.filter(Collections.singletonList(e), matchListener); + Assert.assertEquals(1, results.size()); + Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); + Assert.assertEquals(1, matchListener.getMatchCount()); } @Test public void testCreateFunction1() { - Context context = new ContextImpl(null, null); - RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); - Event e = new org.logstash.Event(); - TestMatchListener matchListener = new TestMatchListener(); - e.setField("pid", "796"); - e.setField("db", "dev"); - e.setField("user", "awsuser"); - e.setField("timestamp", "2021-08-23T16:28:14Z"); - e.setField("dbprefix", "979326520502:guardiumredshift"); - e.setField("serverHostnamePrefix", "979326520502-guardiumredshift"); - e.setField("sql_query", - "LOG: CREATE OR REPLACE function f_sql_commission (float, float )\r\n" - + " returns float\r\n" - + "stable\r\n" - + "as $$\r\n" - + " select f_sql_greater ($1, $2) \r\n" - + "$$ language sql;"); - Collection results = filter.filter(Collections.singletonList(e), matchListener); - Assert.assertEquals(1, results.size()); - Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); - Assert.assertEquals(1, matchListener.getMatchCount()); + Context context = new ContextImpl(null, null); + RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); + Event e = new org.logstash.Event(); + TestMatchListener matchListener = new TestMatchListener(); + e.setField("pid", "796"); + e.setField("db", "dev"); + e.setField("user", "awsuser"); + e.setField("timestamp", "2021-08-23T16:28:14Z"); + e.setField("dbprefix", "979326520502:guardiumredshift"); + e.setField("serverHostnamePrefix", "979326520502-guardiumredshift"); + e.setField("sql_query", + "LOG: CREATE OR REPLACE function f_sql_commission (float, float )\r\n" + + " returns float\r\n" + + "stable\r\n" + + "as $$\r\n" + + " select f_sql_greater ($1, $2) \r\n" + + "$$ language sql;"); + Collection results = filter.filter(Collections.singletonList(e), matchListener); + Assert.assertEquals(1, results.size()); + Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); + Assert.assertEquals(1, matchListener.getMatchCount()); } @Test public void testCreateFunction2() { - Context context = new ContextImpl(null, null); - RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); - Event e = new org.logstash.Event(); - TestMatchListener matchListener = new TestMatchListener(); - e.setField("pid", "796"); - e.setField("db", "dev"); - e.setField("user", "awsuser"); - e.setField("timestamp", "2021-08-23T16:28:14Z"); - e.setField("dbprefix", "979326520502:guardiumredshift"); - e.setField("serverHostnamePrefix", "979326520502-guardiumredshift"); - e.setField("sql_query", - "LOG: create function f_sql_greater (float, float)\r\n" - + " returns float\r\n" - + "stable\r\n" - + "as $$\r\n" - + " select case when $1 > $2 then $1\r\n" - + " else $2\r\n" - + " end\r\n" - + "$$ language sql;"); - Collection results = filter.filter(Collections.singletonList(e), matchListener); - Assert.assertEquals(1, results.size()); - Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); - Assert.assertEquals(1, matchListener.getMatchCount()); + Context context = new ContextImpl(null, null); + RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); + Event e = new org.logstash.Event(); + TestMatchListener matchListener = new TestMatchListener(); + e.setField("pid", "796"); + e.setField("db", "dev"); + e.setField("user", "awsuser"); + e.setField("timestamp", "2021-08-23T16:28:14Z"); + e.setField("dbprefix", "979326520502:guardiumredshift"); + e.setField("serverHostnamePrefix", "979326520502-guardiumredshift"); + e.setField("sql_query", + "LOG: create function f_sql_greater (float, float)\r\n" + + " returns float\r\n" + + "stable\r\n" + + "as $$\r\n" + + " select case when $1 > $2 then $1\r\n" + + " else $2\r\n" + + " end\r\n" + + "$$ language sql;"); + Collection results = filter.filter(Collections.singletonList(e), matchListener); + Assert.assertEquals(1, results.size()); + Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); + Assert.assertEquals(1, matchListener.getMatchCount()); } @Test public void testDropFunction1() { - Context context = new ContextImpl(null, null); - RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); - Event e = new org.logstash.Event(); - TestMatchListener matchListener = new TestMatchListener(); - e.setField("pid", "796"); - e.setField("db", "dev"); - e.setField("user", "awsuser"); - e.setField("timestamp", "2021-08-23T16:28:14Z"); - e.setField("dbprefix", "979326520502:guardiumredshift"); - e.setField("serverHostnamePrefix", "979326520502-guardiumredshift"); - e.setField("sql_query", - "LOG: drop function f_sqrt(int);"); - Collection results = filter.filter(Collections.singletonList(e), matchListener); - Assert.assertEquals(1, results.size()); - Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); - Assert.assertEquals(1, matchListener.getMatchCount()); + Context context = new ContextImpl(null, null); + RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); + Event e = new org.logstash.Event(); + TestMatchListener matchListener = new TestMatchListener(); + e.setField("pid", "796"); + e.setField("db", "dev"); + e.setField("user", "awsuser"); + e.setField("timestamp", "2021-08-23T16:28:14Z"); + e.setField("dbprefix", "979326520502:guardiumredshift"); + e.setField("serverHostnamePrefix", "979326520502-guardiumredshift"); + e.setField("sql_query", + "LOG: drop function f_sqrt(int);"); + Collection results = filter.filter(Collections.singletonList(e), matchListener); + Assert.assertEquals(1, results.size()); + Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); + Assert.assertEquals(1, matchListener.getMatchCount()); } @Test public void testDropFunction2() { - Context context = new ContextImpl(null, null); - RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); - Event e = new org.logstash.Event(); - TestMatchListener matchListener = new TestMatchListener(); - e.setField("pid", "796"); - e.setField("db", "dev"); - e.setField("user", "awsuser"); - e.setField("timestamp", "2021-08-23T16:28:14Z"); - e.setField("dbprefix", "979326520502:guardiumredshift"); - e.setField("serverHostnamePrefix", "979326520502-guardiumredshift"); - e.setField("sql_query", - "LOG: drop function f_sqrt(int)restrict;"); - Collection results = filter.filter(Collections.singletonList(e), matchListener); - Assert.assertEquals(1, results.size()); - Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); - Assert.assertEquals(1, matchListener.getMatchCount()); + Context context = new ContextImpl(null, null); + RedShiftGuardiumConnector filter = new RedShiftGuardiumConnector("rds", null, context); + Event e = new org.logstash.Event(); + TestMatchListener matchListener = new TestMatchListener(); + e.setField("pid", "796"); + e.setField("db", "dev"); + e.setField("user", "awsuser"); + e.setField("timestamp", "2021-08-23T16:28:14Z"); + e.setField("dbprefix", "979326520502:guardiumredshift"); + e.setField("serverHostnamePrefix", "979326520502-guardiumredshift"); + e.setField("sql_query", + "LOG: drop function f_sqrt(int)restrict;"); + Collection results = filter.filter(Collections.singletonList(e), matchListener); + Assert.assertEquals(1, results.size()); + Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); + Assert.assertEquals(1, matchListener.getMatchCount()); } } @@ -971,4 +975,4 @@ public void filterMatched(Event event) { public int getMatchCount() { return matchCount.get(); } -} +} \ No newline at end of file diff --git a/filter-plugin/logstash-filter-s3-guardium/build.gradle b/filter-plugin/logstash-filter-s3-guardium/build.gradle index b743e1a9d..1b74723f1 100644 --- a/filter-plugin/logstash-filter-s3-guardium/build.gradle +++ b/filter-plugin/logstash-filter-s3-guardium/build.gradle @@ -2,6 +2,30 @@ import java.nio.file.Files import static java.nio.file.StandardCopyOption.REPLACE_EXISTING apply plugin: 'java' + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + + dependencies { + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' + classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.1.0" + classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' + } +} + +def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" // =========================================================================== @@ -20,10 +44,10 @@ pluginInfo.pluginClass = "LogstashFilterS3Guardium" pluginInfo.pluginName = "logstash_filter_s3_guardium" // must match the @LogstashPlugin annotation in the main plugin class // =========================================================================== -sourceCompatibility = 1.8 -targetCompatibility = 1.8 +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 -def jacocoVersion = '0.8.4' +def jacocoVersion = '0.8.11' // minimumCoverage can be set by Travis ENV def minimumCoverageStr = System.getenv("MINIMUM_COVERAGE") ?: "50.0%" if (minimumCoverageStr.endsWith("%")) { @@ -31,27 +55,16 @@ if (minimumCoverageStr.endsWith("%")) { } def minimumCoverage = Float.valueOf(minimumCoverageStr) / 100 -buildscript { - repositories { - maven { - url "https://plugins.gradle.org/m2/" - } - mavenCentral() - jcenter() - } - - dependencies { - classpath 'com.github.jengelman.gradle.plugins:shadow:4.0.4' - classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.0.0-RC2" - classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' - } -} - -def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); -def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) repositories { - mavenCentral() + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } } tasks.register("vendor"){ @@ -62,14 +75,13 @@ tasks.register("vendor"){ File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") projectJarFile.mkdirs() Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) - validatePluginJar(projectJarFile, project.group) } } apply plugin: 'com.github.johnrengelman.shadow' shadowJar { - classifier = null + archiveClassifier = null } dependencies { @@ -77,7 +89,8 @@ dependencies { implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: versions.dependencies.log4jCore implementation 'org.apache.commons:commons-lang3:' + versions.dependencies.commonsLang implementation 'com.google.code.gson:gson:' + versions.dependencies.gson - implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "build/libs/logstash-core-*.*.*.jar") + implementation group: 'commons-beanutils', name: 'commons-beanutils', version: versions.dependencies.commonsBeanutils + implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "build/libs/logstash-core.jar") implementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "common-*.*.*.jar") testImplementation 'junit:junit:' + versions.dependencies.junit @@ -100,6 +113,19 @@ tasks.withType(JavaCompile) { options.encoding = 'UTF-8' } +test { + jvmArgs = [ + '--add-opens=java.base/java.lang=ALL-UNNAMED', + '--add-opens=java.base/java.util=ALL-UNNAMED', + '--add-opens=java.base/java.text=ALL-UNNAMED', + '--add-opens=java.base/java.lang.reflect=ALL-UNNAMED', + '--add-opens=java.base/sun.util.resources=ALL-UNNAMED', + '--add-opens=java.base/sun.util.cldr=ALL-UNNAMED', + '--add-opens=java.base/jdk.internal.reflect=ALL-UNNAMED', + '--illegal-access=permit' + ] +} + tasks.register("generateRubySupportFiles") { @@ -142,17 +168,16 @@ apply plugin: "org.barfuin.gradle.jacocolog" // ------------------------------------ jacoco { toolVersion = "${jacocoVersion}" - reportsDir = file("$buildDir/reports/jacoco") } jacocoTestReport { // You will see "Report -> file://...." at the end of a JaCoCo build // If no output, run this first: ./gradlew test reports { - html.enabled true - xml.enabled true - csv.enabled true - html.destination file("${buildDir}/reports/jacoco") - csv.destination file("${buildDir}/reports/jacoco/all.csv") + html.required = true + xml.required = true + csv.required = true + html.outputLocation = file("${buildDir}/reports/jacoco") + csv.outputLocation = file("${buildDir}/reports/jacoco/all.csv") } executionData.from fileTree(dir: "${buildDir}/jacoco/", includes: [ '**/*.exec' diff --git a/filter-plugin/logstash-filter-s3-guardium/gradle/wrapper/gradle-wrapper.properties b/filter-plugin/logstash-filter-s3-guardium/gradle/wrapper/gradle-wrapper.properties index aa991fcea..81aa1c044 100644 --- a/filter-plugin/logstash-filter-s3-guardium/gradle/wrapper/gradle-wrapper.properties +++ b/filter-plugin/logstash-filter-s3-guardium/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/filter-plugin/logstash-filter-s3-guardium/src/main/java/com/ibm/guardium/s3/Parser.java b/filter-plugin/logstash-filter-s3-guardium/src/main/java/com/ibm/guardium/s3/Parser.java index e5d053924..d12092e2e 100644 --- a/filter-plugin/logstash-filter-s3-guardium/src/main/java/com/ibm/guardium/s3/Parser.java +++ b/filter-plugin/logstash-filter-s3-guardium/src/main/java/com/ibm/guardium/s3/Parser.java @@ -11,11 +11,20 @@ import java.util.*; import com.google.gson.*; -import com.ibm.guardium.universalconnector.commons.structures.*; +import com.ibm.guardium.universalconnector.commons.structures.Accessor; +import com.ibm.guardium.universalconnector.commons.structures.Construct; +import com.ibm.guardium.universalconnector.commons.structures.Data; +import com.ibm.guardium.universalconnector.commons.structures.ExceptionRecord; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import com.ibm.guardium.universalconnector.commons.structures.Sentence; +import com.ibm.guardium.universalconnector.commons.structures.SentenceObject; +import com.ibm.guardium.universalconnector.commons.structures.SessionLocator; +import com.ibm.guardium.universalconnector.commons.structures.Time; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import sun.net.util.IPAddressUtil; +import java.net.InetAddress; +import java.net.UnknownHostException; public class Parser { @@ -28,7 +37,7 @@ public class Parser { public static final String BUCKETNAME_PROPERTY = "bucketName"; public static final String BUCKET_PROPERTY = "Bucket"; public static final String S3_TYPE = "S3"; - public static final String S3_PROTOCOL = "S3 native audit"; + public static final String S3_PROTOCOL = "S3"; private static final Gson gson = new Gson(); @@ -279,9 +288,11 @@ public static String getHost(JsonObject requestParameters){ } public static String validateIP(String sourceIPAddress){ - if (IPAddressUtil.isIPv4LiteralAddress(sourceIPAddress) || IPAddressUtil.isIPv6LiteralAddress(sourceIPAddress)){ + try { + // InetAddress.getByName() validates both IPv4 and IPv6 addresses + InetAddress.getByName(sourceIPAddress); return sourceIPAddress; - } else { + } catch (UnknownHostException e) { return UNKNOWN_IP; } } diff --git a/filter-plugin/logstash-filter-saphana-guardium/LogstashREADME.md b/filter-plugin/logstash-filter-saphana-guardium/LogstashREADME.md new file mode 100644 index 000000000..5f5370653 --- /dev/null +++ b/filter-plugin/logstash-filter-saphana-guardium/LogstashREADME.md @@ -0,0 +1,88 @@ +# SAP HANA-Guardium Logstash filter plug-in +### Meet SAP HANA +* Tested versions: 2.00.033.00.1535711040 +* Environment: On-premise, Saas +* Supported Guardium versions: + * Guardium Data Protection: 11.4 and above + * Supported inputs: + * Filebeat (push) + * JDBC (pull) + * Guardium Data Security Center SaaS: 1.0 + * Supported inputs: + * Filebeat (push) + * JDBC (pull) + +This is a [Logstash](https://github.com/elastic/logstash) filter plug-in for the universal connector that is featured in IBM Security Guardium. +It parses events and messages from the SAP HANA audit log into a [Guardium record](https://github.com/IBM/universal-connectors/blob/main/common/src/main/java/com/ibm/guardium/universalconnector/commons/structures/Record.java) instance (which is a standard +structure made out of several parts). The information is then sent over to Guardium. Guardium records include the +accessor (the person who tried to access the data), the session, data, and exceptions. If there are no errors, the data +contains details about the query "construct". The construct details the main action (verb) and +collections (objects) involved. + +The plug-in is free and open-source (Apache 2.0). It can be used as a starting point to develop additional filter +plug-ins for Guardium universal connector. + +## Configuring the SAP HANA + +There are multiple ways to install a SAP HANA server. For this example, we will assume that we already have a working +SAP HANA setup. + +## Enabling the audit logs: +### Procedure +In the SAP HANA Studio, expand the system on which you would like to enable auditing. +1. Expand the Security folder. +2. Double click on the ‘Security option’. +3. Click on the auditing status drop-down menu, by default it will be disabled. +4. Select "Enabled". +5. Click "Deploy" or press F8 to save the changes. +6. Restart database instance to reflect new changes. + + +There are multiple ways to enable auditing in SAP HANA, You can choose as per your requirement. +* CSTABLE base auditing - Audit-trail target is a table, requires JDBC input plug-in. + + [SAPHANA Using JDBC Input](./saphanaUsingJDBCREADME.md) + + [SAPHANA Cloud Using JDBC Input](./saphanaCloudUsingJDBCREADME.md) + +* CSVTEXTFILE base auditing - Audit-trail target is a file, requires Beat input plugin. + + [SAPHANA Using FILEBEAT Input](./saphanaUsingFilebeatREADME.md) + + +## Troubleshooting + +If you encounter an error like the following when trying to connect to the database: + +``` +2024-07-04 21:15:48 ERROR jdbc:127 - Unable to connect to database. Tried 1 times {:message=>"Java::ComSapDbJdbcExceptions::SQLInvalidAuthorizationSpecExceptionSapDB: [10]: authentication failed", :exception=>Sequel::DatabaseConnectionError, :cause=>#, :backtrace=>["com.sap.db.jdbc.exceptions.SQLExceptionSapDB._newInstance(com/sap/db/jdbc/exceptions/SQLExceptionSapDB.java:183)", "com.sap.db.jdbc.exceptions.SQLExceptionSapDB.newInstance(com/sap/db/jdbc/exceptions/SQLExceptionSapDB.java:42)", +``` + +This error indicates an **authentication failure** when connecting to the database. It is often caused by issues with the provided credentials (username or password) or incorrect database configuration. + +## Steps to resolve: + +1. **Verify Database Credentials** + Check the **username** and **password** provided in your database connection configuration. Ensure that: + - The username and password are correct. + - The username has sufficient privileges to access the database. + - The password is correctly formatted, without any trailing spaces or special characters that may cause issues. + + + +3. **Test Credentials Manually** + Try connecting to the database manually using a database client or command line tool (e.g., SAP HANA Studio, `hdbsql`, or another SQL client). This can help verify that the credentials are valid and that the database is accessible. + +4. **Check for Locked or Expired Accounts** + If you have confirmed the credentials are correct, check if the account is locked or if the password has expired. Some databases automatically lock accounts after multiple failed login attempts. + + +### Limitations: + +1. SAP HANA auditing only supports error logs for authentication failures. +2. SAP HANA does not audit multiple line query properly. +3. Single line and multi-line comments in between the query are not supported. +4. The SAP HANA CSVTEXTFILE(audit target) does not audit the DB_Name. +5. SAP HANA with JDBC shows server ip as 0.0.0.0 +6. Duplicate records will be seen in load balancing. +7. SAPHANA SYSLOG does not natively support load balancing. \ No newline at end of file diff --git a/filter-plugin/logstash-filter-saphana-guardium/README.md b/filter-plugin/logstash-filter-saphana-guardium/README.md index 5f5370653..cd6c81e7f 100644 --- a/filter-plugin/logstash-filter-saphana-guardium/README.md +++ b/filter-plugin/logstash-filter-saphana-guardium/README.md @@ -1,88 +1,9 @@ -# SAP HANA-Guardium Logstash filter plug-in -### Meet SAP HANA -* Tested versions: 2.00.033.00.1535711040 -* Environment: On-premise, Saas -* Supported Guardium versions: - * Guardium Data Protection: 11.4 and above - * Supported inputs: - * Filebeat (push) - * JDBC (pull) - * Guardium Data Security Center SaaS: 1.0 - * Supported inputs: - * Filebeat (push) - * JDBC (pull) +# SapHana Universal Connector -This is a [Logstash](https://github.com/elastic/logstash) filter plug-in for the universal connector that is featured in IBM Security Guardium. -It parses events and messages from the SAP HANA audit log into a [Guardium record](https://github.com/IBM/universal-connectors/blob/main/common/src/main/java/com/ibm/guardium/universalconnector/commons/structures/Record.java) instance (which is a standard -structure made out of several parts). The information is then sent over to Guardium. Guardium records include the -accessor (the person who tried to access the data), the session, data, and exceptions. If there are no errors, the data -contains details about the query "construct". The construct details the main action (verb) and -collections (objects) involved. +## Follow this link to set up and use SapHana Universal Connector over JDBC Logstash Plugin -The plug-in is free and open-source (Apache 2.0). It can be used as a starting point to develop additional filter -plug-ins for Guardium universal connector. +[SapHanaOverJDBC](./LogstashREADME.md) -## Configuring the SAP HANA +## Follow this link to set up and use SapHana Universal Connector over JDBC Connect -There are multiple ways to install a SAP HANA server. For this example, we will assume that we already have a working -SAP HANA setup. - -## Enabling the audit logs: -### Procedure -In the SAP HANA Studio, expand the system on which you would like to enable auditing. -1. Expand the Security folder. -2. Double click on the ‘Security option’. -3. Click on the auditing status drop-down menu, by default it will be disabled. -4. Select "Enabled". -5. Click "Deploy" or press F8 to save the changes. -6. Restart database instance to reflect new changes. - - -There are multiple ways to enable auditing in SAP HANA, You can choose as per your requirement. -* CSTABLE base auditing - Audit-trail target is a table, requires JDBC input plug-in. - - [SAPHANA Using JDBC Input](./saphanaUsingJDBCREADME.md) - - [SAPHANA Cloud Using JDBC Input](./saphanaCloudUsingJDBCREADME.md) - -* CSVTEXTFILE base auditing - Audit-trail target is a file, requires Beat input plugin. - - [SAPHANA Using FILEBEAT Input](./saphanaUsingFilebeatREADME.md) - - -## Troubleshooting - -If you encounter an error like the following when trying to connect to the database: - -``` -2024-07-04 21:15:48 ERROR jdbc:127 - Unable to connect to database. Tried 1 times {:message=>"Java::ComSapDbJdbcExceptions::SQLInvalidAuthorizationSpecExceptionSapDB: [10]: authentication failed", :exception=>Sequel::DatabaseConnectionError, :cause=>#, :backtrace=>["com.sap.db.jdbc.exceptions.SQLExceptionSapDB._newInstance(com/sap/db/jdbc/exceptions/SQLExceptionSapDB.java:183)", "com.sap.db.jdbc.exceptions.SQLExceptionSapDB.newInstance(com/sap/db/jdbc/exceptions/SQLExceptionSapDB.java:42)", -``` - -This error indicates an **authentication failure** when connecting to the database. It is often caused by issues with the provided credentials (username or password) or incorrect database configuration. - -## Steps to resolve: - -1. **Verify Database Credentials** - Check the **username** and **password** provided in your database connection configuration. Ensure that: - - The username and password are correct. - - The username has sufficient privileges to access the database. - - The password is correctly formatted, without any trailing spaces or special characters that may cause issues. - - - -3. **Test Credentials Manually** - Try connecting to the database manually using a database client or command line tool (e.g., SAP HANA Studio, `hdbsql`, or another SQL client). This can help verify that the credentials are valid and that the database is accessible. - -4. **Check for Locked or Expired Accounts** - If you have confirmed the credentials are correct, check if the account is locked or if the password has expired. Some databases automatically lock accounts after multiple failed login attempts. - - -### Limitations: - -1. SAP HANA auditing only supports error logs for authentication failures. -2. SAP HANA does not audit multiple line query properly. -3. Single line and multi-line comments in between the query are not supported. -4. The SAP HANA CSVTEXTFILE(audit target) does not audit the DB_Name. -5. SAP HANA with JDBC shows server ip as 0.0.0.0 -6. Duplicate records will be seen in load balancing. -7. SAPHANA SYSLOG does not natively support load balancing. \ No newline at end of file +[SapHanaOverConnectJdbc](../../docs/KafkaBasedUCs/SapHanaJDBCKafkaConnect.md) diff --git a/filter-plugin/logstash-filter-saphana-guardium/build.gradle b/filter-plugin/logstash-filter-saphana-guardium/build.gradle index 1efa269cf..e858ff17f 100644 --- a/filter-plugin/logstash-filter-saphana-guardium/build.gradle +++ b/filter-plugin/logstash-filter-saphana-guardium/build.gradle @@ -2,6 +2,30 @@ import java.nio.file.Files import static java.nio.file.StandardCopyOption.REPLACE_EXISTING apply plugin: 'java' + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + + dependencies { + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' + classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.1.0" + classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' + } +} + +def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" // =========================================================================== @@ -20,10 +44,10 @@ pluginInfo.pluginClass = "SapHanaGuardiumPluginFilter" pluginInfo.pluginName = "saphana_guardium_plugin_filter" // must match the @LogstashPlugin annotation in the main plugin class // =========================================================================== -sourceCompatibility = 1.8 -targetCompatibility = 1.8 +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 -def jacocoVersion = '0.8.4' +def jacocoVersion = '0.8.11' // minimumCoverage can be set by Travis ENV def minimumCoverageStr = System.getenv("MINIMUM_COVERAGE") ?: "50.0%" @@ -33,28 +57,17 @@ if (minimumCoverageStr.endsWith("%")) { def minimumCoverage = Float.valueOf(minimumCoverageStr) / 100 -buildscript { - repositories { - maven { - url "https://plugins.gradle.org/m2/" - } - mavenCentral() - jcenter() - } - - dependencies { - classpath 'com.github.jengelman.gradle.plugins:shadow:4.0.4' - classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.0.0-RC2" - classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' - } -} - -def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); -def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) repositories { - mavenCentral() + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } } tasks.register("vendor"){ @@ -65,14 +78,13 @@ tasks.register("vendor"){ File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") projectJarFile.mkdirs() Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) - validatePluginJar(projectJarFile, project.group) } } apply plugin: 'com.github.johnrengelman.shadow' shadowJar { - classifier = null + archiveClassifier = null } dependencies { @@ -80,6 +92,7 @@ dependencies { implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: versions.dependencies.log4jCore implementation 'org.apache.commons:commons-lang3:' + versions.dependencies.commonsLang implementation 'com.google.code.gson:gson:' + versions.dependencies.gson + implementation group: 'commons-beanutils', name: 'commons-beanutils', version: versions.dependencies.commonsBeanutils implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "build/libs/logstash-core-*.*.*.jar") implementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "common-*.*.*.jar") @@ -103,6 +116,17 @@ tasks.withType(JavaCompile) { options.encoding = 'UTF-8' } +test { + jvmArgs = [ + '--add-opens=java.base/java.lang=ALL-UNNAMED', + '--add-opens=java.base/java.util=ALL-UNNAMED', + '--add-opens=java.base/java.text=ALL-UNNAMED', + '--add-opens=java.base/java.lang.reflect=ALL-UNNAMED', + '--add-opens=java.base/sun.util.resources=ALL-UNNAMED', + '--add-opens=java.base/sun.util.cldr=ALL-UNNAMED' + ] +} + tasks.register("generateRubySupportFiles") { @@ -139,18 +163,17 @@ apply plugin: "org.barfuin.gradle.jacocolog" jacoco { toolVersion = "${jacocoVersion}" - reportsDir = file("$buildDir/reports/jacoco") } jacocoTestReport { // You will see "Report -> file://...." at the end of a JaCoCo build // If no output, run this first: ./gradlew test reports { - html.enabled true - xml.enabled true - csv.enabled true - html.destination file("${buildDir}/reports/jacoco") - csv.destination file("${buildDir}/reports/jacoco/all.csv") + html.required = true + xml.required = true + csv.required = true + html.outputLocation = file("${buildDir}/reports/jacoco") + csv.outputLocation = file("${buildDir}/reports/jacoco/all.csv") } executionData.from fileTree(dir: "${buildDir}/jacoco/", includes: [ '**/*.exec' diff --git a/filter-plugin/logstash-filter-saphana-guardium/gradle/wrapper/gradle-wrapper.properties b/filter-plugin/logstash-filter-saphana-guardium/gradle/wrapper/gradle-wrapper.properties index 60c76b340..ba9ccfe4c 100644 --- a/filter-plugin/logstash-filter-saphana-guardium/gradle/wrapper/gradle-wrapper.properties +++ b/filter-plugin/logstash-filter-saphana-guardium/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists \ No newline at end of file diff --git a/filter-plugin/logstash-filter-saphana-guardium/saphanaUsingFilebeatREADME.md b/filter-plugin/logstash-filter-saphana-guardium/saphanaUsingFilebeatREADME.md index e6c1e9f04..afd330aaa 100644 --- a/filter-plugin/logstash-filter-saphana-guardium/saphanaUsingFilebeatREADME.md +++ b/filter-plugin/logstash-filter-saphana-guardium/saphanaUsingFilebeatREADME.md @@ -125,7 +125,8 @@ https://www.elastic.co/guide/en/beats/filebeat/current/directory-layout.html * Locate "filebeat.inputs" in the filebeat.yml file, then add the following parameters. ``` filebeat.inputs: - - type: log + - type: filestream + - id: enabled: true paths: - /*.audit_trail.csv> diff --git a/filter-plugin/logstash-filter-saphana-guardium/src/main/java/com/ibm/guardium/saphana/Parser.java b/filter-plugin/logstash-filter-saphana-guardium/src/main/java/com/ibm/guardium/saphana/Parser.java index a59c5ea54..e2526be5f 100644 --- a/filter-plugin/logstash-filter-saphana-guardium/src/main/java/com/ibm/guardium/saphana/Parser.java +++ b/filter-plugin/logstash-filter-saphana-guardium/src/main/java/com/ibm/guardium/saphana/Parser.java @@ -6,7 +6,15 @@ import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import com.google.gson.JsonObject; -import com.ibm.guardium.universalconnector.commons.structures.*; +import com.ibm.guardium.universalconnector.commons.structures.Accessor; +import com.ibm.guardium.universalconnector.commons.structures.Construct; +import com.ibm.guardium.universalconnector.commons.structures.Data; +import com.ibm.guardium.universalconnector.commons.structures.ExceptionRecord; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import com.ibm.guardium.universalconnector.commons.structures.Sentence; +import com.ibm.guardium.universalconnector.commons.structures.SentenceObject; +import com.ibm.guardium.universalconnector.commons.structures.SessionLocator; +import com.ibm.guardium.universalconnector.commons.structures.Time; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/filter-plugin/logstash-filter-saphana-guardium/src/main/java/com/ibm/guardium/saphana/SapHanaGuardiumPluginFilter.java b/filter-plugin/logstash-filter-saphana-guardium/src/main/java/com/ibm/guardium/saphana/SapHanaGuardiumPluginFilter.java index 9f42023f1..2cc7eec24 100644 --- a/filter-plugin/logstash-filter-saphana-guardium/src/main/java/com/ibm/guardium/saphana/SapHanaGuardiumPluginFilter.java +++ b/filter-plugin/logstash-filter-saphana-guardium/src/main/java/com/ibm/guardium/saphana/SapHanaGuardiumPluginFilter.java @@ -9,7 +9,15 @@ import co.elastic.logstash.api.PluginConfigSpec; import com.google.gson.*; import com.ibm.guardium.universalconnector.commons.GuardConstants; -import com.ibm.guardium.universalconnector.commons.structures.*; +import com.ibm.guardium.universalconnector.commons.structures.Accessor; +import com.ibm.guardium.universalconnector.commons.structures.Construct; +import com.ibm.guardium.universalconnector.commons.structures.Data; +import com.ibm.guardium.universalconnector.commons.structures.ExceptionRecord; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import com.ibm.guardium.universalconnector.commons.structures.Sentence; +import com.ibm.guardium.universalconnector.commons.structures.SentenceObject; +import com.ibm.guardium.universalconnector.commons.structures.SessionLocator; +import com.ibm.guardium.universalconnector.commons.structures.Time; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/filter-plugin/logstash-filter-saphana-guardium/src/test/java/com/ibm/guardium/saphana/ParserTest.java b/filter-plugin/logstash-filter-saphana-guardium/src/test/java/com/ibm/guardium/saphana/ParserTest.java index cfcd17d0e..8113d30f1 100644 --- a/filter-plugin/logstash-filter-saphana-guardium/src/test/java/com/ibm/guardium/saphana/ParserTest.java +++ b/filter-plugin/logstash-filter-saphana-guardium/src/test/java/com/ibm/guardium/saphana/ParserTest.java @@ -11,7 +11,15 @@ import com.google.gson.JsonParser; import com.ibm.guardium.saphana.Constants; import com.ibm.guardium.saphana.Parser; -import com.ibm.guardium.universalconnector.commons.structures.*; +import com.ibm.guardium.universalconnector.commons.structures.Accessor; +import com.ibm.guardium.universalconnector.commons.structures.Construct; +import com.ibm.guardium.universalconnector.commons.structures.Data; +import com.ibm.guardium.universalconnector.commons.structures.ExceptionRecord; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import com.ibm.guardium.universalconnector.commons.structures.Sentence; +import com.ibm.guardium.universalconnector.commons.structures.SentenceObject; +import com.ibm.guardium.universalconnector.commons.structures.SessionLocator; +import com.ibm.guardium.universalconnector.commons.structures.Time; import com.ibm.guardium.universalconnector.commons.structures.Record; import org.junit.Assert; diff --git a/filter-plugin/logstash-filter-scylldb-guardium/build.gradle b/filter-plugin/logstash-filter-scylldb-guardium/build.gradle index e495f5cbe..62efd77b1 100644 --- a/filter-plugin/logstash-filter-scylldb-guardium/build.gradle +++ b/filter-plugin/logstash-filter-scylldb-guardium/build.gradle @@ -2,6 +2,31 @@ import java.nio.file.Files import static java.nio.file.StandardCopyOption.REPLACE_EXISTING apply plugin: 'java' + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + + dependencies { + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' + //classpath 'org.owasp:dependency-check-gradle:8.3.1' + //classpath "com.github.spotbugs.snom:spotbugs-gradle-plugin:4.7.8" + classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' + } +} + +def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + apply plugin: 'jacoco' apply plugin: 'eclipse' apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" @@ -22,36 +47,25 @@ pluginInfo.pluginClass = "ScyllaDbGuardiumFilter" pluginInfo.pluginName = "scylladb_guardium_filter" // must match the @LogstashPlugin annotation in the main plugin class // =========================================================================== -sourceCompatibility = 1.8 -targetCompatibility = 1.8 -buildscript { - repositories { - mavenCentral() - jcenter() - maven { - url "https://plugins.gradle.org/m2/" - } - } - - dependencies { - classpath 'com.github.jengelman.gradle.plugins:shadow:6.1.0' - //classpath 'org.owasp:dependency-check-gradle:8.3.1' - //classpath "com.github.spotbugs.snom:spotbugs-gradle-plugin:4.7.8" - classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' - } -} -def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); -def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 repositories { - mavenCentral() + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } } apply plugin: "com.github.johnrengelman.shadow" //apply plugin: 'org.owasp.dependencycheck' shadowJar { - classifier = null + archiveClassifier = null transform(com.github.jengelman.gradle.plugins.shadow.transformers.Log4j2PluginsCacheFileTransformer) } @@ -61,6 +75,7 @@ dependencies { implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: versions.dependencies.log4jCore implementation 'org.apache.commons:commons-lang3:' + versions.dependencies.commonsLang implementation 'com.google.code.gson:gson:' + versions.dependencies.gson + implementation group: 'commons-beanutils', name: 'commons-beanutils', version: versions.dependencies.commonsBeanutils testImplementation group: 'org.mockito', name: 'mockito-all', version: versions.dependencies.mockitoAll testImplementation 'org.junit.jupiter:junit-jupiter:' + versions.dependencies.junitJupiter implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "build/libs/logstash-core-*.*.*.jar") @@ -83,7 +98,6 @@ tasks.withType(JavaCompile) { /*spotbugs { ignoreFailures = true showStackTraces = false - reportsDir = file("$buildDir/spotbugs") } spotbugsMain { @@ -98,8 +112,8 @@ spotbugsMain { } tasks.withType(com.github.spotbugs.snom.SpotBugsTask) { reports { - xml.enabled = false - html.enabled = true + xml.required = false + html.required = true } }*/ @@ -120,7 +134,6 @@ task vendor(dependsOn: shadowJar) { File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") projectJarFile.mkdirs() Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) - validatePluginJar(projectJarFile, project.group) } } @@ -151,8 +164,8 @@ tasks.register("gem"){ jacocoTestReport { reports { - xml.enabled true - html.enabled true + xml.required = true + html.required = true } afterEvaluate { // (optional) : to exclude classes / packages from coverage diff --git a/filter-plugin/logstash-filter-scylldb-guardium/gradle/wrapper/gradle-wrapper.properties b/filter-plugin/logstash-filter-scylldb-guardium/gradle/wrapper/gradle-wrapper.properties index aa991fcea..81aa1c044 100644 --- a/filter-plugin/logstash-filter-scylldb-guardium/gradle/wrapper/gradle-wrapper.properties +++ b/filter-plugin/logstash-filter-scylldb-guardium/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/filter-plugin/logstash-filter-singlestore-guardium/CHANGELOG.md b/filter-plugin/logstash-filter-singlestore-guardium/CHANGELOG.md new file mode 100755 index 000000000..0d87cac45 --- /dev/null +++ b/filter-plugin/logstash-filter-singlestore-guardium/CHANGELOG.md @@ -0,0 +1,12 @@ +# Changelog +Notable changes will be documented in this file. + +## [1.0.1] +- Change DB protocol from "Singlestore Native Audit" to "SinglestoreDB" + +## [1.0.0] + +### Added +- Initial release of SingleStore UC + + diff --git a/filter-plugin/logstash-filter-singlestore-guardium/LICENSE b/filter-plugin/logstash-filter-singlestore-guardium/LICENSE new file mode 100755 index 000000000..8f71f43fe --- /dev/null +++ b/filter-plugin/logstash-filter-singlestore-guardium/LICENSE @@ -0,0 +1,202 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/filter-plugin/logstash-filter-singlestore-guardium/README.md b/filter-plugin/logstash-filter-singlestore-guardium/README.md new file mode 100755 index 000000000..9d65d6406 --- /dev/null +++ b/filter-plugin/logstash-filter-singlestore-guardium/README.md @@ -0,0 +1,134 @@ +# SingleStore-Guardium Logstash filter plug-in + +### Meet SingleStore +* Tested versions: 8.7.12 (memsql version) +* Environment: On-premise, Iaas +* Supported inputs: Filebeat (push) +* Supported Guardium versions: + * Guardium Data Protection: 12.0 and above + +This is a [Logstash](https://github.com/elastic/logstash) filter plug-in for the universal connector that is featured in IBM Security Guardium. It parses events and messages from the SingleStore audit log into a [Guardium record](https://github.com/IBM/universal-connectors/blob/main/common/src/main/java/com/ibm/guardium/universalconnector/commons/structures/Record.java) instance (which is a standard structure made out of several parts). The information is then sent over to Guardium. Guardium records include the accessor (the person who tried to access the data), the session, data, and exceptions. If there are no errors, the data contains details about the query "construct". The construct details the main action (verb) and collections (objects) involved. + +Currently, this plug-in will work only with IBM Security Guardium Data Protection, and not Guardium Insights. + +The plug-in is free and open-source (Apache 2.0). It can be used as a starting point to develop additional filter plug-ins for Guardium universal connector. + +## 1. Enabling the audit logs + +### Procedure +1. Enable the audit logs. + + To capture parameterized SQL statements, use: + ```text + sdb-admin update-config --all --key "auditlog_level" --value "ALL-QUERIES" + ``` + + To capture plaintext SQL statements, use: + ```text + sdb-admin update-config --all --key "auditlog_level" --value "ALL-QUERIES-PLAINTEXT" + ``` + + For more information about audit logging levels, see [SingleStore audit logging levels](https://docs.singlestore.com/db/v9.0/security/audit-logging/audit-logging-levels/). + +2. Restart the nodes. + ```text + sdb-admin restart-node --all + ``` + +3. Verify if the configuration is saved and enabled. + ```text + SHOW GLOBAL VARIABLES LIKE 'audit%'; + ``` + +## 2. Viewing the audit logs configuration +Use the following command to retrieve the log files that are stored in the auditlogsdir variable. + ```text + SHOW GLOBAL VARIABLES LIKE 'audit%'; + ``` + +## 3. Configuring Filebeat to push logs to Guardium +1. To install Filebeat on your system, refer to the [Filebeat quick start: installation and configuration](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-installation-configuration.html#installation) topic. + + +2. Configuring Filebeat + To use Logstash to process additional data collected by Filebeat, configure Filebeat to use Logstash. To do so, modify the `filebeat.yml` file. + **Note:** Search for the `filebeat.yml` file in the filebeat installation directory. You can also refer [Directory layout](https://www.elastic.co/guide/en/beats/filebeat/current/directory-layout.html) to search the `filebeat.yml` file. + + a. Update the filebeat.inputs section with the following parameters. + ```text + filebeat.inputs: + - type: filestream + - id: + enabled: true + paths : - /path/to/query.log + parsers: + - multiline: + type: pattern + pattern: '^\d+,' + negate: true + match: after + tags : ["singlestore"] + ``` + ```text + # If filestream is not supported, use the log input as shown below: + filebeat.inputs: + type: log + # Unique ID among all inputs, an ID is required. + id: + + # Change to true to enable this input configuration. + enabled: true + allow_deprecated_use: true + # Paths that should be crawled and fetched. Glob based paths. + paths: + - /singlestoredata/*/auditlogs/*.log + + multiline.type: pattern + multiline.pattern: '^\d+,' + multiline.negate: true + multiline.match: after + + tags: ["singlestore"] + ``` + **Note:** Add the tags to uniquely identify the SingleStore events from the rest. + + b. Configuring the output section. +   1.In the output section, disable the Elasticsearch output by commenting it out. +   2. Enable Logstash output by uncommenting the Logstash section. For more information, see [Configure the Logstash output +](https://www.elastic.co/guide/en/beats/filebeat/current/logstash-output.html#logstash-output).

+ **Note:** The hosts option specifies the Logstash server and the port where Logstash is configured to listen for incoming Beats connections. + For example: + ```text + output.logstash: + hosts: [":"] + ``` + +### Limitations +• Source Program is not part of the SingleStore logs. +• Client IP can only be retrieved in login / logout actions. +• Queries with SQL errors are included in the `Full SQL` report and are not displayed in `SQL Errors`. + +**Note:** For details on configuring Filebeat connection over SSL, refer [Configuring Filebeat to push logs to Guardium](https://github.com/IBM/universal-connectors/blob/main/input-plugin/logstash-input-beats/README.md#configuring-filebeat-to-push-logs-to-guardium). + + +## 4. Configuring the SingleStore filters in Guardium + +The Guardium universal connector is the Guardium entry point for native audit logs. The universal connector identifies and parses received events, and then converts them to a standard Guardium format. The output of the universal connector is forwarded to the Guardium sniffer on the collector, for policy and auditing enforcements. Configure Guardium to read the native audit logs by customizing the SingleStore template. + +### Before you begin + +• Configure the [policies](/docs/#policies). + +• You must have permission for the S-Tap Management role. The admin user includes this role, by default. + +# Procedure + +1. On the collector, go to Setup > Tools and Views > Configure Universal Connector. +2. First enable the Universal Guardium connector, if it is disabled already. +3. Click Upload File and select the offline [logstash-filter-singlestoredb_guardium_filter.zip](SingleStoreOverFilebeatPackage/logstash-filter-singlestoredb_guardium_filter.zip) plug-in. After it is uploaded, click OK. +4. Click the Plus sign to open the Connector Configuration dialog box. +5. Type a name in the Connector name field. +6. Update the input section to add the details from the [singlestoreFilebeat.conf](./singleStoreFilebeat.conf) file input section, omitting the keyword "input{" at the beginning and its corresponding "}" at the end. +7. Update the filter section to add the details from the [singlestoreFilebeat.conf](./singleStoreFilebeat.conf) file filter section, omitting the keyword "filter{" at the beginning and its corresponding "}" at the end. +8. The "type" fields should match in the input and the filter configuration section. This field should be unique for every individual connector added +9. Click Save. Guardium validates the new connector and displays it in the Configure Universal Connector page. diff --git a/filter-plugin/logstash-filter-singlestore-guardium/SingleStoreOverFilebeatPackage/SingleStoreDB/filter.conf b/filter-plugin/logstash-filter-singlestore-guardium/SingleStoreOverFilebeatPackage/SingleStoreDB/filter.conf new file mode 100644 index 000000000..cf4ff8e6c --- /dev/null +++ b/filter-plugin/logstash-filter-singlestore-guardium/SingleStoreOverFilebeatPackage/SingleStoreDB/filter.conf @@ -0,0 +1,21 @@ +filter { + if [type] == "filebeat" and "singlestore" in [tags] { + + mutate { + split => { "message" => "," } + } + + if [message][0] == "0" or [message][7] == "" or [message][7] == "distributed" or [message][8] == "[unknown]" { + drop { } + } + + mutate { + remove_tag => ["beats_input_codec_plain_applied"] + join => { "message" => "," } + add_field => { "serverIP" => "%{[host][ip][0]}" } + add_field => { "serverHostname" => "%{[host][name]}" } + } + + singlestoredb_guardium_filter{} + } +} \ No newline at end of file diff --git a/filter-plugin/logstash-filter-singlestore-guardium/SingleStoreOverFilebeatPackage/SingleStoreDB/manifest.json b/filter-plugin/logstash-filter-singlestore-guardium/SingleStoreOverFilebeatPackage/SingleStoreDB/manifest.json new file mode 100644 index 000000000..9bec4f8c0 --- /dev/null +++ b/filter-plugin/logstash-filter-singlestore-guardium/SingleStoreOverFilebeatPackage/SingleStoreDB/manifest.json @@ -0,0 +1,14 @@ +{ + "name": "Guardium_SingleStore_filter", + "alias": "SingleStore", + "type": "filter", + "pipeline_type":"push", + "plugin_version": "1.0.0", + "datasourceTypes": [{"type":"SingleStore","supportedVersions": ["8.7.1"]}], + "supported_input_plugins": ["Filebeat_input"], + "developer": "IBM", + "license": "Apache 2.0", + "description": "This package provides the data needed in order to support SingleStore DB in Guardium Universal Connector.", + "configuration_notes": "", + "documentation_path": "" +} diff --git a/filter-plugin/logstash-filter-singlestore-guardium/SingleStoreOverFilebeatPackage/logstash-filter-singlestoredb_guardium_filter.zip b/filter-plugin/logstash-filter-singlestore-guardium/SingleStoreOverFilebeatPackage/logstash-filter-singlestoredb_guardium_filter.zip new file mode 100644 index 000000000..0d4d3b24c Binary files /dev/null and b/filter-plugin/logstash-filter-singlestore-guardium/SingleStoreOverFilebeatPackage/logstash-filter-singlestoredb_guardium_filter.zip differ diff --git a/filter-plugin/logstash-filter-singlestore-guardium/VERSION b/filter-plugin/logstash-filter-singlestore-guardium/VERSION new file mode 100755 index 000000000..7f207341d --- /dev/null +++ b/filter-plugin/logstash-filter-singlestore-guardium/VERSION @@ -0,0 +1 @@ +1.0.1 \ No newline at end of file diff --git a/filter-plugin/logstash-filter-singlestore-guardium/build.gradle b/filter-plugin/logstash-filter-singlestore-guardium/build.gradle new file mode 100755 index 000000000..3aa426930 --- /dev/null +++ b/filter-plugin/logstash-filter-singlestore-guardium/build.gradle @@ -0,0 +1,214 @@ +import java.nio.file.Files +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING + +apply plugin: 'java' + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + + dependencies { + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' + classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.1.0" + classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' + } +} + +def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + +apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" + +// =========================================================================== +// plugin info +// =========================================================================== +group 'com.ibm.guardium.singlestore' // must match the package of the main plugin class +version "${file("VERSION").text.trim()}" // read from required VERSION file +description = "SingleStore-Guardium filter plugin" +pluginInfo.licenses = ['Apache-2.0'] // list of SPDX license IDs +pluginInfo.longDescription = "This gem is a Logstash SingleStoreDB filter plugin required to be installed as part of IBM Security Guardium, Guardium Universal connector configuration. This gem is not a stand-alone program." +pluginInfo.authors = ['IBM'] +pluginInfo.email = [] +pluginInfo.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html" +pluginInfo.pluginType = "filter" +pluginInfo.pluginClass = "SingleStoredbGuardiumFilter" +pluginInfo.pluginName = "singlestoredb_guardium_filter" // must match the @LogstashPlugin annotation in the main plugin class +// =========================================================================== + + +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 + +def jacocoVersion = '0.8.11' +// minimumCoverage can be set by Travis ENV +def minimumCoverageStr = System.getenv("MINIMUM_COVERAGE") ?: "50.0%" +if (minimumCoverageStr.endsWith("%")) { + minimumCoverageStr = minimumCoverageStr.substring(0, minimumCoverageStr.length() - 1) +} +def minimumCoverage = Float.valueOf(minimumCoverageStr) / 100 + + + +repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } +} + +tasks.register("vendor"){ + dependsOn shadowJar + doLast { + String vendorPathPrefix = "vendor/jar-dependencies" + String projectGroupPath = project.group.replaceAll('\\.', '/') + File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") + projectJarFile.mkdirs() + Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) + } +} + +apply plugin: 'com.github.johnrengelman.shadow' + +shadowJar { + archiveClassifier = null +} + +dependencies { + implementation group: 'commons-validator', name: 'commons-validator', version: versions.dependencies.commonsValidator + implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: versions.dependencies.log4jCore + implementation 'org.apache.commons:commons-lang3:' + versions.dependencies.commonsLang + implementation 'com.google.code.gson:gson:' + versions.dependencies.gson + implementation group: 'commons-beanutils', name: 'commons-beanutils', version: versions.dependencies.commonsBeanutils + implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "build/libs/logstash-core-*.*.*.jar") + implementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "guardium-universalconnector-commons-*.*.*.jar") + + testImplementation 'junit:junit:' + versions.dependencies.junit + testImplementation 'org.jruby:jruby-complete:' + versions.dependencies.jrubyComplete + + testImplementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "guardium-universalconnector-commons-*.*.*.jar") +} + +clean { + delete "${projectDir}/Gemfile" + delete "${projectDir}/" + pluginInfo.pluginFullName() + ".gemspec" + delete "${projectDir}/lib/" + delete "${projectDir}/vendor/" + new FileNameFinder().getFileNames(projectDir.toString(), pluginInfo.pluginFullName() + "-*.*.*.gem").each { filename -> + delete filename + } +} + +tasks.withType(JavaCompile) { + options.encoding = 'UTF-8' +} + +test { + jvmArgs = [ + '--add-opens=java.base/java.lang=ALL-UNNAMED', + '--add-opens=java.base/java.util=ALL-UNNAMED', + '--add-opens=java.base/java.text=ALL-UNNAMED', + '--add-opens=java.base/java.lang.reflect=ALL-UNNAMED', + '--add-opens=java.base/sun.util.resources=ALL-UNNAMED', + '--add-opens=java.base/sun.util.cldr=ALL-UNNAMED' + ] +} + + + +tasks.register("generateRubySupportFiles") { + doLast { + generateRubySupportFilesForPlugin(project.description, project.group, version) + } +} + +tasks.register("removeObsoleteJars") { + doLast { + new FileNameFinder().getFileNames( + projectDir.toString(), + "vendor/**/" + pluginInfo.pluginFullName() + "*.jar", + "vendor/**/" + pluginInfo.pluginFullName() + "-" + version + ".jar").each { f -> + delete f + } + } +} + +tasks.register("gem"){ + dependsOn = [downloadAndInstallJRuby, removeObsoleteJars, vendor, generateRubySupportFiles] + doLast { + buildGem(projectDir, buildDir, pluginInfo.pluginFullName() + ".gemspec") + } +} + +tasks.register("copyDependencyLibs", Copy){ + into "dependenciesLib" + from configurations.compileClasspath + from configurations.runtimeClasspath + from configurations.testCompileClasspath + from configurations.testRuntimeClasspath +} + +apply plugin: 'jacoco' +//apply plugin: 'org.barfuin.gradle.jacocolog' version '2.0.0' +apply plugin: "org.barfuin.gradle.jacocolog" +// ------------------------------------ +// JaCoCo is a code coverage tool +// ------------------------------------ +jacoco { + toolVersion = "${jacocoVersion}" +} +jacocoTestReport { + // You will see "Report -> file://...." at the end of a JaCoCo build + // If no output, run this first: ./gradlew test + reports { + html.required = true + xml.required = true + csv.required = true + html.outputLocation = file("${buildDir}/reports/jacoco") + csv.outputLocation = file("${buildDir}/reports/jacoco/all.csv") + } + executionData.from fileTree(dir: "${buildDir}/jacoco/", includes: [ + '**/*.exec' + ]) + afterEvaluate { + // objective is to test TicketingService class + classDirectories.setFrom(files(classDirectories.files.collect { + fileTree(dir: it, exclude: []) + })) + } + doLast { + println "Report -> file://${buildDir}/reports/jacoco/index.html" + } +} +test.finalizedBy jacocoTestReport +jacocoTestCoverageVerification { + violationRules { + rule { + limit { + minimum = minimumCoverage + } + } + } + executionData.from fileTree(dir: "${buildDir}/jacoco/", includes: [ + '**/*.exec' + ]) + afterEvaluate { + // objective is to test TicketingService class + classDirectories.setFrom(files(classDirectories.files.collect { + fileTree(dir: it, exclude: []) + })) + } +} +project.tasks.check.dependsOn(jacocoTestCoverageVerification, jacocoTestReport) \ No newline at end of file diff --git a/filter-plugin/logstash-filter-singlestore-guardium/filter-test-generator.conf b/filter-plugin/logstash-filter-singlestore-guardium/filter-test-generator.conf new file mode 100755 index 000000000..adaa86283 --- /dev/null +++ b/filter-plugin/logstash-filter-singlestore-guardium/filter-test-generator.conf @@ -0,0 +1,39 @@ +input { + generator { lines => [ + '202548,2024-07-02 12:59:31.847,UTC,53b9ae806c1c:3306,agg,USER_LOGIN,99997,,localhost,root@%,password,SUCCESS', + '202549,2024-07-02 12:59:31.867,UTC,53b9ae806c1c:3306,agg,1,99997,,vector_db,temp_1_46531_1,4745626564866103597,SELECT * FROM (SELECT TABLE_NAME as `Tables_in_vector_db`\, TABLE_TYPE as Table_type FROM information_schema.tables where TABLE_SCHEMA =\'vector_db\' AND CAST(FLAGS AS UNSIGNED INTEGER) & 1 = 0) sub WHERE `table_type` IN (\'BASE TABLE\'\, \'TEMPORARY TABLE\')', + '202550,2024-07-02 12:59:39.567,UTC,53b9ae806c1c:3306,agg,1,99980,root,vector_db,temp_1_46525_17,13738294579036747261,SELECT 1', + '202551,2024-07-02 12:59:39.839,UTC,53b9ae806c1c:3306,agg,1,99996,root,vector_db,temp_1_46528_12,13738294579036747261,SELECT 1', + '202552,2024-07-02 12:59:40.041,UTC,53b9ae806c1c:3306,agg,1,99996,root,vector_db,temp_1_46528_13,13343714012096239519,select * from vectors', + '202553,2024-07-02 12:59:40.231,UTC,53b9ae806c1c:3306,agg,1,99980,root,vector_db,temp_1_46525_18,15577690842814585134,SELECT @@memsql_version', + '202554,2024-07-02 12:59:40.691,UTC,53b9ae806c1c:3306,agg,1,99980,root,vector_db,temp_1_46525_19,12471391318107449066,select @@table_name_case_sensitivity' + ] + type => "test" + count => 1 + } +} + +filter { + if [type] == "filebeat" and "SingleStore" in [tags] { + + mutate { + split => { "message" => "," } + } + + if [message][0] == "0" or [message][7] == "distributed" or [message][8] == "[unknown]" { + drop { } + } + + mutate { + join => { "message" => "," } + add_field => { "serverIP" => "%{[host][ip][0]}" } + add_field => { "serverHostname" => "%{[host][name]}" } + } + + singlestoredb_guardium_filter{} + } +} + +output { + stdout { codec => rubydebug } +} diff --git a/filter-plugin/logstash-filter-singlestore-guardium/gradle/wrapper/gradle-wrapper.jar b/filter-plugin/logstash-filter-singlestore-guardium/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 000000000..1948b9074 Binary files /dev/null and b/filter-plugin/logstash-filter-singlestore-guardium/gradle/wrapper/gradle-wrapper.jar differ diff --git a/filter-plugin/logstash-filter-singlestore-guardium/gradle/wrapper/gradle-wrapper.properties b/filter-plugin/logstash-filter-singlestore-guardium/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 000000000..81aa1c044 --- /dev/null +++ b/filter-plugin/logstash-filter-singlestore-guardium/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,5 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/filter-plugin/logstash-filter-singlestore-guardium/gradlew b/filter-plugin/logstash-filter-singlestore-guardium/gradlew new file mode 100755 index 000000000..cccdd3d51 --- /dev/null +++ b/filter-plugin/logstash-filter-singlestore-guardium/gradlew @@ -0,0 +1,172 @@ +#!/usr/bin/env sh + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS="" + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin, switch paths to Windows format before running java +if $cygwin ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=$((i+1)) + done + case $i in + (0) set -- ;; + (1) set -- "$args0" ;; + (2) set -- "$args0" "$args1" ;; + (3) set -- "$args0" "$args1" "$args2" ;; + (4) set -- "$args0" "$args1" "$args2" "$args3" ;; + (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=$(save "$@") + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong +if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then + cd "$(dirname "$0")" +fi + +exec "$JAVACMD" "$@" diff --git a/filter-plugin/logstash-filter-singlestore-guardium/gradlew.bat b/filter-plugin/logstash-filter-singlestore-guardium/gradlew.bat new file mode 100755 index 000000000..f9553162f --- /dev/null +++ b/filter-plugin/logstash-filter-singlestore-guardium/gradlew.bat @@ -0,0 +1,84 @@ +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS= + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto init + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto init + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:init +@rem Get command-line arguments, handling Windows variants + +if not "%OS%" == "Windows_NT" goto win9xME_args + +:win9xME_args +@rem Slurp the command line arguments. +set CMD_LINE_ARGS= +set _SKIP=2 + +:win9xME_args_slurp +if "x%~1" == "x" goto execute + +set CMD_LINE_ARGS=%* + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/filter-plugin/logstash-filter-singlestore-guardium/gradlew.unix b/filter-plugin/logstash-filter-singlestore-guardium/gradlew.unix new file mode 100755 index 000000000..cccdd3d51 --- /dev/null +++ b/filter-plugin/logstash-filter-singlestore-guardium/gradlew.unix @@ -0,0 +1,172 @@ +#!/usr/bin/env sh + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS="" + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin, switch paths to Windows format before running java +if $cygwin ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=$((i+1)) + done + case $i in + (0) set -- ;; + (1) set -- "$args0" ;; + (2) set -- "$args0" "$args1" ;; + (3) set -- "$args0" "$args1" "$args2" ;; + (4) set -- "$args0" "$args1" "$args2" "$args3" ;; + (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=$(save "$@") + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong +if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then + cd "$(dirname "$0")" +fi + +exec "$JAVACMD" "$@" diff --git a/filter-plugin/logstash-filter-singlestore-guardium/singleStoreFilebeat.conf b/filter-plugin/logstash-filter-singlestore-guardium/singleStoreFilebeat.conf new file mode 100755 index 000000000..1c93536ef --- /dev/null +++ b/filter-plugin/logstash-filter-singlestore-guardium/singleStoreFilebeat.conf @@ -0,0 +1,28 @@ +input { + beats { + port => + type => "filebeat" + } +} + +filter { + if [type] == "filebeat" and "singlestore" in [tags] { + + mutate { + split => { "message" => "," } + } + + if [message][0] == "0" or [message][7] == "" or [message][7] == "distributed" or [message][8] == "[unknown]" { + drop { } + } + + mutate { + remove_tag => ["beats_input_codec_plain_applied"] + join => { "message" => "," } + add_field => { "serverIP" => "%{[host][ip][0]}" } + add_field => { "serverHostname" => "%{[host][name]}" } + } + + singlestoredb_guardium_filter{} + } +} \ No newline at end of file diff --git a/filter-plugin/logstash-filter-singlestore-guardium/src/main/java/com/ibm/guardium/singlestore/Constants.java b/filter-plugin/logstash-filter-singlestore-guardium/src/main/java/com/ibm/guardium/singlestore/Constants.java new file mode 100644 index 000000000..b8854dab1 --- /dev/null +++ b/filter-plugin/logstash-filter-singlestore-guardium/src/main/java/com/ibm/guardium/singlestore/Constants.java @@ -0,0 +1,34 @@ +// +// Copyright 2020-2021 IBM Inc. All rights reserved +// SPDX-License-Identifier: Apache2.0 +// + +package com.ibm.guardium.singlestore; + +public class Constants { + + public static final String UNKNOWN_STRING = ""; + public static final String DB_PROTOCOL = "SinglestoreDB"; + public static final String NOT_AVAILABLE = "N.A."; + public static final String SERVER_TYPE_STRING = "SINGLESTORE"; + public static final String LOGSTASH_TAG_JSON_PARSE_ERROR = "_SingleStore_guardium_json_parse_error"; + public static final String SERVER_HOSTNAME = "serverHostName"; + public static final String SERVER_PORT = "serverPort"; + public static final String EXCEPTION_TYPE_LOGIN_FAILED_STRING = "LOGIN_FAILED"; + public static final String CLIENT_IP_VALUE = "0.0.0.0"; + public static final int CLIENT_PORT_VALUE = 0; + public static final String SERVER_IP_VALUE = "0.0.0.0"; + public static final int SERVER_PORT_VALUE = 0; + + // Input parameters + public static final String CLIENT_IP = "client_ip"; + public static final String SERVER_IP = "server_ip"; + public static final String TIMESTAMP = "ts"; + public static final String DB_USER = "dbuser"; + public static final String DB_NAME = "dbname"; + public static final String QUERY_STATEMENT = "queryStatement"; + public static final String MESSAGE = "message"; + public static final String TYPE = "graph"; + + public static final String LANGUAGE_MEMSQL_STRING = "MEMSQL"; +} diff --git a/filter-plugin/logstash-filter-singlestore-guardium/src/main/java/com/ibm/guardium/singlestore/Parser.java b/filter-plugin/logstash-filter-singlestore-guardium/src/main/java/com/ibm/guardium/singlestore/Parser.java new file mode 100644 index 000000000..864b69d80 --- /dev/null +++ b/filter-plugin/logstash-filter-singlestore-guardium/src/main/java/com/ibm/guardium/singlestore/Parser.java @@ -0,0 +1,375 @@ +// Copyright 2020-2021 IBM Inc. All rights reserved +// SPDX-License-Identifier: Apache2.0 +// + +package com.ibm.guardium.singlestore; + +import com.google.gson.JsonObject; +import com.ibm.guardium.universalconnector.commons.structures.Accessor; +import com.ibm.guardium.universalconnector.commons.structures.Construct; +import com.ibm.guardium.universalconnector.commons.structures.Data; +import com.ibm.guardium.universalconnector.commons.structures.ExceptionRecord; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import com.ibm.guardium.universalconnector.commons.structures.Sentence; +import com.ibm.guardium.universalconnector.commons.structures.SentenceObject; +import com.ibm.guardium.universalconnector.commons.structures.SessionLocator; +import com.ibm.guardium.universalconnector.commons.structures.Time; +import org.apache.commons.lang3.StringEscapeUtils; +import org.apache.commons.validator.routines.InetAddressValidator; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; +import java.time.format.TextStyle; +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class Parser { + + private static Logger log = LogManager.getLogger(Parser.class); + + private DateTimeFormatterBuilder dateTimeFormatterBuilder = new DateTimeFormatterBuilder() + .appendPattern("yyyy-MM-dd HH:mm:ss.SSS") + .optionalStart() + .appendLiteral(' ') + .optionalEnd() + .optionalStart() + .appendZoneText(TextStyle.SHORT) + .optionalEnd(); + + private DateTimeFormatter DATE_TIME_FORMATTER = dateTimeFormatterBuilder.toFormatter(); + + // ---------------------- Record---------------------------- + public Record parseRecord(final JsonObject data) { + try { + if (data != null) { + log.info("Start Parsing Record"); + Record record = getInitialRecord(data); + + // Get message and parse to map with null check + if (data.has("message") && !data.get("message").isJsonNull()) { + String message = data.get("message").getAsString(); + Map logMap = SingleStoreLogFormat.parseLog(message); + + // Check if EVENT_TYPE exists in the map + String eventType = logMap.get(SingleStoreLogFormat.EVENT_TYPE); + if (eventType != null && !SingleStoreLogFormat.USER_LOGIN.equals(eventType)) { + record.setData(parseData(data)); + } + } + + return record; + } else { + return null; + } + + } catch (Exception e) { + log.error("SingleStore filter: Error parsing Record {}", e.getMessage()); + return null; + } + } + + // ---------------------- ExceptionRecord----------------------------- + public Record parseExceptionRecord(final JsonObject data) { + try { + log.info("Start Parsing ExceptionRecord"); + + Record record = getInitialRecord(data); + + if (data.has("message") && !data.get("message").isJsonNull()) { + String message = data.get("message").getAsString(); + Map logMap = SingleStoreLogFormat.parseLog(message); + + ExceptionRecord exceptionRecord = new ExceptionRecord(); + exceptionRecord.setExceptionTypeId(Constants.EXCEPTION_TYPE_LOGIN_FAILED_STRING); + + // Get login status with null check + String loginStatus = logMap.get(SingleStoreLogFormat.LOGIN_STATUS); + exceptionRecord.setDescription("Login Failed (" + (loginStatus != null ? loginStatus : "Unknown") + ")"); + exceptionRecord.setSqlString(Constants.UNKNOWN_STRING); + record.setException(exceptionRecord); + } + + return record; + } catch (Exception e) { + log.error("SingleStore filter: Error parsing ExceptionRecord {}", e.getMessage()); + return null; + } + } + + // ---------------------- Create Record with Time, session ....------------------- + public Record getInitialRecord(final JsonObject data) { + Record record = new Record(); + + record.setAppUserName(Constants.NOT_AVAILABLE); + + // DB Name + String dbName = Constants.NOT_AVAILABLE; + + if (data.has(Constants.DB_NAME) && !data.get(Constants.DB_NAME).isJsonNull()) { + dbName = data.get(Constants.DB_NAME).getAsString(); + } + record.setDbName(dbName); + + // Time + Time time = getTime(data); + if (time != null) { + record.setTime(time); + } + + // SessionLocator + record.setSessionLocator(parseSessionLocator(data)); + + // Accessor + record.setAccessor(parseAccessor(data, record.getDbName())); + + // SessionId + record.setSessionId(""); + + return record; + } + + // ---------------------- Timestamp------------------- + protected Time getTime(final JsonObject data) { + + String dateString = null; + if (data.has(Constants.TIMESTAMP) && !data.get(Constants.TIMESTAMP).isJsonNull()) { + dateString = data.get(Constants.TIMESTAMP).getAsString(); + } + + if (dateString != null && !dateString.isEmpty()) { + try { + ZonedDateTime date = ZonedDateTime.parse(dateString, DATE_TIME_FORMATTER); + long millis = date.toInstant().toEpochMilli(); + int minOffset = date.getOffset().getTotalSeconds() / 60; + return new Time(millis, minOffset, 0); + } catch (Exception e) { + log.error("Error parsing timestamp '{}': {}", dateString, e.getMessage()); + return null; + } + } else { + return new Time(0, 0, 0); + } + } + + // ---------------------- SessionLocator------------------------------ + protected SessionLocator parseSessionLocator(JsonObject data) { + SessionLocator sessionLocator = new SessionLocator(); + + // Get an `InetAddressValidator` + InetAddressValidator validator = InetAddressValidator.getInstance(); + + int clientPort = Constants.CLIENT_PORT_VALUE; + int serverPort = Constants.SERVER_PORT_VALUE; + String clientIpAdd = Constants.CLIENT_IP_VALUE; + String serverIpAdd = Constants.SERVER_IP_VALUE; + String clientIpv6Add = Constants.UNKNOWN_STRING; + String serverIpv6Add = Constants.UNKNOWN_STRING; + + if (data.has(Constants.CLIENT_IP) && !data.get(Constants.CLIENT_IP).isJsonNull()) { + String clientIp = data.get(Constants.CLIENT_IP).getAsString(); + + if (clientIp != null && !clientIp.isEmpty()) { + if (validator.isValidInet4Address(clientIp)) { + sessionLocator.setIpv6(false); + clientIpAdd = clientIp; + } else if (validator.isValidInet6Address(clientIp)) { + sessionLocator.setIpv6(true); + clientIpv6Add = clientIp; + } + } + } + + if (data.has(Constants.SERVER_IP) && !data.get(Constants.SERVER_IP).isJsonNull()) { + String serverIp = data.get(Constants.SERVER_IP).getAsString(); + if (serverIp != null && !serverIp.isEmpty()) { + if (validator.isValidInet4Address(serverIp)) { + serverIpAdd = serverIp; + } else if (validator.isValidInet6Address(serverIp)) { + if (!sessionLocator.isIpv6()) { + sessionLocator.setIpv6(true); + serverIpv6Add = serverIp; + } + } + } + } + + if (data.has(Constants.SERVER_PORT) && !data.get(Constants.SERVER_PORT).isJsonNull()) { + try { + serverPort = Integer.parseInt(data.get(Constants.SERVER_PORT).getAsString()); + } catch (NumberFormatException e) { + log.error("Error parsing server port: {}", e.getMessage()); + } + } + + sessionLocator.setClientIp(clientIpAdd); + sessionLocator.setClientPort(clientPort); + sessionLocator.setServerIp(serverIpAdd); + sessionLocator.setServerPort(serverPort); + + sessionLocator.setClientIpv6(clientIpv6Add); + sessionLocator.setServerIpv6(serverIpv6Add); + + return sessionLocator; + } + + // ---------------------- Accessor-------------------------------------- + protected Accessor parseAccessor(JsonObject data, String dbName) { + Accessor accessor = new Accessor(); + + accessor.setDbProtocol(Constants.DB_PROTOCOL); + accessor.setSourceProgram(Constants.UNKNOWN_STRING); + accessor.setServerType(Constants.SERVER_TYPE_STRING); + + String dbUser = Constants.NOT_AVAILABLE; + if (data.has(Constants.DB_USER) && !data.get(Constants.DB_USER).isJsonNull()) { + dbUser = data.get(Constants.DB_USER).getAsString(); + } + accessor.setDbUser(dbUser); + + // GRD-114546: Set to N.A. to fix multiple S-Taps + accessor.setServerHostName(Constants.NOT_AVAILABLE); + + accessor.setLanguage(Constants.LANGUAGE_MEMSQL_STRING); + accessor.setDataType(Accessor.DATA_TYPE_GUARDIUM_SHOULD_PARSE_SQL); + + accessor.setClient_mac(Constants.UNKNOWN_STRING); + accessor.setClientHostName(Constants.UNKNOWN_STRING); + accessor.setClientOs(Constants.UNKNOWN_STRING); + accessor.setCommProtocol(Constants.UNKNOWN_STRING); + accessor.setDbProtocolVersion(Constants.UNKNOWN_STRING); + accessor.setOsUser(Constants.UNKNOWN_STRING); + accessor.setServerDescription(Constants.UNKNOWN_STRING); + accessor.setServerOs(Constants.UNKNOWN_STRING); + accessor.setServiceName(dbName != null ? dbName : Constants.UNKNOWN_STRING); + + return accessor; + } + + protected Data parseData(JsonObject inputJSON) { + Data data = new Data(); + + String originalQuery = inputJSON.get(Constants.QUERY_STATEMENT).getAsString(); + + if (originalQuery != null && originalQuery.startsWith("\"") && originalQuery.endsWith("\"")) { + originalQuery = originalQuery.substring(1); + originalQuery = originalQuery.substring(0, originalQuery.length() - 1); + } + + if (originalQuery == null || originalQuery.trim().isEmpty()) { + originalQuery = Constants.UNKNOWN_STRING; + } + + String cleanedQuery = cleanQuery(originalQuery); + + // Use the cleaned query instead of the original query + data.setOriginalSqlCommand(cleanedQuery); + + return data; + } + + // ---------------------- Clean The Query -------- + protected String cleanQuery(String query) { + if (query == null || query.isEmpty()) { + return ""; + } + + try { + String regexObject = "OBJECT\\(\\)\\*/\\s*SELECT"; + String regex = "/\\*!\\d+\\s+"; + String commentRegex = "^\\s*/\\*.*?\\*/\\s*"; // More general pattern to match any comment at the beginning + String NESTED_SELECT_REGEX = "(?i)(?<=SELECT\\s)(.*?\\((?:[^()]*|\\((?:[^()]*|\\([^()]*\\))*\\))*\\))"; + + // Compile the pattern + Pattern pattern = Pattern.compile(regex, Pattern.CASE_INSENSITIVE); + + // Create a matcher for the input query + Matcher matcher = pattern.matcher(query); + + if (matcher.find()) { + query = matcher.replaceAll(""); + } + + // Compile the pattern + pattern = Pattern.compile(regexObject, Pattern.CASE_INSENSITIVE); + + // Create a matcher for the input query + matcher = pattern.matcher(query); + + if (matcher.find()) { + query = matcher.replaceAll("SELECT"); + } + + // Compile the pattern for SQL comments at the beginning of queries + pattern = Pattern.compile(commentRegex, Pattern.CASE_INSENSITIVE | Pattern.DOTALL); + + // Create a matcher for the input query + matcher = pattern.matcher(query); + + // Only remove the comment if it exists, otherwise leave the query unchanged + if (matcher.find()) { + query = matcher.replaceAll(""); + } + + query = query.replace("`", ""); + + if (query.length() >= 6 && query.toUpperCase().startsWith("SELECT")) { + if (hasNestedSelectInFields(query)) { + int indexMainFrom = findMainFromIndexInQuery(query); + if (indexMainFrom > 0) { + query = query.substring(0, 6) + " * " + query.substring(indexMainFrom); + } + } + } + } catch (Exception e) { + log.error("Error cleaning query: {}", e.getMessage()); + } + + return query; + } + + protected int findMainFromIndexInQuery(String query) { + if (query == null || query.isEmpty()) { + return -1; + } + + String lowerQuery = query.toLowerCase(); + int openParenCount = 0; + + for (int i = 0; i < lowerQuery.length(); i++) { + char c = lowerQuery.charAt(i); + + if (c == '(') { + openParenCount++; + } else if (c == ')') { + openParenCount--; + } else if (i + 4 <= lowerQuery.length() && lowerQuery.startsWith("from", i)) { + // Check if 'from' is not within parentheses + if (openParenCount == 0) { + return i; + } + } + } + + return -1; // No valid FROM found + } + + protected boolean hasNestedSelectInFields(String query) { + if (query == null || query.isEmpty()) { + return false; + } + + try { + // Regex pattern to find nested SELECT statements within parentheses in the field part + Pattern nestedSelectPattern = Pattern.compile("\\(\\s*select\\s+.*?\\)", Pattern.CASE_INSENSITIVE | Pattern.DOTALL); + Matcher matcher = nestedSelectPattern.matcher(query); + return matcher.find(); + } catch (Exception e) { + log.error("Error checking for nested SELECT: {}", e.getMessage()); + return false; + } + } +} \ No newline at end of file diff --git a/filter-plugin/logstash-filter-singlestore-guardium/src/main/java/com/ibm/guardium/singlestore/SingleStoreLogFormat.java b/filter-plugin/logstash-filter-singlestore-guardium/src/main/java/com/ibm/guardium/singlestore/SingleStoreLogFormat.java new file mode 100644 index 000000000..e2fddea67 --- /dev/null +++ b/filter-plugin/logstash-filter-singlestore-guardium/src/main/java/com/ibm/guardium/singlestore/SingleStoreLogFormat.java @@ -0,0 +1,111 @@ +// Copyright 2020-2021 IBM Inc. All rights reserved +// SPDX-License-Identifier: Apache2.0 +// + +package com.ibm.guardium.singlestore; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +/** + * Utility class for parsing SingleStore log messages into key-value pairs + * instead of relying on array indices. + */ +public class SingleStoreLogFormat { + private static final Logger log = LogManager.getLogger(SingleStoreLogFormat.class); + + // Field names + public static final String EVENT_TYPE = "eventType"; + public static final String DB_USER = "dbUser"; + public static final String DB_NAME = "dbName"; + public static final String SERVER_PORT = "serverPort"; + public static final String QUERY = "query"; + public static final String LOGIN_STATUS = "loginStatus"; + public static final String TIMESTAMP_DATE = "timestampDate"; + public static final String TIMESTAMP_TIME = "timestampTime"; + + // Known values + public static final String USER_LOGIN = "USER_LOGIN"; + public static final String LOGIN_FAILURE = "FAILURE: Access denied"; + + /** + * Parse a SingleStore log message into a map of key-value pairs + * + * @param logMessage The raw log message string + * @return Map containing parsed fields + */ + public static Map parseLog(String logMessage) { + Map logMap = new HashMap<>(); + + if (logMessage == null || logMessage.isEmpty()) { + log.warn("Empty log message provided to parser"); + return logMap; + } + + try { + // Split on commas that are NOT preceded by backslash + // This handles escaped commas (\,) in the log format + String[] values = logMessage.split("(? 1 && portParts[1] != null) { + logMap.put(SERVER_PORT, portParts[1]); + } else { + logMap.put(SERVER_PORT, ""); + } + } else { + logMap.put(SERVER_PORT, ""); + } + + // Query is now correctly in values[11] - no need to reassemble + // Backslashes are preserved as-is from the original log + if (values.length > 11 && values[11] != null) { + logMap.put(QUERY, values[11]); + } else { + logMap.put(QUERY, ""); + } + + // For login events, extract status + if (USER_LOGIN.equals(logMap.get(EVENT_TYPE)) && values.length > 11 && values[11] != null) { + logMap.put(LOGIN_STATUS, values[11]); + } else { + logMap.put(LOGIN_STATUS, ""); + } + + return logMap; + } catch (Exception e) { + log.error("Error parsing log message: {}", e.getMessage()); + return logMap; + } + } +} + diff --git a/filter-plugin/logstash-filter-singlestore-guardium/src/main/java/com/ibm/guardium/singlestore/SingleStoredbGuardiumFilter.java b/filter-plugin/logstash-filter-singlestore-guardium/src/main/java/com/ibm/guardium/singlestore/SingleStoredbGuardiumFilter.java new file mode 100644 index 000000000..9784f6589 --- /dev/null +++ b/filter-plugin/logstash-filter-singlestore-guardium/src/main/java/com/ibm/guardium/singlestore/SingleStoredbGuardiumFilter.java @@ -0,0 +1,271 @@ +// Copyright 2020-2021 IBM Inc. All rights reserved +// SPDX-License-Identifier: Apache2.0 +// + +package com.ibm.guardium.singlestore; + +import java.io.File; +import java.util.Collection; +import java.util.Collections; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.core.LoggerContext; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonObject; +import com.ibm.guardium.universalconnector.commons.GuardConstants; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import co.elastic.logstash.api.Configuration; +import co.elastic.logstash.api.Context; +import co.elastic.logstash.api.Event; +import co.elastic.logstash.api.Filter; +import co.elastic.logstash.api.FilterMatchListener; +import co.elastic.logstash.api.LogstashPlugin; +import co.elastic.logstash.api.PluginConfigSpec; + +@LogstashPlugin(name = "singlestoredb_guardium_filter") +public class SingleStoredbGuardiumFilter implements Filter { + + public static final String LOG42_CONF = "log4j2uc.properties"; + + static { + try { + String uc_etc = System.getenv("UC_ETC"); + if (uc_etc != null) { + LoggerContext context = (LoggerContext) LogManager.getContext(false); + File file = new File(uc_etc + File.separator + LOG42_CONF); + context.setConfigLocation(file.toURI()); + } + } catch (Exception e) { + System.err.println("Failed to load log4j configuration: " + e.getMessage()); + e.printStackTrace(); + } + } + + final private String id; + public static final PluginConfigSpec SOURCE_CONFIG = PluginConfigSpec.stringSetting("source", "message"); + final private static Logger log = LogManager.getLogger(SingleStoredbGuardiumFilter.class); + + public SingleStoredbGuardiumFilter(String id, Configuration config, Context context) { + this.id = id; + } + + @Override + public Collection> configSchema() { + // should return a list of all configuration options for this plugin + return Collections.singletonList(SOURCE_CONFIG); + } + + @Override + public String getId() { + return this.id; + } + + @Override + public Collection filter(Collection events, FilterMatchListener matchListener) { + if (events == null || matchListener == null) { + log.error("Null events or matchListener provided to filter"); + return events; + } + + for (Event e : events) { + if (e == null) { + continue; + } + + if (log.isDebugEnabled()) { + log.debug("Event Now: {}", e.getData()); + } + + try { + + Record record; + + final GsonBuilder builder = new GsonBuilder(); + final Gson gson = builder.create(); + builder.serializeNulls(); + + // Check if message field exists + if (e.getField("message") == null) { + log.error("Event missing 'message' field"); + e.tag(Constants.LOGSTASH_TAG_JSON_PARSE_ERROR); + continue; + } + + getParsedEvent(e.getField("message").toString(), e); + + JsonObject inputData = inputData(e); + if (inputData == null) { + log.error("Failed to create inputData from event"); + e.tag(Constants.LOGSTASH_TAG_JSON_PARSE_ERROR); + continue; + } + + Parser parser = new Parser(); + if (isFailedLogin(inputData)) { + record = parser.parseExceptionRecord(inputData); + } else { + record = parser.parseRecord(inputData); + } + + if (record != null) { + e.setField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME, gson.toJson(record)); + matchListener.filterMatched(e); + log.debug("==========>Final JSON to be send to Guardium: {}", gson.toJson(record)); + } else { + log.error("Failed to create record from event"); + e.tag(Constants.LOGSTASH_TAG_JSON_PARSE_ERROR); + } + } catch (Exception exception) { + log.error("SingleStore filter: Error parsing SingleStore event", exception); + e.tag(Constants.LOGSTASH_TAG_JSON_PARSE_ERROR); + } + } + return events; + } + + private boolean isFailedLogin(JsonObject inputData) { + if (inputData == null || !inputData.has("message") || inputData.get("message").isJsonNull()) { + return false; + } + + try { + String message = inputData.get("message").getAsString(); + Map logMap = SingleStoreLogFormat.parseLog(message); + + String eventType = logMap.get(SingleStoreLogFormat.EVENT_TYPE); + String loginStatus = logMap.get(SingleStoreLogFormat.LOGIN_STATUS); + + return eventType != null && loginStatus != null && + SingleStoreLogFormat.USER_LOGIN.equals(eventType) && + SingleStoreLogFormat.LOGIN_FAILURE.equals(loginStatus); + } catch (Exception e) { + log.error("Error checking for failed login: {}", e.getMessage()); + return false; + } + } + + private JsonObject inputData(Event e) { + if (e == null) { + log.error("Null event provided to inputData"); + return null; + } + + JsonObject data = new JsonObject(); + + try { + addPropertyIfNotNull(data, Constants.TIMESTAMP, e, Constants.TIMESTAMP, true); + addPropertyIfNotNull(data, Constants.CLIENT_IP, e, Constants.CLIENT_IP, false); + addPropertyIfNotNull(data, Constants.SERVER_IP, e, Constants.SERVER_IP, false); + addPropertyIfNotNull(data, Constants.SERVER_HOSTNAME, e, Constants.SERVER_HOSTNAME, false); + addPropertyIfNotNull(data, Constants.SERVER_PORT, e, Constants.SERVER_PORT, false); + addPropertyIfNotNull(data, Constants.DB_USER, e, Constants.DB_USER, false); + addPropertyIfNotNull(data, Constants.DB_NAME, e, Constants.DB_NAME, false); + addPropertyIfNotNull(data, Constants.QUERY_STATEMENT, e, Constants.QUERY_STATEMENT, false); + addPropertyIfNotNull(data, Constants.MESSAGE, e, Constants.MESSAGE, false); + } catch (Exception ex) { + log.error("Error creating inputData: {}", ex.getMessage()); + } + + return data; + } + + // Helper method to add property if not null + private void addPropertyIfNotNull(JsonObject data, String propertyName, Event e, String fieldName, boolean isTimestamp) { + if (data == null || propertyName == null || e == null || fieldName == null) { + return; + } + + try { + Object fieldValue = e.getField(fieldName); + if (fieldValue != null) { + String value = fieldValue.toString(); + if (value != null && !value.isEmpty()) { + if (isTimestamp) { + value = value.replace("UTC", "Z"); + } + data.addProperty(propertyName, value); + } + } + } catch (Exception ex) { + log.error("Error adding property {}: {}", propertyName, ex.getMessage()); + } + } + + public static void getParsedEvent(String logEvent, Event event) { + if (logEvent == null || event == null) { + log.error("Null logEvent or event provided to getParsedEvent"); + return; + } + + try { + Map logMap = SingleStoreLogFormat.parseLog(logEvent); + + if (logMap.isEmpty()) { + log.error("Failed to parse log event: {}", logEvent); + return; + } + + event.setField("message", logEvent); + + // Set fields with null checks + String dbName = logMap.get(SingleStoreLogFormat.DB_NAME); + if (dbName != null) { + event.setField(Constants.DB_NAME, dbName); + } else { + event.setField(Constants.DB_NAME, ""); + } + + event.setField(Constants.CLIENT_IP, Constants.CLIENT_IP_VALUE); + + String eventType = logMap.get(SingleStoreLogFormat.EVENT_TYPE); + if (eventType != null && SingleStoreLogFormat.USER_LOGIN.equals(eventType)) { + if (dbName != null) { + event.setField(Constants.CLIENT_IP, dbName); + } + event.setField(Constants.DB_NAME, ""); + } + + if (event.getData() != null) { + if (event.getData().containsKey("serverIP")) { + Object serverIP = event.getField("serverIP"); + if (serverIP != null) { + event.setField(Constants.SERVER_IP, serverIP.toString()); + } + } + + if (event.getData().containsKey("serverHostname")) { + Object serverHostname = event.getField("serverHostname"); + if (serverHostname != null) { + event.setField(Constants.SERVER_HOSTNAME, serverHostname.toString()); + } + } + } + + String serverPort = logMap.get(SingleStoreLogFormat.SERVER_PORT); + if (serverPort != null) { + event.setField(Constants.SERVER_PORT, serverPort); + } + + String timestampDate = logMap.get(SingleStoreLogFormat.TIMESTAMP_DATE); + String timestampTime = logMap.get(SingleStoreLogFormat.TIMESTAMP_TIME); + if (timestampDate != null && timestampTime != null) { + event.setField(Constants.TIMESTAMP, timestampDate + " " + timestampTime); + } + + String dbUser = logMap.get(SingleStoreLogFormat.DB_USER); + if (dbUser != null) { + event.setField(Constants.DB_USER, dbUser); + } + + String query = logMap.get(SingleStoreLogFormat.QUERY); + if (query != null) { + event.setField(Constants.QUERY_STATEMENT, query); + } + } catch (Exception e) { + log.error("getParsedEvent Function {}", e.getMessage()); + } + } +} \ No newline at end of file diff --git a/filter-plugin/logstash-filter-singlestore-guardium/src/test/java/com/ibm/guardium/singlestore/ParserTest.java b/filter-plugin/logstash-filter-singlestore-guardium/src/test/java/com/ibm/guardium/singlestore/ParserTest.java new file mode 100644 index 000000000..e94859c91 --- /dev/null +++ b/filter-plugin/logstash-filter-singlestore-guardium/src/test/java/com/ibm/guardium/singlestore/ParserTest.java @@ -0,0 +1,294 @@ +// +// Copyright 2020-2021 IBM Inc. All rights reserved +// SPDX-License-Identifier: Apache2.0 +// + +package com.ibm.guardium.singlestore; + +import co.elastic.logstash.api.Event; +import com.google.gson.JsonObject; +import com.ibm.guardium.universalconnector.commons.structures.Accessor; +import com.ibm.guardium.universalconnector.commons.structures.Construct; +import com.ibm.guardium.universalconnector.commons.structures.Data; +import com.ibm.guardium.universalconnector.commons.structures.ExceptionRecord; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import com.ibm.guardium.universalconnector.commons.structures.Sentence; +import com.ibm.guardium.universalconnector.commons.structures.SentenceObject; +import com.ibm.guardium.universalconnector.commons.structures.SessionLocator; +import com.ibm.guardium.universalconnector.commons.structures.Time; +import com.ibm.guardium.universalconnector.commons.structures.Record; + +import java.util.ArrayList; + +import org.junit.Assert; +import org.junit.Test; + +public class ParserTest { + + String singlestoreString = "133855,2024-06-24 07:16:16.901,UTC,53b9ae806c1c:3306,agg,1,100000,root,test_db,,1308432953418920798,CREATE DATABASE uc_test_db"; + Parser parser = new Parser(); + + @Test + public void testParseSessionLocator() { + Event e = getParsedEvent(singlestoreString); + JsonObject inputData = inputData(e); + final SessionLocator result = parser.parseSessionLocator(inputData); + + SessionLocator expectedSessionLocator = new SessionLocator(); + expectedSessionLocator.setClientIp(Constants.CLIENT_IP_VALUE); + expectedSessionLocator.setClientPort(Constants.CLIENT_PORT_VALUE); + expectedSessionLocator.setServerIp(Constants.SERVER_IP_VALUE); + expectedSessionLocator.setServerPort(3306); + + Assert.assertEquals(expectedSessionLocator.getClientIp(), result.getClientIp()); + Assert.assertEquals(expectedSessionLocator.getClientPort(), result.getClientPort()); + Assert.assertEquals(expectedSessionLocator.getServerIp(), result.getServerIp()); + Assert.assertEquals(expectedSessionLocator.getServerPort(), result.getServerPort()); + } + + @Test + public void testParseAccessor() { + Event e = getParsedEvent(singlestoreString); + JsonObject inputData = inputData(e); + final Accessor result = parser.parseAccessor(inputData, "test_db"); + + Accessor expectedAccessor = new Accessor(); + expectedAccessor.setDbUser("root"); + expectedAccessor.setServiceName("test_db"); + expectedAccessor.setLanguage("MEMSQL"); + expectedAccessor.setDataType("TEXT"); + expectedAccessor.setServerType(Constants.SERVER_TYPE_STRING); + expectedAccessor.setDbProtocol(Constants.DB_PROTOCOL); + + Assert.assertEquals(expectedAccessor.getDbUser(), result.getDbUser()); + Assert.assertEquals(expectedAccessor.getServiceName(), result.getServiceName()); + Assert.assertEquals(expectedAccessor.getLanguage(), result.getLanguage()); + Assert.assertEquals(expectedAccessor.getDataType(), result.getDataType()); + Assert.assertEquals(expectedAccessor.getServerType(), result.getServerType()); + Assert.assertEquals(expectedAccessor.getDbProtocol(), result.getDbProtocol()); + } + + @Test + public void testGetTime() { + Event e = getParsedEvent(singlestoreString); + JsonObject inputData = inputData(e); + final Time time = parser.getTime(inputData); + + Assert.assertEquals(0, time.getMinDst()); + Assert.assertEquals(0, time.getMinOffsetFromGMT()); + Assert.assertEquals(1719213376901L, time.getTimstamp()); + } + + @Test + public void testParseData() { + Event e = getParsedEvent(singlestoreString); + JsonObject inputData = inputData(e); + Data result = parser.parseData(inputData); + + Data expectedData = new Data(); + expectedData.setOriginalSqlCommand("CREATE DATABASE uc_test_db"); + + Assert.assertEquals(expectedData.getOriginalSqlCommand(), result.getOriginalSqlCommand()); + Assert.assertNull(result.getConstruct()); + } + + @Test + public void testParseExceptionRecord() { + String failedLoginString = "1546,2025-05-07 11:07:20.523,PDT,53b9ae806c1c:3308,leaf,USER_LOGIN,99985,root,localhost,,password,FAILURE: Access denied"; + Event e = getParsedEvent(failedLoginString); + JsonObject inputData = inputData(e); + Record result = parser.parseExceptionRecord(inputData); + + Record expectedRecord = new Record(); + ExceptionRecord expectedException = new ExceptionRecord(); + expectedException.setExceptionTypeId("LOGIN_FAILED"); + expectedException.setDescription("Login Failed (FAILURE: Access denied)"); + expectedRecord.setException(expectedException); + + Assert.assertEquals(expectedRecord.getException().getDescription(), result.getException().getDescription()); + Assert.assertEquals(expectedRecord.getException().getExceptionTypeId(), result.getException().getExceptionTypeId()); + } + + @Test + public void testParseRecord() { + Event e = getParsedEvent(singlestoreString); + JsonObject inputData = inputData(e); + Record result = parser.parseRecord(inputData); + + Assert.assertNotNull(result); + Assert.assertEquals("root", result.getAccessor().getDbUser()); + Assert.assertEquals("test_db", result.getDbName()); + Assert.assertNotNull(result.getTime()); + Assert.assertNotNull(result.getSessionLocator()); + Assert.assertEquals("CREATE DATABASE uc_test_db", result.getData().getOriginalSqlCommand()); + } + + @Test + public void testParseRecordWithLoginEvent() { + String loginString = "1546,2025-05-07 11:07:20.523,PDT,53b9ae806c1c:3308,leaf,USER_LOGIN,99985,root,localhost,,password,SUCCESS"; + Event e = getParsedEvent(loginString); + JsonObject inputData = inputData(e); + Record result = parser.parseRecord(inputData); + + Assert.assertNotNull(result); + Assert.assertNull(result.getData()); // Login events should not have data + } + + @Test + public void testGetInitialRecord() { + Event e = getParsedEvent(singlestoreString); + JsonObject inputData = inputData(e); + Record result = parser.getInitialRecord(inputData); + + Assert.assertNotNull(result); + Assert.assertEquals("N.A.", result.getAppUserName()); + Assert.assertEquals("test_db", result.getDbName()); + Assert.assertEquals("", result.getSessionId()); + Assert.assertNotNull(result.getTime()); + Assert.assertNotNull(result.getSessionLocator()); + Assert.assertNotNull(result.getAccessor()); + } + + @Test + public void testParseSessionLocatorWithIPv6Client() { + Event e = getParsedEvent(singlestoreString); + JsonObject inputData = inputData(e); + inputData.addProperty(Constants.CLIENT_IP, "2001:0db8:85a3:0000:0000:8a2e:0370:7334"); + + SessionLocator result = parser.parseSessionLocator(inputData); + + Assert.assertTrue(result.isIpv6()); + Assert.assertEquals("2001:0db8:85a3:0000:0000:8a2e:0370:7334", result.getClientIpv6()); + } + + @Test + public void testParseSessionLocatorWithIPv6Server() { + Event e = getParsedEvent(singlestoreString); + JsonObject inputData = inputData(e); + + // Only server is IPv6, client is IPv4 + inputData.addProperty(Constants.SERVER_IP, "2001:0db8:85a3:0000:0000:8a2e:0370:7335"); + SessionLocator result = parser.parseSessionLocator(inputData); + + // When only server is IPv6, isIpv6 should be true and serverIpv6 should be set + Assert.assertTrue(result.isIpv6()); + Assert.assertEquals("2001:0db8:85a3:0000:0000:8a2e:0370:7335", result.getServerIpv6()); + } + + @Test + public void testParseDataWithNullQuery() { + Event e = getParsedEvent(singlestoreString); + JsonObject inputData = inputData(e); + inputData.remove(Constants.QUERY_STATEMENT); + + try { + Data result = parser.parseData(inputData); + // Should handle null gracefully + Assert.assertNotNull(result); + } catch (Exception ex) { + // Expected to handle null + } + } + + @Test + public void testParseDataWithQuotedQuery() { + Event e = getParsedEvent(singlestoreString); + JsonObject inputData = inputData(e); + inputData.addProperty(Constants.QUERY_STATEMENT, "\"SELECT * FROM table\""); + Data result = parser.parseData(inputData); + Assert.assertNotNull(result); + Assert.assertEquals("SELECT * FROM table", result.getOriginalSqlCommand()); + } + + @Test + public void testGetTimeWithInvalidTimestamp() { + JsonObject inputData = new JsonObject(); + inputData.addProperty(Constants.TIMESTAMP, "invalid-timestamp"); + Time result = parser.getTime(inputData); + Assert.assertNull(result); // Should return null for invalid timestamp + } + + @Test + public void testGetTimeWithNullTimestamp() { + JsonObject inputData = new JsonObject(); + Time result = parser.getTime(inputData); + Assert.assertNotNull(result); + Assert.assertEquals(0, result.getTimstamp()); + } + +// ----------------------------------- --------------------------------------------------- + + private JsonObject inputData(Event e) { + JsonObject data = new JsonObject(); + + if (e.getField(Constants.CLIENT_IP).toString() != null && !e.getField(Constants.CLIENT_IP).toString().isEmpty()) { + data.addProperty(Constants.CLIENT_IP, e.getField(Constants.CLIENT_IP).toString()); + } + if (e.getField(Constants.SERVER_IP).toString() != null && !e.getField(Constants.SERVER_IP).toString().isEmpty()) { + data.addProperty(Constants.SERVER_IP, e.getField(Constants.SERVER_IP).toString()); + } + if (e.getField(Constants.SERVER_HOSTNAME).toString() != null && e.getField(Constants.SERVER_HOSTNAME).toString().isEmpty()) { + data.addProperty(Constants.SERVER_HOSTNAME, e.getField(Constants.SERVER_HOSTNAME).toString()); + } + if (e.getField(Constants.TIMESTAMP).toString() != null && !e.getField(Constants.TIMESTAMP).toString().isEmpty()) { + data.addProperty(Constants.TIMESTAMP, e.getField(Constants.TIMESTAMP).toString()); + } + if (e.getField(Constants.SERVER_PORT).toString() != null && !e.getField(Constants.SERVER_PORT).toString().isEmpty()) { + data.addProperty(Constants.SERVER_PORT, e.getField(Constants.SERVER_PORT).toString()); + } + if (e.getField(Constants.DB_USER).toString() != null && !e.getField(Constants.DB_USER).toString().isEmpty()) { + data.addProperty(Constants.DB_USER, e.getField(Constants.DB_USER).toString()); + } + if (e.getField(Constants.DB_NAME).toString() != null && !e.getField(Constants.DB_NAME).toString().isEmpty()) { + data.addProperty(Constants.DB_NAME, e.getField(Constants.DB_NAME).toString()); + } + if (e.getField(Constants.QUERY_STATEMENT).toString() != null && !e.getField(Constants.QUERY_STATEMENT).toString().isEmpty()) { + data.addProperty(Constants.QUERY_STATEMENT, e.getField(Constants.QUERY_STATEMENT).toString()); + } + if (e.getField(Constants.MESSAGE).toString() != null && !e.getField(Constants.MESSAGE).toString().isEmpty()) { + data.addProperty(Constants.MESSAGE, e.getField(Constants.MESSAGE).toString()); + } + return data; + } + + + public static Event getParsedEvent(String logEvent) { + try { + + String[] values = logEvent.split(","); + + String server_hostname = values[3].split(":")[0]; + String server_port = values[3].split(":")[1]; + + Event e = new org.logstash.Event(); + + e.setField("message", logEvent); + e.setField(Constants.CLIENT_IP, "0.0.0.0"); + e.setField(Constants.DB_NAME, values[8]); + + if (values[5].equals("USER_LOGIN")) { + e.setField(Constants.CLIENT_IP, values[8]); + e.setField(Constants.DB_NAME, ""); + } + e.setField(Constants.SERVER_IP, "0.0.0.0"); + e.setField(Constants.SERVER_HOSTNAME, server_hostname); + e.setField(Constants.SERVER_PORT, server_port); + e.setField(Constants.TIMESTAMP, values[1] + " " + values[2]); + e.setField(Constants.DB_USER, values[7]); + + if (values.length > 12) { + for (int i = 12; i < values.length; i++) { + values[11] = values[11] + "," + values[i]; + } + + } + e.setField(Constants.QUERY_STATEMENT, values[11]); + return e; + } catch (Exception e) { + System.out.println(e.getMessage()); + return null; + } + } +} + + diff --git a/filter-plugin/logstash-filter-singlestore-guardium/src/test/java/com/ibm/guardium/singlestore/SingleStoredbGuardiumFilterTest.java b/filter-plugin/logstash-filter-singlestore-guardium/src/test/java/com/ibm/guardium/singlestore/SingleStoredbGuardiumFilterTest.java new file mode 100644 index 000000000..4b188abd2 --- /dev/null +++ b/filter-plugin/logstash-filter-singlestore-guardium/src/test/java/com/ibm/guardium/singlestore/SingleStoredbGuardiumFilterTest.java @@ -0,0 +1,91 @@ +// +// Copyright 2020-2021 IBM Inc. All rights reserved +// SPDX-License-Identifier: Apache2.0 +// + +package com.ibm.guardium.singlestore; + +import co.elastic.logstash.api.Context; +import co.elastic.logstash.api.Event; +import co.elastic.logstash.api.FilterMatchListener; +import org.apache.commons.lang3.StringEscapeUtils; +import org.junit.Assert; +import org.junit.Test; +import org.logstash.plugins.ContextImpl; + +import com.ibm.guardium.universalconnector.commons.GuardConstants; + +import java.util.*; +import java.util.concurrent.atomic.AtomicInteger; + +public class SingleStoredbGuardiumFilterTest { + + + /** + * To feed Guardium universal connector, a "GuardRecord" fields must exist. + * Filter should add field "GuardRecord" to the Event, which Universal connector then inserts into Guardium. + */ + @Test + public void testFieldGuardRecord_singlestoredb() { + System.out.println(" ================================"); + System.out.println("========================================||testFieldGuardRecord_singlestoredb||========================================"); + System.out.println(" ================================"); + + String singlestoreString = "133855,2024-06-24 07:16:16.901,UTC,53b9ae806c1c:3306,agg,1,100000,root,vector_db,,1308432953418920798,CREATE DATABASE `uc_vector_db`"; + //String singlestoreString= "133865,2024-06-24 07:17:27.705,UTC,53b9ae806c1c:3306,agg,1,100000,root,uc_vector_db,,9259882968147880232,CREATE TABLE uc_vector_table (AND SCHEMA_NAME LIKE 'uc_vector_db' AND TABLE_SCHEMA LIKE 'uc_vector_db' WHERE t.table_schema like 'uc_vector_db'"; + //String singlestoreString = "21596,2024-06-10 10:42:43.236,UTC,53b9ae806c1c:3306,agg,1,99995,root,information_schema,temp_1_5015_0,694459544968825767,SELECT IP_ADDR\\, PORT\\, MEMSQL_DIR\\, DISK_USED_B\\ FROM information_schema.mv_disk_usage JOIN information_schema.mv_nodes\\ ON mv_disk_usage.NODE_ID = mv_nodes.ID\\ ORDER BY IP_ADDR\\, PORT"; + //String singlestoreString="21622,2024-06-10 10:47:43.235,UTC,53b9ae806c1c:3306,agg,USER_LOGIN,99995,root,localhost,root@%,password,SUCCESS"; + //String singlestoreString="152310,2024-06-26 13:54:39.421,UTC,53b9ae806c1c:3306,agg,USER_LOGIN,99997,ayoub,10.3.220.131,,authentication_none,FAILURE: Access denied"; + //String singlestoreString="133899,2024-06-24 07:21:24.876,UTC,53b9ae806c1c:3306,agg,1,100000,amine,uc_vector_db,temp_1_31066_6,5822492499834975395,update uc_vector_table set id=999 where id=122"; + //String singlestoreString="133894,2024-06-24 07:20:50.842,UTC,53b9ae806c1c:3306,agg,1,100000,root,uc_vector_db,temp_1_31066_5,17413739249988095469,UPDATE employees SET salary = 80000.00 WHERE id = 1"; + //String singlestoreString="133894,2024-06-24 07:20:50.842,UTC,53b9ae806c1c:3306,agg,1,100000,root,uc_vector_db,temp_1_31066_5,17413739249988095469,DELETE FROM employees WHERE id = 3"; + //String singlestoreString="133894,2024-06-24 07:20:50.842,UTC,53b9ae806c1c:3306,agg,1,100000,root,uc_vector_db,temp_1_31066_5,17413739249988095469,INSERT INTO high_earners (id, name, salary) SELECT id, name, salary FROM employees WHERE salary > 100000"; + //String singlestoreString="133894,2024-06-24 07:20:50.842,UTC,53b9ae806c1c:3306,agg,1,100000,root,uc_vector_db,temp_1_31066_5,17413739249988095469,INSERT INTO employees VALUES (1, 'John Doe', 'Engineer', 75000.00)"; + //String singlestoreString="21881,2024-06-10 11:30:23.310,UTC,53b9ae806c1c:3306,agg,1,99995,root,information_schema,temp_1_5079_5,14799535652165267194,select * from information_schema.schemata where schema_name not in ('cluster', 'memsql') order by schema_name"; + //String singlestoreString="21869,2024-06-10 11:30:19.227,UTC,53b9ae806c1c:3306,agg,1,99995,root,information_schema,temp_1_5078_1,11951698597271110224,SELECT @@max_allowed_packet\\, @@aggregator_id"; + //String singlestoreString="133872,2024-06-24 07:17:33.287,UTC,53b9ae806c1c:3306,agg,1,99993,distributed,uc_vector_db,,6408909967012182963,SHOW TABLE STATUS FROM `uc_vector_db`"; + //String singlestoreString="21866,2024-06-10 11:29:58.831,UTC,53b9ae806c1c:3306,agg,1,99993,distributed,information_schema,temp_1_5077_3,4952243803313514788,/*!90621 OBJECT()*/ SELECT WITH(binary_serialization=1\\, binary_serialization_internal=1) `_MV_QUERY_PROSPECTIVE_HISTOGRAMS`.`DATABASE_NAME` AS `DATABASE_NAME`\\, `_MV_QUERY_PROSPECTIVE_HISTOGRAMS`.`TABLE_NAME` AS `TABLE_NAME`\\, `_MV_QUERY_PROSPECTIVE_HISTOGRAMS`.`COLUMN_NAME` AS `COLUMN_NAME`\\, `_MV_QUERY_PROSPECTIVE_HISTOGRAMS`.`JSON_KEY` AS `JSON_KEY`\\, MAX(`_MV_QUERY_PROSPECTIVE_HISTOGRAMS`.`USAGE_COUNT`) AS `USAGE_COUNT`\\, `_MV_QUERY_PROSPECTIVE_HISTOGRAMS`.`ACTIVITY_NAME` AS `ACTIVITY_NAME` FROM `_MV_QUERY_PROSPECTIVE_HISTOGRAMS` as `_MV_QUERY_PROSPECTIVE_HISTOGRAMS` WITH (disable_encoded_joins = TRUE) WHERE (NOT 0) GROUP BY 6\\, 1\\, 2\\, 3\\, 4 /*!90623 OPTION(NO_QUERY_REWRITE=1\\, INTERPRETER_MODE=INTERPRET_FIRST)*/"; + //String singlestoreString="21866,2024-06-10 11:29:58.831,UTC,53b9ae806c1c:3306,agg,1,99993,distributed,information_schema,temp_1_5077_3,4952243803313514788,SELECT first_name, (SELECT department_name FROM departments WHERE departments.department_id = employees.department_id) AS department_name FROM employees"; + //String singlestoreString="21866,2024-06-10 11:29:58.831,UTC,53b9ae806c1c:3306,agg,1,99993,distributed,information_schema,temp_1_5077_3,4952243803313514788,SELECT first_name FROM employees WHERE department_id IN (SELECT department_id FROM departments WHERE location_id>1500)"; + + + Context context = new ContextImpl(null, null); + SingleStoredbGuardiumFilter filter = new SingleStoredbGuardiumFilter("test-id", null, context); + + Event e = ParserTest.getParsedEvent(singlestoreString); + + TestMatchListener matchListener = new TestMatchListener(); + + if (e != null) { + e.setField(Constants.SERVER_IP, "1.1.1.1"); + e.setField(Constants.SERVER_HOSTNAME, "singlestore.server.com"); + Collection results = filter.filter(Collections.singletonList(e), matchListener); + Assert.assertEquals(1, results.size()); + Assert.assertNotNull(e.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME)); + } + + } + + @Test + public void TestCleanQuery() { + String originalQuery = "/* ApplicationName=DBeaver 25.1.0 - SQLEditor */ SELECT * FROM customers WHERE city = \"Pune\""; + String expectedCleanedQuery = "SELECT * FROM customers WHERE city = \"Pune\""; + + Parser parser = new Parser(); + String actualCleanedQuery = parser.cleanQuery(originalQuery); + + Assert.assertEquals(expectedCleanedQuery, actualCleanedQuery); + } + +} + +class TestMatchListener implements FilterMatchListener { + + private final AtomicInteger matchCount = new AtomicInteger(0); + + @Override + public void filterMatched(Event event) { + matchCount.incrementAndGet(); + } +} + diff --git a/filter-plugin/logstash-filter-snowflake-guardium/CHANGELOG.md b/filter-plugin/logstash-filter-snowflake-guardium/CHANGELOG.md new file mode 100644 index 000000000..c906d477a --- /dev/null +++ b/filter-plugin/logstash-filter-snowflake-guardium/CHANGELOG.md @@ -0,0 +1,8 @@ +# Changelog +Notable changes will be documented in this file. + +## [1.0.3] +- GRD-124942: Log all broken events for snowflake, uc audit error and uc parser error + +## [1.0.2] +- GRD-116966: Update Snowflake UC to support custom start time. \ No newline at end of file diff --git a/filter-plugin/logstash-filter-snowflake-guardium/KeyPairAuth_README.md b/filter-plugin/logstash-filter-snowflake-guardium/KeyPairAuth_README.md index b6db86185..b8079ff5f 100644 --- a/filter-plugin/logstash-filter-snowflake-guardium/KeyPairAuth_README.md +++ b/filter-plugin/logstash-filter-snowflake-guardium/KeyPairAuth_README.md @@ -11,12 +11,12 @@ The following steps are required to enable Key Pair authentication. 1. Generate a private key using the command, ```shell - openssl genrsa 2048 | openssl pkcs8 -topk8 -v2 des3 -inform PEM -out -nocrypt + openssl genrsa 2048 | openssl pkcs8 -topk8 -inform PEM -out -nocrypt ``` The key is created in PEM format. 2. From the created private key, use the following command to generate a public key. ```shell - openssl rsa -in rsa_key.p8 -pubout -out + openssl rsa -in rsa_key.p8 -pubout -out ``` The public key is created in the PEM format. 3. Now you need to assign the generated public key to the Snowflake user who is connecting to the database. diff --git a/filter-plugin/logstash-filter-snowflake-guardium/README.md b/filter-plugin/logstash-filter-snowflake-guardium/README.md index 5740559ec..ee6b27306 100644 --- a/filter-plugin/logstash-filter-snowflake-guardium/README.md +++ b/filter-plugin/logstash-filter-snowflake-guardium/README.md @@ -117,8 +117,8 @@ you can turn it off. To do this, set the value to `false` for the following two 2. You must have permission for the S-Tap Management role. The admin user includes this role by default. 3. Snowflake-Guardium Logstash filter plug-in is automatically available with Guardium Data Protection. versions 12.x, 11.4 with appliance bundle 11.0p490 or later or Guardium Data Protection version 11.5 with appliance bundle 11.0p540 or later releases. **Note:** For Guardium Data Protection version 11.4 without appliance bundle 11.0p490 or prior or Guardium Data Protection version 11.5 without appliance bundle 11.0p540 or prior, download the [logstash-offline-plugins-7.12.1.zip](SnowflakeOverJbdcPackage/Snowflake/logstash-offline-plugins-7.12.1.zip) plug-in. (Do not unzip the offline-package file throughout the procedure). -5. The plugin is tested with Snowflake JDBC driver v3.13.30. - Download the jdbc driver `jar` file from the maven repository [here.](https://repo1.maven.org/maven2/net/snowflake/snowflake-jdbc/3.13.30/snowflake-jdbc-3.13.30.jar) +5. The plugin is tested with Snowflake JDBC driver v3.13.30 and v3.16.0. + Download the jdbc driver `jar` file from the maven repository 3.13.30 from [here](https://repo1.maven.org/maven2/net/snowflake/snowflake-jdbc/3.13.30/snowflake-jdbc-3.13.30.jar), 3.16.0 from [here](https://repo1.maven.org/maven2/net/snowflake/snowflake-jdbc/3.16.0/snowflake-jdbc-3.16.0.jar). ### Procedure diff --git a/filter-plugin/logstash-filter-snowflake-guardium/VERSION b/filter-plugin/logstash-filter-snowflake-guardium/VERSION index 7dea76edb..21e8796a0 100644 --- a/filter-plugin/logstash-filter-snowflake-guardium/VERSION +++ b/filter-plugin/logstash-filter-snowflake-guardium/VERSION @@ -1 +1 @@ -1.0.1 +1.0.3 diff --git a/filter-plugin/logstash-filter-snowflake-guardium/build.gradle b/filter-plugin/logstash-filter-snowflake-guardium/build.gradle index b59919c0c..edd7d3579 100644 --- a/filter-plugin/logstash-filter-snowflake-guardium/build.gradle +++ b/filter-plugin/logstash-filter-snowflake-guardium/build.gradle @@ -2,6 +2,30 @@ import java.nio.file.Files import static java.nio.file.StandardCopyOption.REPLACE_EXISTING apply plugin: 'java' + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + + dependencies { + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' + classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.1.0" + classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' + } +} + +def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" // =========================================================================== @@ -20,10 +44,10 @@ pluginInfo.pluginName = "guardium_snowflake_filter" // must match the @Logs pluginInfo.email = "" // =========================================================================== -sourceCompatibility = 1.8 -targetCompatibility = 1.8 +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 -def jacocoVersion = '0.8.4' +def jacocoVersion = '0.8.11' // minimumCoverage can be set by Travis ENV def minimumCoverageStr = System.getenv("MINIMUM_COVERAGE") ?: "50.0%" if (minimumCoverageStr.endsWith("%")) { @@ -31,28 +55,17 @@ if (minimumCoverageStr.endsWith("%")) { } def minimumCoverage = Float.valueOf(minimumCoverageStr) / 100 -buildscript { - repositories { - maven { - url "https://plugins.gradle.org/m2/" - } - mavenCentral() - jcenter() - } - - dependencies { - classpath 'com.github.jengelman.gradle.plugins:shadow:4.0.4' - classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.0.0-RC2" - classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' - } -} - -def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); -def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) repositories { - mavenCentral() + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } } tasks.register("vendor"){ @@ -63,14 +76,13 @@ tasks.register("vendor"){ File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") projectJarFile.mkdirs() Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) - validatePluginJar(projectJarFile, project.group) } } apply plugin: 'com.github.johnrengelman.shadow' shadowJar { - classifier = null + archiveClassifier = null } dependencies { @@ -78,6 +90,7 @@ dependencies { implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: versions.dependencies.log4jCore implementation 'org.apache.commons:commons-lang3:' + versions.dependencies.commonsLang implementation 'com.google.code.gson:gson:' + versions.dependencies.gson + implementation group: 'commons-beanutils', name: 'commons-beanutils', version: versions.dependencies.commonsBeanutils implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "build/libs/logstash-core-*.*.*.jar") implementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "common-*.*.*.jar") @@ -101,6 +114,17 @@ tasks.withType(JavaCompile) { options.encoding = 'UTF-8' } +test { + jvmArgs = [ + '--add-opens=java.base/java.lang=ALL-UNNAMED', + '--add-opens=java.base/java.util=ALL-UNNAMED', + '--add-opens=java.base/java.text=ALL-UNNAMED', + '--add-opens=java.base/java.lang.reflect=ALL-UNNAMED', + '--add-opens=java.base/sun.util.resources=ALL-UNNAMED', + '--add-opens=java.base/sun.util.cldr=ALL-UNNAMED' + ] +} + tasks.register("generateRubySupportFiles") { @@ -143,17 +167,16 @@ apply plugin: "org.barfuin.gradle.jacocolog" // ------------------------------------ jacoco { toolVersion = "${jacocoVersion}" - reportsDir = file("$buildDir/reports/jacoco") } jacocoTestReport { // You will see "Report -> file://...." at the end of a JaCoCo build // If no output, run this first: ./gradlew test reports { - html.enabled true - xml.enabled true - csv.enabled true - html.destination file("${buildDir}/reports/jacoco") - csv.destination file("${buildDir}/reports/jacoco/all.csv") + html.required = true + xml.required = true + csv.required = true + html.outputLocation = file("${buildDir}/reports/jacoco") + csv.outputLocation = file("${buildDir}/reports/jacoco/all.csv") } executionData.from fileTree(dir: "${buildDir}/jacoco/", includes: [ '**/*.exec' diff --git a/filter-plugin/logstash-filter-snowflake-guardium/gradle/wrapper/gradle-wrapper.properties b/filter-plugin/logstash-filter-snowflake-guardium/gradle/wrapper/gradle-wrapper.properties index 60c76b340..ba9ccfe4c 100644 --- a/filter-plugin/logstash-filter-snowflake-guardium/gradle/wrapper/gradle-wrapper.properties +++ b/filter-plugin/logstash-filter-snowflake-guardium/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists \ No newline at end of file diff --git a/filter-plugin/logstash-filter-snowflake-guardium/snowflakeJDBC.conf b/filter-plugin/logstash-filter-snowflake-guardium/snowflakeJDBC.conf index 7bf96c4fb..aff0205ac 100644 --- a/filter-plugin/logstash-filter-snowflake-guardium/snowflakeJDBC.conf +++ b/filter-plugin/logstash-filter-snowflake-guardium/snowflakeJDBC.conf @@ -8,9 +8,10 @@ input { type => "snowflake" #To prevent authentication token expiration please add in jdbc_connection_string as mentioned below. #&CLIENT_SESSION_KEEP_ALIVE=true&CLIENT_SESSION_KEEP_ALIVE_HEARTBEAT_FREQUENCY=60 - jdbc_connection_string => "jdbc:snowflake://...snowflakecomputing.com/?warehouse=&db=" + #JDBC_QUERY_RESULT_FORMAT=JSON prevents Arrow format errors + jdbc_connection_string => "jdbc:snowflake://...snowflakecomputing.com/?warehouse=&db=&JDBC_QUERY_RESULT_FORMAT=JSON" # When using the KeyPair Authentication. use the below Connection string, - # jdbc_connection_string => "jdbc:snowflake://...snowflakecomputing.com/?user=&private_key_file=${THIRD_PARTY_PATH}/.p8&authenticator=snowflake_jwt" + # jdbc_connection_string => "jdbc:snowflake://...snowflakecomputing.com/?user=&warehouse=&db=&schema=&private_key_file=${THIRD_PARTY_PATH}/.p8&authenticator=snowflake_jwt" jdbc_user => "" # When using the KeyPair Authentication. remove the jdbc_password property below, jdbc_password => "" @@ -28,8 +29,10 @@ input { "event_type" => "login_success" "skip_logging_audit_query" => false } + # Set a start time to specify the date from which events should be pulled (set in same time zone as DB) parameters => { "execution_status" => "RUNNING" + "custom_start_time" => "2020-01-01 00:00:00" } statement => " SELECT @@ -49,16 +52,17 @@ input { LEFT JOIN SNOWFLAKE.ACCOUNT_USAGE.LOGIN_HISTORY LH ON S.LOGIN_EVENT_ID = LH.EVENT_ID WHERE (QH.EXECUTION_STATUS <> :execution_status) - AND DATE_PART(epoch_millisecond, QH.END_TIME) > :sql_last_value + AND DATE_PART(epoch_millisecond, QH.END_TIME) > :sql_last_value AND QH.END_TIME > :custom_start_time ORDER BY QH.END_TIME " } jdbc { type => "snowflake" - jdbc_connection_string => "jdbc:snowflake://...snowflakecomputing.com/?warehouse=&db=" + #JDBC_QUERY_RESULT_FORMAT=JSON prevents Arrow format errors + jdbc_connection_string => "jdbc:snowflake://...snowflakecomputing.com/?warehouse=&db=&JDBC_QUERY_RESULT_FORMAT=JSON" # When using the KeyPair Authentication. use the below Connection string, - # jdbc_connection_string => "jdbc:snowflake://...snowflakecomputing.com/?user=&private_key_file=${THIRD_PARTY_PATH}/.p8&authenticator=snowflake_jwt" + # jdbc_connection_string => "jdbc:snowflake://...snowflakecomputing.com/?user=&warehouse=&db=&schema=&private_key_file=${THIRD_PARTY_PATH}/.p8&authenticator=snowflake_jwt" jdbc_user => "" # When using the KeyPair Authentication. remove the jdbc_password property below, jdbc_password => "" @@ -76,8 +80,10 @@ input { "event_type" => "login_failed" "skip_logging_auth_audit_query" => false } + # Set a start time to specify the date from which events should be pulled (set in same time zone as DB) parameters => { "login_success" => "%NO%" + "custom_start_time" => "2020-01-01 00:00:00" } statement => " SELECT @@ -92,7 +98,7 @@ input { DATE_PART(epoch_millisecond, LH.EVENT_TIMESTAMP) AS LOGIN_TIMESTAMP_EPOCH FROM SNOWFLAKE.ACCOUNT_USAGE.LOGIN_HISTORY LH WHERE LH.IS_SUCCESS LIKE :login_success - AND DATE_PART(epoch_millisecond, LH.EVENT_TIMESTAMP) > :sql_last_value + AND DATE_PART(epoch_millisecond, LH.EVENT_TIMESTAMP) > :sql_last_value AND LH.EVENT_TIMESTAMP > :custom_start_time ORDER BY LH.EVENT_TIMESTAMP " } diff --git a/filter-plugin/logstash-filter-snowflake-guardium/src/main/java/com/ibm/guardium/snowflakedb/GuardiumSnowflakeFilter.java b/filter-plugin/logstash-filter-snowflake-guardium/src/main/java/com/ibm/guardium/snowflakedb/GuardiumSnowflakeFilter.java index e1c3c99ef..c765cf982 100644 --- a/filter-plugin/logstash-filter-snowflake-guardium/src/main/java/com/ibm/guardium/snowflakedb/GuardiumSnowflakeFilter.java +++ b/filter-plugin/logstash-filter-snowflake-guardium/src/main/java/com/ibm/guardium/snowflakedb/GuardiumSnowflakeFilter.java @@ -20,6 +20,7 @@ import com.ibm.guardium.snowflakedb.exceptions.ParseException; import com.ibm.guardium.snowflakedb.parser.AuthFailedEventParser; +import com.ibm.guardium.snowflakedb.parser.ErrorRecordBuilder; import com.ibm.guardium.snowflakedb.parser.Parser; import com.ibm.guardium.snowflakedb.parser.SQLErrorEventParser; import com.ibm.guardium.snowflakedb.parser.SuccessEventParser; @@ -55,6 +56,7 @@ public Collection filter(Collection events, FilterMatchListener ma ArrayList skippedEvents = new ArrayList<>(); for (Event event : events) { + Record partialRecord = null; try { Optional optEventType = Optional.ofNullable( event.getField(Constants.EVENT_TYPE) @@ -63,7 +65,7 @@ public Collection filter(Collection events, FilterMatchListener ma if(optEventType.isPresent()){ String eventType = optEventType.get(); if(log.isDebugEnabled()){ - log.info("Snowflake Filter - Event Type", eventType); + log.info("Snowflake Filter - Event Type: {}", eventType); } switch (eventType.toUpperCase()){ @@ -88,11 +90,23 @@ public Collection filter(Collection events, FilterMatchListener ma event.setField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME, gson.toJson(rec)); } } else { - throw new ParseException("Snowflake filter: Parser not initialized."); + // Unknown event_type - this is an audit/configuration error + throw new Exception("Unknown event_type - parser not initialized."); } } + } catch (ParseException parseException) { + // Handle parser errors + // ParseException is thrown during parsing, so we may have partial data + log.error("Snowflake filter: Parser error - {}", parseException.getMessage()); + Record errorRecord = ErrorRecordBuilder.parseRecordException(event, partialRecord, parseException.getMessage()); + event.setField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME, gson.toJson(errorRecord)); + event.tag(Constants.LOGSTASH_TAG_JSON_PARSE_ERROR); } catch (Exception exception) { - log.error("Snowflake filter: Error parsing event ", exception); + // Handle generic errors + // Generic exceptions may occur at any point, check if we have partial data + log.error("Snowflake filter: Error parsing event {}", exception); + Record errorRecord = ErrorRecordBuilder.parseRecordException(event, partialRecord, exception.getMessage()); + event.setField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME, gson.toJson(errorRecord)); event.tag(Constants.LOGSTASH_TAG_JSON_PARSE_ERROR); } } diff --git a/filter-plugin/logstash-filter-snowflake-guardium/src/main/java/com/ibm/guardium/snowflakedb/exceptions/ParseException.java b/filter-plugin/logstash-filter-snowflake-guardium/src/main/java/com/ibm/guardium/snowflakedb/exceptions/ParseException.java index 3eaad9c63..992116b43 100644 --- a/filter-plugin/logstash-filter-snowflake-guardium/src/main/java/com/ibm/guardium/snowflakedb/exceptions/ParseException.java +++ b/filter-plugin/logstash-filter-snowflake-guardium/src/main/java/com/ibm/guardium/snowflakedb/exceptions/ParseException.java @@ -7,10 +7,17 @@ public class ParseException extends Exception{ private String message; + public ParseException(String msg){ + super(msg); // Pass message to parent Exception class this.message = msg; } + @Override + public String getMessage() { + return this.message; + } + @Override public String toString() { return this.message; diff --git a/filter-plugin/logstash-filter-snowflake-guardium/src/main/java/com/ibm/guardium/snowflakedb/parser/AuthFailedEventParser.java b/filter-plugin/logstash-filter-snowflake-guardium/src/main/java/com/ibm/guardium/snowflakedb/parser/AuthFailedEventParser.java index 9e51b71fe..85a46d3a6 100644 --- a/filter-plugin/logstash-filter-snowflake-guardium/src/main/java/com/ibm/guardium/snowflakedb/parser/AuthFailedEventParser.java +++ b/filter-plugin/logstash-filter-snowflake-guardium/src/main/java/com/ibm/guardium/snowflakedb/parser/AuthFailedEventParser.java @@ -31,6 +31,7 @@ public AuthFailedEventParser() { guardRecord = builder.buildGuardRecordWithDefaultValues(); eventMap = new HashMap<>(); } + @Override public Record parseRecord(Map event) throws ParseException { @@ -41,7 +42,7 @@ public Record parseRecord(Map event) throws ParseException { } if(log.isDebugEnabled()){ - log.debug("Event Now: ",eventMap); + log.debug("Event Now: {} ",eventMap); } eventMap = event; @@ -49,7 +50,6 @@ public Record parseRecord(Map event) throws ParseException { Time time = getTime(); guardRecord.setTime(time); - SessionLocator sessionLocator = setSessionLocator(); String dbUser = getStringValueOf(Constants.USER_NAME); guardRecord.getAccessor().setDbUser(dbUser); @@ -63,9 +63,20 @@ public Record parseRecord(Map event) throws ParseException { guardRecord.setException(exceptionRecord); guardRecord.setData(null); - Integer hashCode = (sessionLocator.getClientIp() + sessionLocator.getClientPort() - + sessionLocator.getServerIp() + sessionLocator.getServerPort()).hashCode(); - guardRecord.setSessionId(hashCode.toString()); + String sessionId = this.getStringValueOf(Constants.SESSION_ID); + + SessionLocator sessionLocator = setSessionLocator(); + + if(!sessionId.equals(Constants.NOT_AVAILABLE)) { + guardRecord.setSessionId(sessionId); + } else { + guardRecord.setSessionId(Constants.UNKNOWN_STRING); + sessionLocator.setServerPort(-1); + sessionLocator.setClientPort(-1); + } + + guardRecord.setSessionLocator(sessionLocator); + guardRecord.setDbName(Constants.NOT_AVAILABLE); guardRecord.setAppUserName(dbUser); @@ -80,7 +91,7 @@ private ExceptionRecord setExceptionRecord() { String description = getStringValueOf(Constants.LOGIN_ERROR_CODE) + ": " +getStringValueOf(Constants.LOGIN_ERROR_MESSAGE); exceptionRecord.setDescription(description); } catch (Exception e) { - log.error("Snowflake filter: Error occurred while parsing Exception object: " + eventMap, e); + log.error("Snowflake filter: Error occurred while parsing Exception object: {} {}", eventMap, e); throw e; } return exceptionRecord; @@ -95,7 +106,7 @@ private SessionLocator setSessionLocator() { sessionLocator.setServerPort(Constants.SERVER_PORT); } catch (Exception e) { - log.error("Snowflake filter: Error occurred while parsing session locator object: " + eventMap, e); + log.error("Snowflake filter: Error occurred while parsing session locator object: {} {}", eventMap, e); throw e; } return sessionLocator; @@ -113,7 +124,7 @@ private String getStringValueOf(String fieldName){ return value; } - private Time getTime(){ + private Time getTime() throws ParseException { String ts = getStringValueOf(Constants.LOGIN_TIMESTAMP); Time t = guardRecord.getTime(); try { @@ -122,8 +133,8 @@ private Time getTime(){ t.setTimstamp(date.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli()); //Snowflake supplies the date in UTC t.setMinOffsetFromGMT(0); t.setMinDst(0); - } catch (Exception e){ - log.error("Snowflake filter: Error occurred while parsing Time object: " + eventMap, e); + } catch (ParseException e){ + log.error("Snowflake filter: Error occurred while parsing Time object: {} {}", eventMap, e); throw e; } diff --git a/filter-plugin/logstash-filter-snowflake-guardium/src/main/java/com/ibm/guardium/snowflakedb/parser/ErrorRecordBuilder.java b/filter-plugin/logstash-filter-snowflake-guardium/src/main/java/com/ibm/guardium/snowflakedb/parser/ErrorRecordBuilder.java new file mode 100644 index 000000000..3df95f995 --- /dev/null +++ b/filter-plugin/logstash-filter-snowflake-guardium/src/main/java/com/ibm/guardium/snowflakedb/parser/ErrorRecordBuilder.java @@ -0,0 +1,208 @@ +// +// Copyright 2021-2023 IBM Inc. All rights reserved +// SPDX-License-Identifier: Apache2.0 +// + +package com.ibm.guardium.snowflakedb.parser; + +import co.elastic.logstash.api.Event; +import com.ibm.guardium.snowflakedb.utils.Constants; +import com.ibm.guardium.snowflakedb.utils.DefaultGuardRecordBuilder; +import com.ibm.guardium.universalconnector.commons.structures.Accessor; +import com.ibm.guardium.universalconnector.commons.structures.ExceptionRecord; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import com.ibm.guardium.universalconnector.commons.structures.SessionLocator; +import com.ibm.guardium.universalconnector.commons.structures.Time; + +/** + * Utility class for building error records (UC_PARSER_ERRORS and UC_AUDIT_ERRORS) + * Extends DefaultGuardRecordBuilder to reuse default value builders + */ +public class ErrorRecordBuilder extends DefaultGuardRecordBuilder { + + /** + * Create a UC_PARSER_ERROR record when parser fails + * + * @param event The event that failed to parse + * @param errorMessage The error message describing the failure + * @return A Record with UC_PARSER_ERROR exception + */ + public static Record createParserErrorRecord(Event event, String errorMessage) { + return createErrorRecord(event, null, errorMessage, Constants.UC_PARSER_ERROR); + } + + /** + * Create a UC_AUDIT_ERROR record when audit processing fails + * + * @param event The event that failed to process + * @param errorMessage The error message describing the failure + * @return A Record with UC_AUDIT_ERROR exception + */ + public static Record createAuditErrorRecord(Event event, String errorMessage) { + return createErrorRecord(event, null, errorMessage, Constants.UC_AUDIT_ERROR); + } + + /** + * Parse and create an error record based on parsing state. + * Similar to DocumentDB's parseRecordException approach: if we successfully parsed Data + * with SQL command, it's a UC_PARSER_ERROR (data validation issue). Otherwise, it's a + * UC_AUDIT_ERROR (parsing failure). + * + * @param event The event that failed + * @param partialRecord Partially parsed record (may be null) + * @param errorMessage The error message describing the failure + * @return A Record with appropriate exception type + */ + public static Record parseRecordException(Event event, Record partialRecord, String errorMessage) { + // Determine error type based on whether we successfully parsed Data with SQL + String exceptionType; + if (partialRecord != null && partialRecord.getData() != null + && partialRecord.getData().getOriginalSqlCommand() != null + && !partialRecord.getData().getOriginalSqlCommand().isEmpty()) { + // We successfully parsed the event and extracted SQL - this is a data validation issue + exceptionType = Constants.UC_PARSER_ERROR; + } else { + // We couldn't parse the event properly - this is an audit/parsing failure + exceptionType = Constants.UC_AUDIT_ERROR; + } + return createErrorRecord(event, partialRecord, errorMessage, exceptionType); + } + + /** + * Common method to create error records + * + * @param event The event that failed + * @param partialRecord Partially parsed record (may be null), used to extract already-parsed data + * @param errorMessage The error message + * @param exceptionTypeId The exception type (UC_PARSER_ERROR or UC_AUDIT_ERROR) + * @return A Record with exception information and populated fields from event + */ + private static Record createErrorRecord(Event event, Record partialRecord, String errorMessage, String exceptionTypeId) { + ErrorRecordBuilder builder = new ErrorRecordBuilder(); + Record errorRecord = new Record(); + + // Set exception details + ExceptionRecord exceptionRecord = builder.buildDefaultExceptionRecord(); + exceptionRecord.setExceptionTypeId(exceptionTypeId); + + // Add appropriate prefix based on error type + String prefix = exceptionTypeId.equals(Constants.UC_PARSER_ERROR) ? "Parser Error: " : "Audit Error: "; + String description = prefix + (errorMessage != null ? errorMessage : ""); + exceptionRecord.setDescription(description); + + // Set SQL string - prefer from partialRecord if available, otherwise use full event + if (partialRecord != null && partialRecord.getData() != null + && partialRecord.getData().getOriginalSqlCommand() != null + && !partialRecord.getData().getOriginalSqlCommand().isEmpty()) { + exceptionRecord.setSqlString(partialRecord.getData().getOriginalSqlCommand()); + } else { + exceptionRecord.setSqlString(getEventAsString(event)); + } + errorRecord.setException(exceptionRecord); + + // Set time - use from partialRecord if available, otherwise current time + if (partialRecord != null && partialRecord.getTime() != null) { + errorRecord.setTime(partialRecord.getTime()); + } else { + errorRecord.setTime(new Time(System.currentTimeMillis(), 0, 0)); + } + + // Build session locator - use from partialRecord if available, otherwise build from event + SessionLocator sessionLocator; + if (partialRecord != null && partialRecord.getSessionLocator() != null) { + sessionLocator = partialRecord.getSessionLocator(); + } else { + sessionLocator = builder.buildDefaultSessionLocator(); + String clientIp = getFieldAsString(event, Constants.CLIENT_IP, null); + if (clientIp != null) { + sessionLocator.setClientIp(clientIp); + } + String serverIp = getFieldAsString(event, Constants.SERVER_IP, null); + if (serverIp != null) { + sessionLocator.setServerIp(serverIp); + } + sessionLocator.setServerPort(Constants.SERVER_PORT); + sessionLocator.setClientPort(-1); + } + errorRecord.setSessionLocator(sessionLocator); + + // Build accessor - use from partialRecord if available, otherwise build from event + Accessor accessor; + if (partialRecord != null && partialRecord.getAccessor() != null) { + accessor = partialRecord.getAccessor(); + } else { + accessor = builder.buildDefaultAccessor(); + String dbUser = getFieldAsString(event, Constants.USER_NAME, null); + if (dbUser != null) { + accessor.setDbUser(dbUser); + } + String sourceProgram = getFieldAsString(event, Constants.CLIENT_APPLICATION_ID, null); + if (sourceProgram != null) { + accessor.setSourceProgram(sourceProgram); + } + String serviceName = getFieldAsString(event, Constants.DATABASE_NAME, null); + if (serviceName != null) { + accessor.setServiceName(serviceName); + } + } + + // Always update serverHostName from event to ensure consistency + String serverHostName = getFieldAsString(event, Constants.SERVER_HOST_NAME, null); + if (serverHostName != null) { + accessor.setServerHostName(serverHostName); + } + } + errorRecord.setAccessor(accessor); + + // Set session ID and database name - use from partialRecord if available + if (partialRecord != null && partialRecord.getSessionId() != null) { + errorRecord.setSessionId(partialRecord.getSessionId()); + } else { + errorRecord.setSessionId(getFieldAsString(event, Constants.SESSION_ID, Constants.UNKNOWN_STRING)); + } + + if (partialRecord != null && partialRecord.getDbName() != null) { + errorRecord.setDbName(partialRecord.getDbName()); + } else { + errorRecord.setDbName(getFieldAsString(event, Constants.DATABASE_NAME, Constants.NOT_AVAILABLE)); + } + + errorRecord.setAppUserName(Constants.NOT_AVAILABLE); + + return errorRecord; + } + + /** + * Get field value from event as string, with default if not present + * + * @param event The event + * @param fieldName The field name to retrieve + * @param defaultValue Default value if field is not present or null + * @return Field value as string or default value + */ + private static String getFieldAsString(Event event, String fieldName, String defaultValue) { + try { + Object value = event.getField(fieldName); + if (value != null) { + return value.toString(); + } + } catch (Exception e) { + // Field not present or error accessing it + } + return defaultValue; + } + + /** + * Convert event to string for error logging + * + * @param event The event to convert + * @return String representation of the event, or error message if conversion fails + */ + private static String getEventAsString(Event event) { + try { + return event.toMap().toString(); + } catch (Exception e) { + return "Unable to convert event to string"; + } + } +} diff --git a/filter-plugin/logstash-filter-snowflake-guardium/src/main/java/com/ibm/guardium/snowflakedb/parser/Parser.java b/filter-plugin/logstash-filter-snowflake-guardium/src/main/java/com/ibm/guardium/snowflakedb/parser/Parser.java index f9cc98998..00e907161 100644 --- a/filter-plugin/logstash-filter-snowflake-guardium/src/main/java/com/ibm/guardium/snowflakedb/parser/Parser.java +++ b/filter-plugin/logstash-filter-snowflake-guardium/src/main/java/com/ibm/guardium/snowflakedb/parser/Parser.java @@ -14,6 +14,8 @@ import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatterBuilder; import java.time.format.DateTimeParseException; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.Optional; @@ -48,7 +50,7 @@ default String getClientOS(Map clientEnvironment) { return Constants.UNKNOWN_STRING; } - static LocalDateTime parseTime(String ts){ + static LocalDateTime parseTime(String ts) throws ParseException { DateTimeFormatterBuilder dateTimeFormatterBuilder = new DateTimeFormatterBuilder() .append(DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSS[[XXX][X]]")); @@ -57,11 +59,14 @@ static LocalDateTime parseTime(String ts){ try { return LocalDateTime.parse(ts,formatter); } catch (DateTimeParseException e) { - DateTimeFormatterBuilder dateTimeFormatterBuilderUTC = new DateTimeFormatterBuilder() - .append(DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ssZ")); - formatter = dateTimeFormatterBuilderUTC.toFormatter(); - - return LocalDateTime.parse(ts, formatter); + try { + DateTimeFormatterBuilder dateTimeFormatterBuilderUTC = new DateTimeFormatterBuilder() + .append(DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ssZ")); + formatter = dateTimeFormatterBuilderUTC.toFormatter(); + return LocalDateTime.parse(ts, formatter); + } catch (DateTimeParseException e2) { + throw new ParseException("Failed to parse timestamp: " + ts + ". Error: " + e2.getMessage()); + } } } } diff --git a/filter-plugin/logstash-filter-snowflake-guardium/src/main/java/com/ibm/guardium/snowflakedb/parser/SQLErrorEventParser.java b/filter-plugin/logstash-filter-snowflake-guardium/src/main/java/com/ibm/guardium/snowflakedb/parser/SQLErrorEventParser.java index 733038cf9..d2fd77a19 100644 --- a/filter-plugin/logstash-filter-snowflake-guardium/src/main/java/com/ibm/guardium/snowflakedb/parser/SQLErrorEventParser.java +++ b/filter-plugin/logstash-filter-snowflake-guardium/src/main/java/com/ibm/guardium/snowflakedb/parser/SQLErrorEventParser.java @@ -10,6 +10,7 @@ import com.ibm.guardium.snowflakedb.utils.DefaultGuardRecordBuilder; import com.ibm.guardium.snowflakedb.exceptions.ParseException; import com.ibm.guardium.universalconnector.commons.structures.*; +import com.ibm.guardium.universalconnector.commons.structures.Record; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -22,6 +23,7 @@ public class SQLErrorEventParser implements Parser{ private static Logger log = LogManager.getLogger(SuccessEventParser.class); + private static final Gson GSON = new Gson(); private Map eventMap; private Record guardRecord; @@ -30,6 +32,7 @@ public SQLErrorEventParser(){ guardRecord = builder.buildGuardRecordWithDefaultValues(); eventMap = new HashMap<>(); } + @Override public Record parseRecord(Map event) throws ParseException { @@ -40,7 +43,7 @@ public Record parseRecord(Map event) throws ParseException { } if(log.isDebugEnabled()){ - log.debug("Event Now: ",eventMap); + log.debug("Event Now: {} ",eventMap); } eventMap = event; @@ -58,7 +61,15 @@ public Record parseRecord(Map event) throws ParseException { guardRecord.setException(exceptionRecord); guardRecord.setData(null); - guardRecord.setSessionId(this.getStringValueOf(Constants.SESSION_ID)); + + String sessionId = this.getStringValueOf(Constants.SESSION_ID); + + if(!sessionId.equals(Constants.NOT_AVAILABLE)) { + guardRecord.setSessionId(sessionId); + } else { + guardRecord.setSessionId(Constants.UNKNOWN_STRING); + } + guardRecord.setDbName(this.getStringValueOf(Constants.DATABASE_NAME)); return guardRecord; @@ -77,8 +88,7 @@ private Accessor setAccessor() { ).map(Object::toString); if(optClientEnv.isPresent() && !optClientEnv.get().isEmpty()){ - Gson gson = new Gson(); - Map clientEnv = gson.fromJson(optClientEnv.get(), Map.class); + Map clientEnv = GSON.fromJson(optClientEnv.get(), Map.class); String clientOS = getClientOS(clientEnv); accessor.setClientOs(clientOS); @@ -94,7 +104,7 @@ private Accessor setAccessor() { accessor.setServerHostName(getStringValueOf(Constants.SERVER_HOST_NAME)); accessor.setDbProtocol(Constants.DB_PROTOCOL); } catch (Exception e) { - log.error("Snowflake filter: Error occurred while parsing Accessor object: " + eventMap, e); + log.error("Snowflake filter: Error occurred while parsing Accessor object: {} {}",eventMap, e); throw e; } @@ -110,7 +120,7 @@ private ExceptionRecord setExceptionRecord() { exceptionRecord.setDescription(description); exceptionRecord.setSqlString(getStringValueOf(Constants.QUERY_TEXT)); } catch (Exception e) { - log.error("Snowflake filter: Error occurred while parsing Exception object: " + eventMap, e); + log.error("Snowflake filter: Error occurred while parsing Exception object: {} {}", eventMap, e); throw e; } return exceptionRecord; @@ -120,12 +130,15 @@ private SessionLocator setSessionLocator() { SessionLocator sessionLocator = guardRecord.getSessionLocator(); try { - sessionLocator.setClientIp(getStringValueOf(Constants.CLIENT_IP)); - sessionLocator.setServerIp(getStringValueOf(Constants.SERVER_IP)); + String clientIp = getStringValueOf(Constants.CLIENT_IP); + String serverIp = getStringValueOf(Constants.SERVER_IP); + + sessionLocator.setClientIp((clientIp == null || clientIp.isEmpty()) ? Constants.DEFAULT_IP : clientIp); + sessionLocator.setServerIp((serverIp == null || serverIp.isEmpty()) ? Constants.DEFAULT_IP : serverIp); sessionLocator.setServerPort(Constants.SERVER_PORT); } catch (Exception e) { - log.error("Snowflake filter: Error occurred while parsing session locator object: " + eventMap, e); + log.error("Snowflake filter: Error occurred while parsing session locator object: {} {}", eventMap, e); throw e; } return sessionLocator; @@ -143,7 +156,7 @@ private String getStringValueOf(String fieldName){ return value; } - private Time getTime(){ + private Time getTime() throws ParseException { String ts = getStringValueOf(Constants.QUERY_TIMESTAMP); Time t = guardRecord.getTime(); try { @@ -152,8 +165,8 @@ private Time getTime(){ t.setTimstamp(date.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli()); //Snowflake supplies the date in UTC t.setMinOffsetFromGMT(0); t.setMinDst(0); - } catch (Exception e){ - log.error("Snowflake filter: Error occurred while parsing Time object: " + eventMap, e); + } catch (ParseException e){ + log.error("Snowflake filter: Error occurred while parsing Time object: {} {}", eventMap, e); throw e; } diff --git a/filter-plugin/logstash-filter-snowflake-guardium/src/main/java/com/ibm/guardium/snowflakedb/parser/SuccessEventParser.java b/filter-plugin/logstash-filter-snowflake-guardium/src/main/java/com/ibm/guardium/snowflakedb/parser/SuccessEventParser.java index 2ddaa316c..ed0616b4d 100644 --- a/filter-plugin/logstash-filter-snowflake-guardium/src/main/java/com/ibm/guardium/snowflakedb/parser/SuccessEventParser.java +++ b/filter-plugin/logstash-filter-snowflake-guardium/src/main/java/com/ibm/guardium/snowflakedb/parser/SuccessEventParser.java @@ -16,14 +16,16 @@ import com.ibm.guardium.snowflakedb.utils.DefaultGuardRecordBuilder; import com.ibm.guardium.snowflakedb.exceptions.ParseException; import com.ibm.guardium.universalconnector.commons.structures.*; +import com.ibm.guardium.universalconnector.commons.structures.Record; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; public class SuccessEventParser implements Parser{ private static Logger log = LogManager.getLogger(SuccessEventParser.class); + private static final Gson GSON = new Gson(); private Map eventMap; - private Record guardRecord; + private com.ibm.guardium.universalconnector.commons.structures.Record guardRecord; public SuccessEventParser() { DefaultGuardRecordBuilder builder = new DefaultGuardRecordBuilder(); @@ -47,7 +49,7 @@ public Record parseRecord(final Map event) throws ParseException } if(log.isDebugEnabled()){ - log.debug("Event Now: ",eventMap); + log.debug("Event Now: {}",eventMap); } eventMap = event; @@ -66,7 +68,13 @@ public Record parseRecord(final Map event) throws ParseException guardRecord.setData(dataObj); guardRecord.setException(null); - guardRecord.setSessionId(this.getStringValueOf(Constants.SESSION_ID)); + String sessionId = this.getStringValueOf(Constants.SESSION_ID); + + if(!sessionId.equals(Constants.NOT_AVAILABLE)) { + guardRecord.setSessionId(sessionId); + } else { + guardRecord.setSessionId(Constants.UNKNOWN_STRING); + } guardRecord.setDbName(this.getStringValueOf(Constants.DATABASE_NAME)); return this.guardRecord; @@ -109,8 +117,7 @@ private Accessor setAccessor() { ).map(Object::toString); if(optClientEnv.isPresent() && !optClientEnv.get().isEmpty()){ - Gson gson = new Gson(); - Map clientEnv = gson.fromJson(optClientEnv.get(), Map.class); + Map clientEnv = GSON.fromJson(optClientEnv.get(), Map.class); String clientOS = getClientOS(clientEnv); accessor.setClientOs(clientOS); @@ -140,8 +147,11 @@ private SessionLocator setSessionLocator() { SessionLocator sessionLocator = guardRecord.getSessionLocator(); try { - sessionLocator.setClientIp(getStringValueOf(Constants.CLIENT_IP)); - sessionLocator.setServerIp(getStringValueOf(Constants.SERVER_IP)); + String clientIp = getStringValueOf(Constants.CLIENT_IP); + String serverIp = getStringValueOf(Constants.SERVER_IP); + + sessionLocator.setClientIp((clientIp == null || clientIp.isEmpty()) ? Constants.DEFAULT_IP : clientIp); + sessionLocator.setServerIp((serverIp == null || serverIp.isEmpty()) ? Constants.DEFAULT_IP : serverIp); sessionLocator.setServerPort(Constants.SERVER_PORT); } catch (Exception e) { @@ -151,7 +161,7 @@ private SessionLocator setSessionLocator() { return sessionLocator; } - private Time getTime(){ + private Time getTime() throws ParseException { String ts = getStringValueOf(Constants.QUERY_TIMESTAMP); Time t = guardRecord.getTime(); try { @@ -160,7 +170,7 @@ private Time getTime(){ t.setTimstamp(date.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli()); //Snowflake supplies the date in UTC t.setMinOffsetFromGMT(0); t.setMinDst(0); - } catch (Exception e){ + } catch (ParseException e){ log.error("Snowflake filter: Error occurred while parsing Time object: " + eventMap, e); throw e; } diff --git a/filter-plugin/logstash-filter-snowflake-guardium/src/main/java/com/ibm/guardium/snowflakedb/utils/Constants.java b/filter-plugin/logstash-filter-snowflake-guardium/src/main/java/com/ibm/guardium/snowflakedb/utils/Constants.java index 017c04ecd..8e3155ee0 100644 --- a/filter-plugin/logstash-filter-snowflake-guardium/src/main/java/com/ibm/guardium/snowflakedb/utils/Constants.java +++ b/filter-plugin/logstash-filter-snowflake-guardium/src/main/java/com/ibm/guardium/snowflakedb/utils/Constants.java @@ -9,6 +9,8 @@ public class Constants { public static final String NOT_AVAILABLE = "NA"; + + public static final String DEFAULT_IP = "0.0.0.0"; public static final String UNKNOWN_STRING = StringUtils.EMPTY; public static final String LANGUAGE_SNOWFLAKE= "SNOWFLAKE"; @@ -63,6 +65,8 @@ public class Constants { public static final String SUCCESS = "SUCCESS"; public static final String SQL_ERROR = "SQL_ERROR"; public static final String LOGIN_FAILED = "LOGIN_FAILED"; + public static final String UC_PARSER_ERROR = "UC_PARSER_ERROR"; + public static final String UC_AUDIT_ERROR = "UC_AUDIT_ERROR"; public static final String TEXT = "TEXT"; diff --git a/filter-plugin/logstash-filter-snowflake-guardium/src/main/java/com/ibm/guardium/snowflakedb/utils/DefaultGuardRecordBuilder.java b/filter-plugin/logstash-filter-snowflake-guardium/src/main/java/com/ibm/guardium/snowflakedb/utils/DefaultGuardRecordBuilder.java index bff467e17..8cecd1afa 100644 --- a/filter-plugin/logstash-filter-snowflake-guardium/src/main/java/com/ibm/guardium/snowflakedb/utils/DefaultGuardRecordBuilder.java +++ b/filter-plugin/logstash-filter-snowflake-guardium/src/main/java/com/ibm/guardium/snowflakedb/utils/DefaultGuardRecordBuilder.java @@ -5,7 +5,16 @@ package com.ibm.guardium.snowflakedb.utils; -import com.ibm.guardium.universalconnector.commons.structures.*; +import com.ibm.guardium.universalconnector.commons.structures.Accessor; +import com.ibm.guardium.universalconnector.commons.structures.Construct; +import com.ibm.guardium.universalconnector.commons.structures.Data; +import com.ibm.guardium.universalconnector.commons.structures.ExceptionRecord; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import com.ibm.guardium.universalconnector.commons.structures.Sentence; +import com.ibm.guardium.universalconnector.commons.structures.SentenceObject; +import com.ibm.guardium.universalconnector.commons.structures.SessionLocator; +import com.ibm.guardium.universalconnector.commons.structures.Time; +import com.ibm.guardium.universalconnector.commons.structures.Record; public class DefaultGuardRecordBuilder { public Record buildGuardRecordWithDefaultValues(){ @@ -23,7 +32,7 @@ public Record buildGuardRecordWithDefaultValues(){ return record; } - private SessionLocator buildDefaultSessionLocator(){ + protected SessionLocator buildDefaultSessionLocator(){ SessionLocator sessionLocator = new SessionLocator(); sessionLocator.setIpv6(false); @@ -38,7 +47,7 @@ private SessionLocator buildDefaultSessionLocator(){ return sessionLocator; } - private Accessor buildDefaultAccessor(){ + protected Accessor buildDefaultAccessor(){ Accessor accessor = new Accessor(); accessor.setDbProtocol(Constants.DB_PROTOCOL); @@ -63,20 +72,20 @@ private Accessor buildDefaultAccessor(){ return accessor; } - private Data buildDefaultData() { + protected Data buildDefaultData() { Data data = new Data(); data.setOriginalSqlCommand(Constants.NOT_AVAILABLE); return data; } - private ExceptionRecord buildDefaultExceptionRecord(){ + protected ExceptionRecord buildDefaultExceptionRecord(){ ExceptionRecord exceptionRecord = new ExceptionRecord(); exceptionRecord.setDescription(Constants.NOT_AVAILABLE); exceptionRecord.setSqlString(Constants.NOT_AVAILABLE); return exceptionRecord; } - private Time buildTime(){ + protected Time buildTime(){ Time t = new Time(); t.setTimstamp(0); t.setMinOffsetFromGMT(0); diff --git a/filter-plugin/logstash-filter-snowflake-guardium/src/test/java/com/ibm/guardium/snowflakedb/AuthFailedEventParserTest.java b/filter-plugin/logstash-filter-snowflake-guardium/src/test/java/com/ibm/guardium/snowflakedb/AuthFailedEventParserTest.java index 856d16941..313daea4a 100644 --- a/filter-plugin/logstash-filter-snowflake-guardium/src/test/java/com/ibm/guardium/snowflakedb/AuthFailedEventParserTest.java +++ b/filter-plugin/logstash-filter-snowflake-guardium/src/test/java/com/ibm/guardium/snowflakedb/AuthFailedEventParserTest.java @@ -8,9 +8,11 @@ import com.ibm.guardium.snowflakedb.exceptions.ParseException; import com.ibm.guardium.snowflakedb.parser.AuthFailedEventParser; import com.ibm.guardium.snowflakedb.parser.Parser; +import com.ibm.guardium.snowflakedb.parser.SQLErrorEventParser; import com.ibm.guardium.snowflakedb.utils.Constants; import com.ibm.guardium.universalconnector.commons.structures.Record; import com.ibm.guardium.universalconnector.commons.structures.SessionLocator; +import org.apache.commons.lang3.StringUtils; import org.junit.Assert; import org.junit.Test; import org.logstash.Event; @@ -29,10 +31,7 @@ public void testAuthError(){ try{ Record record = parser.parseRecord(event); - Integer hashCode = (e.getField(Constants.CLIENT_IP).toString() + SessionLocator.PORT_DEFAULT - + e.getField(Constants.SERVER_IP) + Constants.SERVER_PORT).hashCode(); - - Assert.assertEquals(hashCode.toString(), record.getSessionId()); + Assert.assertEquals(Constants.UNKNOWN_STRING, record.getSessionId()); Assert.assertEquals(record.getAppUserName(), event.get(Constants.USER_NAME).toString()); Assert.assertEquals(record.getTime().getMinDst(), 0); Assert.assertEquals(record.getTime().getMinOffsetFromGMT(), 0); @@ -65,8 +64,7 @@ public void testAuthError(){ Assert.assertEquals(record.getSessionLocator().getClientIp(),event.get(Constants.CLIENT_IP).toString()); Assert.assertEquals(record.getSessionLocator().getServerIp(),event.get(Constants.SERVER_IP).toString()); - Assert.assertEquals(Long.valueOf(record.getSessionLocator().getServerPort()), - Long.valueOf(Constants.SERVER_PORT)); + Assert.assertEquals(Long.valueOf(record.getSessionLocator().getServerPort()), Long.valueOf(-1)); Assert.assertEquals(null,record.getData()); Assert.assertEquals(event.get(Constants.LOGIN_ERROR_CODE) + ": " + @@ -79,4 +77,21 @@ public void testAuthError(){ ex.printStackTrace(); } } + + + @Test + public void testSessionIDWhenClientAndServerSessionNotPresent() throws ParseException { + Event e = FakeEventFactory.getAuthErrorEvent(); + e.remove(Constants.SESSION_ID); + e.remove(Constants.CLIENT_IP); + e.remove(Constants.SERVER_IP); + + Parser parser = new AuthFailedEventParser(); + Record record = parser.parseRecord(e.toMap()); + SessionLocator sessionLocator = record.getSessionLocator(); + + Assert.assertEquals(StringUtils.EMPTY, record.getSessionId()); + Assert.assertEquals(-1, sessionLocator.getClientPort()); + Assert.assertEquals(-1, sessionLocator.getServerPort()); + } } diff --git a/filter-plugin/logstash-filter-snowflake-guardium/src/test/java/com/ibm/guardium/snowflakedb/ErrorRecordBuilderTest.java b/filter-plugin/logstash-filter-snowflake-guardium/src/test/java/com/ibm/guardium/snowflakedb/ErrorRecordBuilderTest.java new file mode 100644 index 000000000..ab770024f --- /dev/null +++ b/filter-plugin/logstash-filter-snowflake-guardium/src/test/java/com/ibm/guardium/snowflakedb/ErrorRecordBuilderTest.java @@ -0,0 +1,234 @@ +// +// Copyright 2021-2023 IBM Inc. All rights reserved +// SPDX-License-Identifier: Apache2.0 +// + +package com.ibm.guardium.snowflakedb; + +import com.ibm.guardium.snowflakedb.parser.ErrorRecordBuilder; +import com.ibm.guardium.snowflakedb.utils.Constants; +import com.ibm.guardium.universalconnector.commons.structures.Accessor; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import org.junit.Assert; +import org.junit.Test; +import org.logstash.Event; + +import java.util.HashMap; +import java.util.Map; + +public class ErrorRecordBuilderTest { + + @Test + public void testCreateParserErrorRecord() { + // Create a mock event + Event event = new Event(); + event.setField("test_field", "test_value"); + event.setField(Constants.QUERY_TEXT, "SELECT * FROM invalid_table"); + + String errorMessage = "Failed to parse query syntax"; + + // Create parser error record + Record errorRecord = ErrorRecordBuilder.createParserErrorRecord(event, errorMessage); + + // Verify exception details + Assert.assertNotNull("Exception should not be null", errorRecord.getException()); + Assert.assertEquals("Exception type should be UC_PARSER_ERROR", + Constants.UC_PARSER_ERROR, errorRecord.getException().getExceptionTypeId()); + Assert.assertTrue("Description should contain error message", + errorRecord.getException().getDescription().contains(errorMessage)); + Assert.assertTrue("Description should have 'Parser Error:' prefix", + errorRecord.getException().getDescription().startsWith("Parser Error:")); + Assert.assertNotNull("SQL string should not be null", + errorRecord.getException().getSqlString()); + + // Verify time is set + Assert.assertNotNull("Time should not be null", errorRecord.getTime()); + Assert.assertTrue("Timestamp should be greater than 0", + errorRecord.getTime().getTimstamp() > 0); + + // Verify session locator is set + Assert.assertNotNull("SessionLocator should not be null", errorRecord.getSessionLocator()); + + // Verify accessor details + Assert.assertNotNull("Accessor should not be null", errorRecord.getAccessor()); + Assert.assertEquals("DB Protocol should be SNOWFLAKE", + Constants.DB_PROTOCOL, errorRecord.getAccessor().getDbProtocol()); + Assert.assertEquals("Server Type should be SNOWFLAKE", + Constants.SERVER_TYPE, errorRecord.getAccessor().getServerType()); + Assert.assertEquals("Language should be SNOWFLAKE", + Constants.LANGUAGE_SNOWFLAKE, errorRecord.getAccessor().getLanguage()); + Assert.assertEquals("Data type should be TEXT", + Accessor.DATA_TYPE_GUARDIUM_SHOULD_PARSE_SQL, errorRecord.getAccessor().getDataType()); + + // Verify database name + Assert.assertEquals("DB name should be NA", + Constants.NOT_AVAILABLE, errorRecord.getDbName()); + } + + @Test + public void testCreateAuditErrorRecord() { + // Create a mock event + Event event = new Event(); + event.setField("test_field", "test_value"); + event.setField(Constants.USER_NAME, "test_user"); + + String errorMessage = "Failed to process audit log"; + + // Create audit error record + Record errorRecord = ErrorRecordBuilder.createAuditErrorRecord(event, errorMessage); + + // Verify exception details + Assert.assertNotNull("Exception should not be null", errorRecord.getException()); + Assert.assertEquals("Exception type should be UC_AUDIT_ERROR", + Constants.UC_AUDIT_ERROR, errorRecord.getException().getExceptionTypeId()); + Assert.assertTrue("Description should contain error message", + errorRecord.getException().getDescription().contains(errorMessage)); + Assert.assertTrue("Description should have 'Audit Error:' prefix", + errorRecord.getException().getDescription().startsWith("Audit Error:")); + Assert.assertNotNull("SQL string should not be null", + errorRecord.getException().getSqlString()); + + // Verify time is set + Assert.assertNotNull("Time should not be null", errorRecord.getTime()); + Assert.assertTrue("Timestamp should be greater than 0", + errorRecord.getTime().getTimstamp() > 0); + + // Verify session locator is set + Assert.assertNotNull("SessionLocator should not be null", errorRecord.getSessionLocator()); + + // Verify accessor details + Assert.assertNotNull("Accessor should not be null", errorRecord.getAccessor()); + Assert.assertEquals("DB Protocol should be SNOWFLAKE", + Constants.DB_PROTOCOL, errorRecord.getAccessor().getDbProtocol()); + Assert.assertEquals("Server Type should be SNOWFLAKE", + Constants.SERVER_TYPE, errorRecord.getAccessor().getServerType()); + Assert.assertEquals("Language should be SNOWFLAKE", + Constants.LANGUAGE_SNOWFLAKE, errorRecord.getAccessor().getLanguage()); + Assert.assertEquals("Data type should be TEXT", + Accessor.DATA_TYPE_GUARDIUM_SHOULD_PARSE_SQL, errorRecord.getAccessor().getDataType()); + + // Verify database name + Assert.assertEquals("DB name should be NA", + Constants.NOT_AVAILABLE, errorRecord.getDbName()); + } + + @Test + public void testParserErrorWithNullErrorMessage() { + Event event = new Event(); + event.setField("test_field", "test_value"); + + // Create parser error record with null error message + Record errorRecord = ErrorRecordBuilder.createParserErrorRecord(event, null); + + // Verify it handles null gracefully + Assert.assertNotNull("Exception should not be null", errorRecord.getException()); + Assert.assertEquals("Exception type should be UC_PARSER_ERROR", + Constants.UC_PARSER_ERROR, errorRecord.getException().getExceptionTypeId()); + Assert.assertTrue("Description should have 'Parser Error:' prefix", + errorRecord.getException().getDescription().startsWith("Parser Error:")); + } + + @Test + public void testAuditErrorWithEmptyErrorMessage() { + Event event = new Event(); + event.setField("test_field", "test_value"); + + // Create audit error record with empty error message + Record errorRecord = ErrorRecordBuilder.createAuditErrorRecord(event, ""); + + // Verify it handles empty string gracefully + Assert.assertNotNull("Exception should not be null", errorRecord.getException()); + Assert.assertEquals("Exception type should be UC_AUDIT_ERROR", + Constants.UC_AUDIT_ERROR, errorRecord.getException().getExceptionTypeId()); + Assert.assertTrue("Description should have 'Audit Error:' prefix", + errorRecord.getException().getDescription().startsWith("Audit Error:")); + } + + @Test + public void testErrorRecordContainsEventData() { + Event event = new Event(); + event.setField(Constants.QUERY_TEXT, "SELECT * FROM test_table"); + event.setField(Constants.USER_NAME, "test_user"); + event.setField(Constants.DATABASE_NAME, "test_db"); + + String errorMessage = "Test error"; + + // Create parser error record + Record errorRecord = ErrorRecordBuilder.createParserErrorRecord(event, errorMessage); + + // Verify SQL string contains event data + String sqlString = errorRecord.getException().getSqlString(); + Assert.assertNotNull("SQL string should not be null", sqlString); + Assert.assertTrue("SQL string should contain event data", sqlString.length() > 0); + } + + @Test + public void testErrorRecordPopulatesFieldsFromEvent() { + // Create event with complete data + Event event = new Event(); + event.setField(Constants.USER_NAME, "test_user"); + event.setField(Constants.DATABASE_NAME, "test_db"); + event.setField(Constants.SERVER_HOST_NAME, "test.snowflakecomputing.com"); + event.setField(Constants.CLIENT_IP, "192.168.1.100"); + event.setField(Constants.SERVER_IP, "10.0.0.1"); + event.setField(Constants.SESSION_ID, "session123"); + event.setField(Constants.CLIENT_APPLICATION_ID, "JDBC 3.13.6"); + + String errorMessage = "Test error with complete data"; + + // Create parser error record + Record errorRecord = ErrorRecordBuilder.createParserErrorRecord(event, errorMessage); + + // Verify accessor fields are populated from event + Assert.assertEquals("DB user should be from event", + "test_user", errorRecord.getAccessor().getDbUser()); + Assert.assertEquals("Server hostname should be from event", + "test.snowflakecomputing.com", errorRecord.getAccessor().getServerHostName()); + Assert.assertEquals("Service name should be from event", + "test_db", errorRecord.getAccessor().getServiceName()); + Assert.assertEquals("Source program should be from event", + "JDBC 3.13.6", errorRecord.getAccessor().getSourceProgram()); + + // Verify session locator fields are populated from event + Assert.assertEquals("Client IP should be from event", + "192.168.1.100", errorRecord.getSessionLocator().getClientIp()); + Assert.assertEquals("Server IP should be from event", + "10.0.0.1", errorRecord.getSessionLocator().getServerIp()); + Assert.assertEquals("Server port should be set", + Constants.SERVER_PORT, Integer.valueOf(errorRecord.getSessionLocator().getServerPort())); + + // Verify session ID and database name + Assert.assertEquals("Session ID should be from event", + "session123", errorRecord.getSessionId()); + Assert.assertEquals("DB name should be from event", + "test_db", errorRecord.getDbName()); + } + + @Test + public void testErrorRecordWithMissingFields() { + // Create event with minimal data + Event event = new Event(); + event.setField(Constants.USER_NAME, "test_user"); + + String errorMessage = "Test error with missing fields"; + + // Create audit error record + Record errorRecord = ErrorRecordBuilder.createAuditErrorRecord(event, errorMessage); + + // Verify default values are used for missing fields + Assert.assertEquals("Server hostname should use default", + Constants.UNKNOWN_STRING, errorRecord.getAccessor().getServerHostName()); + Assert.assertEquals("Client IP should use default", + Constants.DEFAULT_IP, errorRecord.getSessionLocator().getClientIp()); + Assert.assertEquals("Server IP should use default", + Constants.DEFAULT_IP, errorRecord.getSessionLocator().getServerIp()); + Assert.assertEquals("DB name should use default", + Constants.NOT_AVAILABLE, errorRecord.getDbName()); + Assert.assertEquals("Session ID should use default", + Constants.UNKNOWN_STRING, errorRecord.getSessionId()); + + // Verify user name is still populated + Assert.assertEquals("DB user should be from event", + "test_user", errorRecord.getAccessor().getDbUser()); + } +} diff --git a/filter-plugin/logstash-filter-snowflake-guardium/src/test/java/com/ibm/guardium/snowflakedb/GuardiumSnowflakeFilterTest.java b/filter-plugin/logstash-filter-snowflake-guardium/src/test/java/com/ibm/guardium/snowflakedb/GuardiumSnowflakeFilterTest.java new file mode 100644 index 000000000..b2d6f4bed --- /dev/null +++ b/filter-plugin/logstash-filter-snowflake-guardium/src/test/java/com/ibm/guardium/snowflakedb/GuardiumSnowflakeFilterTest.java @@ -0,0 +1,246 @@ +// +// Copyright 2021-2023 IBM Inc. All rights reserved +// SPDX-License-Identifier: Apache2.0 +// + +package com.ibm.guardium.snowflakedb; + +import co.elastic.logstash.api.Configuration; +import co.elastic.logstash.api.Context; +import co.elastic.logstash.api.Event; +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import com.ibm.guardium.snowflakedb.utils.Constants; +import com.ibm.guardium.universalconnector.commons.GuardConstants; +import org.junit.Assert; +import org.junit.Test; +import org.logstash.plugins.ConfigurationImpl; + +import java.util.*; + +public class GuardiumSnowflakeFilterTest { + + private final static Context context = null; + private final static Configuration config = new ConfigurationImpl(Collections.emptyMap()); + + @Test + public void testUnknownEventTypeCreatesAuditError() { + // Test that unknown event_type creates UC_AUDIT_ERROR + GuardiumSnowflakeFilter filter = new GuardiumSnowflakeFilter("test-id", config, context); + + Event event = new org.logstash.Event(); + event.setField(Constants.EVENT_TYPE, "UNKNOWN_TYPE"); + event.setField(Constants.USER_NAME, "test_user"); + event.setField(Constants.QUERY_TIMESTAMP, "2024-01-01T10:00:00.000Z"); + event.setField(Constants.CLIENT_IP, "192.168.1.1"); + event.setField(Constants.SERVER_IP, "10.0.0.1"); + event.setField(Constants.SERVER_HOST_NAME, "test.snowflakecomputing.com"); + event.setField(Constants.DATABASE_NAME, "test_db"); + event.setField(Constants.QUERY_TEXT, "SELECT 1"); + + Collection results = filter.filter(Collections.singletonList(event), null); + + Assert.assertEquals("Should return 1 event", 1, results.size()); + Event resultEvent = results.iterator().next(); + + Object guardRecordObj = resultEvent.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME); + Assert.assertNotNull("GuardRecord should be present", guardRecordObj); + + Gson gson = new Gson(); + JsonObject guardRecord = gson.fromJson(guardRecordObj.toString(), JsonObject.class); + JsonObject exception = guardRecord.getAsJsonObject("exception"); + + Assert.assertNotNull("Exception should be present", exception); + Assert.assertEquals("Should be UC_AUDIT_ERROR", + Constants.UC_AUDIT_ERROR, + exception.get("exceptionTypeId").getAsString()); + Assert.assertTrue("Description should mention unknown event_type", + exception.get("description").getAsString().toLowerCase().contains("unknown")); + } + + @Test + public void testInvalidTimestampCreatesAuditError() { + // Test that invalid timestamp creates UC_AUDIT_ERROR + // timestamp parsing fails early, before SQL extraction, + // so no partial record with Data exists -> UC_AUDIT_ERROR + GuardiumSnowflakeFilter filter = new GuardiumSnowflakeFilter("test-id", config, context); + + Event event = new org.logstash.Event(); + event.setField(Constants.EVENT_TYPE, Constants.SUCCESS); + event.setField(Constants.USER_NAME, "test_user"); + event.setField(Constants.QUERY_TIMESTAMP, "INVALID_TIMESTAMP"); + event.setField(Constants.CLIENT_IP, "192.168.1.1"); + event.setField(Constants.SERVER_IP, "10.0.0.1"); + event.setField(Constants.SERVER_HOST_NAME, "test.snowflakecomputing.com"); + event.setField(Constants.DATABASE_NAME, "test_db"); + event.setField(Constants.QUERY_TEXT, "SELECT 1"); + event.setField(Constants.SESSION_ID, "session123"); + event.setField(Constants.QUERY_ID, "query123"); + + Collection results = filter.filter(Collections.singletonList(event), null); + + Assert.assertEquals("Should return 1 event", 1, results.size()); + Event resultEvent = results.iterator().next(); + + Object guardRecordObj = resultEvent.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME); + Assert.assertNotNull("GuardRecord should be present", guardRecordObj); + + Gson gson = new Gson(); + JsonObject guardRecord = gson.fromJson(guardRecordObj.toString(), JsonObject.class); + JsonObject exception = guardRecord.getAsJsonObject("exception"); + + Assert.assertNotNull("Exception should be present", exception); + Assert.assertEquals("Should be UC_AUDIT_ERROR (parsing failed before SQL extraction)", + Constants.UC_AUDIT_ERROR, + exception.get("exceptionTypeId").getAsString()); + } + + @Test + public void testValidSuccessEventProcessedCorrectly() { + // Test that valid SUCCESS event is processed without errors + GuardiumSnowflakeFilter filter = new GuardiumSnowflakeFilter("test-id", config, context); + + Event event = FakeEventFactory.getSuccessEvent(); + event.setField(Constants.EVENT_TYPE, Constants.SUCCESS); + + Collection results = filter.filter(Collections.singletonList(event), null); + + Assert.assertEquals("Should return 1 event", 1, results.size()); + Event resultEvent = results.iterator().next(); + + Object guardRecordObj = resultEvent.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME); + Assert.assertNotNull("GuardRecord should be present", guardRecordObj); + + Gson gson = new Gson(); + JsonObject guardRecord = gson.fromJson(guardRecordObj.toString(), JsonObject.class); + + // Should not have UC error exception for valid event + if (guardRecord.has("exception") && !guardRecord.get("exception").isJsonNull()) { + JsonObject exception = guardRecord.getAsJsonObject("exception"); + if (exception.has("exceptionTypeId")) { + String exceptionType = exception.get("exceptionTypeId").getAsString(); + Assert.assertNotEquals("Should not be UC_PARSER_ERROR", + Constants.UC_PARSER_ERROR, exceptionType); + Assert.assertNotEquals("Should not be UC_AUDIT_ERROR", + Constants.UC_AUDIT_ERROR, exceptionType); + } + } + } + + @Test + public void testValidSQLErrorEventProcessedCorrectly() { + // Test that valid SQL_ERROR event is processed without UC errors + GuardiumSnowflakeFilter filter = new GuardiumSnowflakeFilter("test-id", config, context); + + Event event = FakeEventFactory.getSQLErrorEvent(); + event.setField(Constants.EVENT_TYPE, Constants.SQL_ERROR); + + Collection results = filter.filter(Collections.singletonList(event), null); + + Assert.assertEquals("Should return 1 event", 1, results.size()); + Event resultEvent = results.iterator().next(); + + Object guardRecordObj = resultEvent.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME); + Assert.assertNotNull("GuardRecord should be present", guardRecordObj); + + Gson gson = new Gson(); + JsonObject guardRecord = gson.fromJson(guardRecordObj.toString(), JsonObject.class); + JsonObject exception = guardRecord.getAsJsonObject("exception"); + + Assert.assertNotNull("Exception should be present for SQL error", exception); + // Should have SQL_ERROR, not UC_PARSER_ERROR or UC_AUDIT_ERROR + String exceptionType = exception.get("exceptionTypeId").getAsString(); + Assert.assertNotEquals("Should not be UC_PARSER_ERROR", + Constants.UC_PARSER_ERROR, exceptionType); + Assert.assertNotEquals("Should not be UC_AUDIT_ERROR", + Constants.UC_AUDIT_ERROR, exceptionType); + } + + @Test + public void testValidLoginFailedEventProcessedCorrectly() { + // Test that valid LOGIN_FAILED event is processed without UC errors + GuardiumSnowflakeFilter filter = new GuardiumSnowflakeFilter("test-id", config, context); + + Event event = FakeEventFactory.getAuthErrorEvent(); + event.setField(Constants.EVENT_TYPE, Constants.LOGIN_FAILED); + + Collection results = filter.filter(Collections.singletonList(event), null); + + Assert.assertEquals("Should return 1 event", 1, results.size()); + Event resultEvent = results.iterator().next(); + + Object guardRecordObj = resultEvent.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME); + Assert.assertNotNull("GuardRecord should be present", guardRecordObj); + + Gson gson = new Gson(); + JsonObject guardRecord = gson.fromJson(guardRecordObj.toString(), JsonObject.class); + JsonObject exception = guardRecord.getAsJsonObject("exception"); + + Assert.assertNotNull("Exception should be present for login failure", exception); + // Should have LOGIN_FAILED, not UC_PARSER_ERROR or UC_AUDIT_ERROR + String exceptionType = exception.get("exceptionTypeId").getAsString(); + Assert.assertNotEquals("Should not be UC_PARSER_ERROR", + Constants.UC_PARSER_ERROR, exceptionType); + Assert.assertNotEquals("Should not be UC_AUDIT_ERROR", + Constants.UC_AUDIT_ERROR, exceptionType); + } + + @Test + public void testEmptyEventTypeCreatesAuditError() { + // Test that empty event_type creates UC_AUDIT_ERROR + GuardiumSnowflakeFilter filter = new GuardiumSnowflakeFilter("test-id", config, context); + + Event event = new org.logstash.Event(); + event.setField(Constants.EVENT_TYPE, ""); + event.setField(Constants.USER_NAME, "test_user"); + + Collection results = filter.filter(Collections.singletonList(event), null); + + Assert.assertEquals("Should return 1 event", 1, results.size()); + Event resultEvent = results.iterator().next(); + + Object guardRecordObj = resultEvent.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME); + Assert.assertNotNull("GuardRecord should be present", guardRecordObj); + + Gson gson = new Gson(); + JsonObject guardRecord = gson.fromJson(guardRecordObj.toString(), JsonObject.class); + JsonObject exception = guardRecord.getAsJsonObject("exception"); + + Assert.assertNotNull("Exception should be present", exception); + Assert.assertEquals("Should be UC_AUDIT_ERROR", + Constants.UC_AUDIT_ERROR, + exception.get("exceptionTypeId").getAsString()); + } + + @Test + public void testCaseInsensitiveEventType() { + // Test that event_type is case-insensitive (SUCCESS, success, SuCcEsS should all work) + GuardiumSnowflakeFilter filter = new GuardiumSnowflakeFilter("test-id", config, context); + + Event event = FakeEventFactory.getSuccessEvent(); + event.setField(Constants.EVENT_TYPE, "success"); // lowercase + + Collection results = filter.filter(Collections.singletonList(event), null); + + Assert.assertEquals("Should return 1 event", 1, results.size()); + Event resultEvent = results.iterator().next(); + + Object guardRecordObj = resultEvent.getField(GuardConstants.GUARDIUM_RECORD_FIELD_NAME); + Assert.assertNotNull("GuardRecord should be present", guardRecordObj); + + Gson gson = new Gson(); + JsonObject guardRecord = gson.fromJson(guardRecordObj.toString(), JsonObject.class); + + // Should process successfully without UC errors + if (guardRecord.has("exception") && !guardRecord.get("exception").isJsonNull()) { + JsonObject exception = guardRecord.getAsJsonObject("exception"); + if (exception.has("exceptionTypeId")) { + String exceptionType = exception.get("exceptionTypeId").getAsString(); + Assert.assertNotEquals("Should not be UC_PARSER_ERROR", + Constants.UC_PARSER_ERROR, exceptionType); + Assert.assertNotEquals("Should not be UC_AUDIT_ERROR", + Constants.UC_AUDIT_ERROR, exceptionType); + } + } + } +} \ No newline at end of file diff --git a/filter-plugin/logstash-filter-snowflake-guardium/src/test/java/com/ibm/guardium/snowflakedb/SQLErrorEventParserTest.java b/filter-plugin/logstash-filter-snowflake-guardium/src/test/java/com/ibm/guardium/snowflakedb/SQLErrorEventParserTest.java index 1b8bec6e9..e05672437 100644 --- a/filter-plugin/logstash-filter-snowflake-guardium/src/test/java/com/ibm/guardium/snowflakedb/SQLErrorEventParserTest.java +++ b/filter-plugin/logstash-filter-snowflake-guardium/src/test/java/com/ibm/guardium/snowflakedb/SQLErrorEventParserTest.java @@ -9,8 +9,11 @@ import com.ibm.guardium.snowflakedb.exceptions.ParseException; import com.ibm.guardium.snowflakedb.parser.Parser; import com.ibm.guardium.snowflakedb.parser.SQLErrorEventParser; +import com.ibm.guardium.snowflakedb.parser.SuccessEventParser; import com.ibm.guardium.snowflakedb.utils.Constants; import com.ibm.guardium.universalconnector.commons.structures.Record; +import com.ibm.guardium.universalconnector.commons.structures.SessionLocator; +import org.apache.commons.lang3.StringUtils; import org.junit.Assert; import org.junit.Test; import org.logstash.Event; @@ -85,4 +88,21 @@ public void testSQLError(){ ex.printStackTrace(); } } + + @Test + public void testSessionIDWhenClientAndServerSessionNotPresent() throws ParseException { + Event e = FakeEventFactory.getSQLErrorEvent(); + e.remove(Constants.SESSION_ID); + e.remove(Constants.CLIENT_IP); + e.remove(Constants.SERVER_IP); + + Parser parser = new SQLErrorEventParser(); + Record record = parser.parseRecord(e.toMap()); + SessionLocator sessionLocator = record.getSessionLocator(); + + Assert.assertEquals(StringUtils.EMPTY, record.getSessionId()); + Assert.assertEquals(-1, sessionLocator.getClientPort()); + Assert.assertEquals(443, sessionLocator.getServerPort()); + } + } diff --git a/filter-plugin/logstash-filter-snowflake-guardium/src/test/java/com/ibm/guardium/snowflakedb/SuccessEventParserTest.java b/filter-plugin/logstash-filter-snowflake-guardium/src/test/java/com/ibm/guardium/snowflakedb/SuccessEventParserTest.java index 3c4c2e4e1..0f8728394 100644 --- a/filter-plugin/logstash-filter-snowflake-guardium/src/test/java/com/ibm/guardium/snowflakedb/SuccessEventParserTest.java +++ b/filter-plugin/logstash-filter-snowflake-guardium/src/test/java/com/ibm/guardium/snowflakedb/SuccessEventParserTest.java @@ -11,6 +11,8 @@ import com.ibm.guardium.snowflakedb.parser.SuccessEventParser; import com.ibm.guardium.snowflakedb.utils.Constants; import com.ibm.guardium.universalconnector.commons.structures.Record; +import com.ibm.guardium.universalconnector.commons.structures.SessionLocator; +import org.apache.commons.lang3.StringUtils; import org.junit.Assert; import org.junit.Test; import org.logstash.Event; @@ -83,6 +85,22 @@ public void testSuccessEvent(){ } } + @Test + public void testSessionIDWhenClientAndServerSessionNotPresent() throws ParseException { + Event e = FakeEventFactory.getSuccessEvent(); + e.remove(Constants.SESSION_ID); + e.remove(Constants.CLIENT_IP); + e.remove(Constants.SERVER_IP); + + Parser parser = new SuccessEventParser(); + Record record = parser.parseRecord(e.toMap()); + SessionLocator sessionLocator = record.getSessionLocator(); + + Assert.assertEquals(StringUtils.EMPTY, record.getSessionId()); + Assert.assertEquals(-1, sessionLocator.getClientPort()); + Assert.assertEquals(443, sessionLocator.getServerPort()); + } + @Test public void testGetTime(){ Event e = FakeEventFactory.getSuccessEvent(); diff --git a/filter-plugin/logstash-filter-teradatadb-guardium/CHANGELOG.md b/filter-plugin/logstash-filter-teradatadb-guardium/CHANGELOG.md index 63ce38489..637037d6e 100644 --- a/filter-plugin/logstash-filter-teradatadb-guardium/CHANGELOG.md +++ b/filter-plugin/logstash-filter-teradatadb-guardium/CHANGELOG.md @@ -1,6 +1,12 @@ # Changelog Notable changes will be documented in this file. +## [1.1.4] +- Change DB protocol from "TeradataDB native audit" to "TeradataDB" + +## [1.1.2] +- Replace "TERADATA" with "TRD" for SERVER_TYPE_STRING +- ## [1.1.1] ### Added diff --git a/filter-plugin/logstash-filter-teradatadb-guardium/LogstashREADME.md b/filter-plugin/logstash-filter-teradatadb-guardium/LogstashREADME.md new file mode 100644 index 000000000..7cf74c224 --- /dev/null +++ b/filter-plugin/logstash-filter-teradatadb-guardium/LogstashREADME.md @@ -0,0 +1,167 @@ +# Teradata-Guardium Logstash filter plug-in +### Meet Teradata + +* Tested versions: 16.2, 17.2 and 20.0 +* Environment: On-premises, VCL on AWS and Azure, VCE on Azure, AWS and GCP. +* Supported inputs: JDBC (pull) +* Supported Guardium versions: + * Guardium Data Protection: 11.4 and later + +This is a [Logstash](https://github.com/elastic/logstash) filter plug-in for the universal connector that is featured in IBM Security Guardium. It parses events and messages from the Teradata audit log into a [Guardium record](https://github.com/IBM/universal-connectors/raw/main/common/src/main/java/com/ibm/guardium/universalconnector/commons/structures/Record.java) instance (which is a standard structure comprised of several parts). The information is then sent over to Guardium. Guardium records include the accessor (the person who tried to access the data), session, data, and exceptions. If there are no errors, the data contains details about the query "construct". The construct details the main action (verb) and collections (objects) involved. + +The plug-in is free and open-source (Apache 2.0). It can be used as a starting point to develop additional filter plug-ins for Guardium universal connector. + + +## 1. Configuring the Teradata server + +There are multiple ways to install a Teradata server. For this example, we will assume that we already have a working Teradata server setup. + +## 2. Enabling Auditing +1. Connect to the Teradata server using SSH. + +2. Login with the dbc user (or any other user) of teradata that has access to DBQLAccessMacro using bteq. The commands to log in with dbc user commands are: + ```bteq .logon /dbc,``` + +In the above command, give the password for the dbc user. + +4. Create a user to read logs from audit tables through the logstash JDBC input plug-in. + + CREATE USER AS PERMANENT = 100000000 BYTES PASSWORD = "" + +5. To grant read access of objects inside dbc user to above created user, execute the below command:- + + GRANT SELECT ON "dbc" TO ""; + +6. There are mutiple ways to enable DBQL Query Logging. Query logging includes a variety of table/view combinations in the DBC database. Logging for users, accounts, and applications should be used when it is really required. Query logging can be enabled for all users or for specific users. + +To enable Auditing, use a command like this one: + + BEGIN QUERY LOGGING WITH SQL LIMIT SQLTEXT=0 ON ALL; + +DBQL Query Logging can be explored further in this [document](https://docs.teradata.com/r/qOek~PvFMDdCF0yyBN6zkA/f7yJJ4siIiBUpoQVvvAwpQ). + +6. Type "exit;" to get out of bteq terminal. + +7. Set the database time zone with two dbscontrol fields: + + "18. System TimeZone String" must be set to the timezone which we want to configure for our database. + "57. TimeDateWZControl " must be set to 2. + +8. Close the terminal. + +9. We can verify that a logging rule is created in a table. Log in with dbc user. You can choose to trigger the below query through any client utility. + In this case we are using Teradata Studio Express. + + select * from DBC.DBQLRulesV; +### Notes: +For **Teradata VCE on Azure** or **Teradata VCL on AWS**, run the following query to turn the audit log on. +``` +BEGIN QUERY LOGGING WITH SQL LIMIT SQLTEXT=0 ON ALL; +``` +To check if the audit log is on/off, run: +``` +select * from DBC.DBQLRulesV; +``` +## 3. Steps to disable Auditing +Auditing can be disabled similarly to how we enabled auditing by logging in with dbc user or any other user that has access to DBQLAccessMacro. + +To disable query logging, execute the below command:- + + END QUERY LOGGING WITH SQL LIMIT SQLTEXT=0 ON ALL; + + +## 4. Archiving and Deleting DBQL Logs + +There are many ways to archive and delete DBQL logs. One of the ways is depicted below:- + +Use the following steps to delete old log data from system tables manually: + +It is recommended, though not necessary, to disable DBQL logging before you perform clean up activities on the logs. Otherwise, the delete process locks the DBQL table and if DBQL needs to flush a cache to the same table to continue logging queries, the whole system could experience a slow-down. + +***Note: You cannot delete data that is less than 30 days old.*** + +1. To back up log data + a. Create a duplicate log table in another database using the Copy Table syntax for the CREATE TABLE statement. + CT DBC.tablename AS databasename.tablename + b. Back up the table to tape storage in accordance with your site backup policy. + c. Drop the duplicate table using a DROP TABLE statement. +2. Log on to Teradata Studio as DBADMIN or another administrative user with DELETE privileges on database DBC. +3. In the Query window, enter an SQL statement to purge old log entries. For example: + + DELETE FROM DBC.object_name WHERE (Date - LogDate) > number_of_days ; + +Examples for using above query:- +DELETE FROM DBC.DBQLOGTBL WHERE (DATE '2021-12-16' - cast(starttime as DATE)) > 30 ; +DELETE FROM DBC.DBQLSqlTbl WHERE (DATE '2021-12-16' - cast(collecttimestamp as DATE)) > 30 ; + + +#### Limitations: + +• Teradata sniffer parser does not parse below listed operations properly. Hence this plug-in does not support these operations: + +1] User Management + +2] DBQL Queries + +3] Timestamp configuration + +4] Cast operations + +5] Stored Procedure and User Defined Functions + +6] DBQL tables do not capture 100% of Teradata workload due to configurable filters, aggregation options, memory caching with periodic writes (e.g., every 10 minutes via DBQLFlushRate), and potential data loss during system restarts before cache flush. Reference: https://www.dwhpro.com/teradata-query-logging-dbql/ + +• The Teradata auditing does not audit authentication failure(Login Failed) operations. + +• The **Client HostName** field cannot be mapped with TeradataDB audit logs. + +• In case of EC2 guardium instance, Teradata traffic took more time (25-30 min) to populate data in full sql Report. + +• This plug-in supports queries that are approximately 32,000 characters long. When the count of characters in a query exceed the given count, the remaining part of the query is stored in other rows. This is why the SQLTextInfo column of the table DBC.DBQLSqlTbl has more than one row per QueryID. + +• Client IP and Server IP are retrieved from DBC.QryLogClientAttrV view using ClientIPAddrByClient and ServerIPAddrByServer fields respectively, as recommended by Teradata support. The deprecated logonsource field is no longer used for IP address retrieval. + +For more information on DBC.QryLogClientAttrV, please refer to this [documentation](https://docs.teradata.com/r/Enterprise_IntelliFlex_VMware/Data-Dictionary/Views-Reference/QryLogClientAttrV). + +## 5. Configuring the Teradata filters in Guardium + +The Guardium universal connector is the Guardium entry point for native audit logs. The universal connector identifies and parses received events, and then converts them to a standard Guardium format. The output of the universal connector is forwarded to the Guardium sniffer on the collector, for policy and auditing enforcements. Configure Guardium to read the native audit logs by customizing the Teradata template. + +#### Before you begin + +•  Configure the policies you require. See [policies](/docs/#policies) for more information. + +• You must have permission for the S-Tap Management role. The admin user includes this role by default. + + +• Download the [logstash-filter-teradatadb_guardium_plugin_filter.zip](./TeradataOverJdbcPackage/logstash-filter-teradatadb_guardium_plugin_filter.zip) plug-in. (Do not unzip the offline-package file throughout the procedure). + +• Download driver jar - Go to the URL https://downloads.teradata.com/download/connectivity/jdbc-driver and download the zip/tar for required version. After extracting the downloaded zip/tar, there will be a jar file. + + +#### Procedure: + +1. On the collector, go to Setup > Tools and Views > Configure Universal Connector. +2. First enable the Universal Guardium connector, if it is disabled already. +3. Click ```Upload File``` and upload the jar/jars which you downloaded from the teradata website. +4. Click ```Upload File``` and select the offline [logstash-filter-teradatadb_guardium_plugin_filter.zip](./TeradataOverJdbcPackage/logstash-filter-teradatadb_guardium_plugin_filter.zip) plug-in. After it is uploaded, click ```OK```. This is not necessary for Guardium Data Protection v11.0p490 or later, v11.0p540 or later, v12.0 or later. +5. Click the Plus sign to open the ```Connector Configuration``` dialog box. +6. Type a name in the Connector name field. +7. Update the input section to add the details from the [teradataJDBC.conf](./TeradataOverJdbcPackage/teradataJDBC.conf) file's input part, omitting the keyword "input{" at the beginning and its corresponding "}" at the end. Provide the details for database server name, username, and password that are required for connecting with JDBC. +8. Update the filter section to add the details from the [teradataJDBC.conf](./TeradataOverJdbcPackage/teradataJDBC.conf) file's filter part, omitting the keyword "filter{" at the beginning and its corresponding "}" at the end. Provide the same database server name that you gave in the above step against the Server_Hostname attribute in the filter section. +9. The "type" fields should match in the input and the filter configuration sections. This field should be unique for every individual connector added. +10. If you are using two JDBC plug-ins on the same machine, the last_run_metadata_path file name should be different. +11. Click ```Save```. Guardium validates the new connector, and enables the universal connector if it was disabled. After it is validated, the connector appears in the ```Configure Universal Connector``` page. + +#### Note: + +• ```Records Affected``` column has a valid value for the Select Queries only, for other type of queries it is set to -1. + +• If there is a requirement of having the number of Records Affected from a Select Query in a FULL SQL report, follow the below steps : + +#### Procedure: +1. On the collector, go to Activity Monitoring > Inspection Engines. +2. Check Default capture value, Log Records Affected and Inspect Returned data checkboxes, if unchecked already. +3. Click on Apply. +4. Click on Restart Inspection Engines. +5. To add the column in the Full SQL Report, select the Attribute ```Records Affected``` listed under the Entity ```FULL SQL```. diff --git a/filter-plugin/logstash-filter-teradatadb-guardium/README.md b/filter-plugin/logstash-filter-teradatadb-guardium/README.md index b9ee964d3..aea85b341 100644 --- a/filter-plugin/logstash-filter-teradatadb-guardium/README.md +++ b/filter-plugin/logstash-filter-teradatadb-guardium/README.md @@ -1,161 +1,10 @@ -# Teradata-Guardium Logstash filter plug-in -### Meet Teradata -* Tested versions: 16.20, 17.5 -* Environment: Cloud, On-premise -* Supported inputs: JDBC (pull) -* Supported Guardium versions: - * Guardium Data Protection: 11.4 and above +# Teradata Universal Connector -This is a [Logstash](https://github.com/elastic/logstash) filter plug-in for the universal connector that is featured in IBM Security Guardium. It parses events and messages from the Teradata audit log into a [Guardium record](https://github.com/IBM/universal-connectors/raw/main/common/src/main/java/com/ibm/guardium/universalconnector/commons/structures/Record.java) instance (which is a standard structure comprised of several parts). The information is then sent over to Guardium. Guardium records include the accessor (the person who tried to access the data), session, data, and exceptions. If there are no errors, the data contains details about the query "construct". The construct details the main action (verb) and collections (objects) involved. +## Follow this link to set up and use Teradata Universal Connector over JDBC Logstash Plugin -The plug-in is free and open-source (Apache 2.0). It can be used as a starting point to develop additional filter plug-ins for Guardium universal connector. +[TeradataOverJDBC](./LogstashREADME.md) +## Follow this link to set up and use Teradata Universal Connector over JDBC Connect -## 1. Configuring the Teradata server +[TeradataOverConnectJdbc](../../docs/KafkaBasedUCs/TeradataJDBCKafkaConnect.md) -There are multiple ways to install a Teradata server. For this example, we will assume that we already have a working Teradata server setup. - -## 2. Enabling Auditing - -1. Connect to the Teradata server using SSH. - -2. Login with the dbc user (or any other user) of teradata that has access to DBQLAccessMacro using bteq. The commands to log in with dbc user commands are: - ```bteq .logon /dbc,``` - -In the above command, give the password for the dbc user. - -4. Create a user to read logs from audit tables through the logstash JDBC input plug-in. - - CREATE USER AS PERMANENT = 100000000 BYTES PASSWORD = "" - -5. To grant read access of objects inside dbc user to above created user, execute the below command:- - - GRANT SELECT ON "dbc" TO ""; - -6. There are mutiple ways to enable DBQL Query Logging. Query logging includes a variety of table/view combinations in the DBC database. Logging for users, accounts, and applications should be used when it is really required. Query logging can be enabled for all users or for specific users. - -To enable Auditing, use a command like this one: - - BEGIN QUERY LOGGING WITH SQL LIMIT SQLTEXT=0 ON ALL; - -DBQL Query Logging can be explored further in this [document](https://docs.teradata.com/r/qOek~PvFMDdCF0yyBN6zkA/f7yJJ4siIiBUpoQVvvAwpQ). - -6. Type "exit;" to get out of bteq terminal. - -7. Set the database time zone with two dbscontrol fields: - - "18. System TimeZone String" must be set to the timezone which we want to configure for our database. - "57. TimeDateWZControl " must be set to 2. - -8. Close the terminal. - -9. We can verify that a logging rule is created in a table. Log in with dbc user. You can choose to trigger the below query through any client utility. - In this case we are using Teradata Studio Express. - - select * from DBC.DBQLRulesV; - -## 3. Steps to disable Auditing -Auditing can be disabled similarly to how we enabled auditing by logging in with dbc user or any other user that has access to DBQLAccessMacro. - -To disable query logging, execute the below command:- - - END QUERY LOGGING WITH SQL LIMIT SQLTEXT=0 ON ALL; - - -## 4. Archiving and Deleting DBQL Logs - -There are many ways to archive and delete DBQL logs. One of the ways is depicted below:- - -Use the following steps to delete old log data from system tables manually: - -It is recommended, though not necessary, to disable DBQL logging before you perform clean up activities on the logs. Otherwise, the delete process locks the DBQL table and if DBQL needs to flush a cache to the same table to continue logging queries, the whole system could experience a slow-down. - -***Note: You cannot delete data that is less than 30 days old.*** - -1. To back up log data - a. Create a duplicate log table in another database using the Copy Table syntax for the CREATE TABLE statement. - CT DBC.tablename AS databasename.tablename - b. Back up the table to tape storage in accordance with your site backup policy. - c. Drop the duplicate table using a DROP TABLE statement. -2. Log on to Teradata Studio as DBADMIN or another administrative user with DELETE privileges on database DBC. -3. In the Query window, enter an SQL statement to purge old log entries. For example: - - DELETE FROM DBC.object_name WHERE (Date - LogDate) > number_of_days ; - -Examples for using above query:- - DELETE FROM DBC.DBQLOGTBL WHERE (DATE '2021-12-16' - cast(starttime as DATE)) > 30 ; - DELETE FROM DBC.DBQLSqlTbl WHERE (DATE '2021-12-16' - cast(collecttimestamp as DATE)) > 30 ; - - -#### Limitations: - -• Teradata sniffer parser does not parse below listed operations properly. Hence this plug-in does not support these operations: - -1] User Management - -2] DBQL Queries - -3] Timestamp configuration - -4] Cast operations - -5] Stored Procedure and User Defined Functions - -• The Teradata auditing does not audit authentication failure(Login Failed) operations. - -• Following important field couldn't mapped with TeradataDB audit logs. - -1] Client HostName : Not Available with audit logs. - -2] Database Name : Not Available with audit logs. - -• In case of EC2 guardium instance, Teradata traffic took more time (25-30 min) to populate data in full sql Report. - -• This plug-in supports queries that are approximately 32,000 characters long. When the count of characters in a query exceed the given count, the remaining part of the query is stored in other rows. This is why the SQLTextInfo column of the table DBC.DBQLSqlTbl has more than one row per QueryID. - -• serverIp is hardcoded to "0.0.0.0" in this plugin, as tables referred in configuration file do not have an attribute that directly holds actual serverIp value but that can be checked from column LogonSource(from DBC.DBQLOGTBL ) or sourceProgram attribute. - -For more information on how to check the serverIp from LogonSource, please refer this [doc](https://docs.teradata.com/r/ANYCOtbX9Q1iyd~Uiok8gA/VPQKKhAyOf6hzUc4sfciIQ) - -## 5. Configuring the Teradata filters in Guardium - -The Guardium universal connector is the Guardium entry point for native audit logs. The universal connector identifies and parses received events, and then converts them to a standard Guardium format. The output of the universal connector is forwarded to the Guardium sniffer on the collector, for policy and auditing enforcements. Configure Guardium to read the native audit logs by customizing the Teradata template. - -#### Before you begin - -•  Configure the policies you require. See [policies](/docs/#policies) for more information. - -• You must have permission for the S-Tap Management role. The admin user includes this role by default. - - -• Download the [logstash-filter-teradatadb_guardium_plugin_filter.zip](./TeradataOverJdbcPackage/logstash-filter-teradatadb_guardium_plugin_filter.zip) plug-in. (Do not unzip the offline-package file throughout the procedure). - -• Download driver jar - Go to the URL https://downloads.teradata.com/download/connectivity/jdbc-driver and download the zip/tar for required version. After extracting the downloaded zip/tar, there will be a jar file. - - -#### Procedure: - -1. On the collector, go to Setup > Tools and Views > Configure Universal Connector. -2. First enable the Universal Guardium connector, if it is disabled already. -3. Click ```Upload File``` and upload the jar/jars which you downloaded from the teradata website. -4. Click ```Upload File``` and select the offline [logstash-filter-teradatadb_guardium_plugin_filter.zip](./TeradataOverJdbcPackage/logstash-filter-teradatadb_guardium_plugin_filter.zip) plug-in. After it is uploaded, click ```OK```. This is not necessary for Guardium Data Protection v11.0p490 or later, v11.0p540 or later, v12.0 or later. -5. Click the Plus sign to open the ```Connector Configuration``` dialog box. -6. Type a name in the Connector name field. -7. Update the input section to add the details from the [teradataJDBC.conf](./TeradataOverJdbcPackage/teradataJDBC.conf) file's input part, omitting the keyword "input{" at the beginning and its corresponding "}" at the end. Provide the details for database server name, username, and password that are required for connecting with JDBC. -8. Update the filter section to add the details from the [teradataJDBC.conf](./TeradataOverJdbcPackage/teradataJDBC.conf) file's filter part, omitting the keyword "filter{" at the beginning and its corresponding "}" at the end. Provide the same database server name that you gave in the above step against the Server_Hostname attribute in the filter section. -9. The "type" fields should match in the input and the filter configuration sections. This field should be unique for every individual connector added. -10. If you are using two JDBC plug-ins on the same machine, the last_run_metadata_path file name should be different. -11. Click ```Save```. Guardium validates the new connector, and enables the universal connector if it was disabled. After it is validated, the connector appears in the ```Configure Universal Connector``` page. - -#### Note: - -• ```Records Affected``` column has a valid value for the Select Queries only, for other type of queries it is set to -1. - -• If there is a requirement of having the number of Records Affected from a Select Query in a FULL SQL report, follow the below steps : - -#### Procedure: -1. On the collector, go to Activity Monitoring > Inspection Engines. -2. Check Default capture value, Log Records Affected and Inspect Returned data checkboxes, if unchecked already. -3. Click on Apply. -4. Click on Restart Inspection Engines. -5. To add the column in the Full SQL Report, select the Attribute ```Records Affected``` listed under the Entity ```FULL SQL```. \ No newline at end of file diff --git a/filter-plugin/logstash-filter-teradatadb-guardium/TeradataOverJdbcPackage/logstash-filter-teradatadb_guardium_plugin_filter.zip b/filter-plugin/logstash-filter-teradatadb-guardium/TeradataOverJdbcPackage/logstash-filter-teradatadb_guardium_plugin_filter.zip index 89a44d758..d7a9e4ae0 100644 Binary files a/filter-plugin/logstash-filter-teradatadb-guardium/TeradataOverJdbcPackage/logstash-filter-teradatadb_guardium_plugin_filter.zip and b/filter-plugin/logstash-filter-teradatadb-guardium/TeradataOverJdbcPackage/logstash-filter-teradatadb_guardium_plugin_filter.zip differ diff --git a/filter-plugin/logstash-filter-teradatadb-guardium/TeradataOverJdbcPackage/teradataJDBC.conf b/filter-plugin/logstash-filter-teradatadb-guardium/TeradataOverJdbcPackage/teradataJDBC.conf index e046be399..d9c055939 100644 --- a/filter-plugin/logstash-filter-teradatadb-guardium/TeradataOverJdbcPackage/teradataJDBC.conf +++ b/filter-plugin/logstash-filter-teradatadb-guardium/TeradataOverJdbcPackage/teradataJDBC.conf @@ -1,12 +1,12 @@ input { jdbc { jdbc_driver_class => "com.teradata.jdbc.TeraDriver" - jdbc_connection_string => "jdbc:teradata:///dbc" + jdbc_connection_string => "jdbc:teradata:///DATABASE=,DBS_PORT=" jdbc_user => "" jdbc_password => "" schedule => "*/1 * * * *" clean_run => false - statement => "select SessionID, CAST((((CAST(CAST(CAST(STARTTIME At 'GMT' AS FORMAT 'YYYY-MM-DD') AS VARCHAR(14)) AS DATE FORMAT 'YYYY-MM-DD') - DATE '1970-01-01')*86400 * 1000) + ((EXTRACT(HOUR FROM StartTime at 'gmt')*3600*1000) + (EXTRACT(MINUTE FROM StartTime at 'gmt')*60*1000) + (EXTRACT(SECOND FROM db.STARTTIME AT TIME ZONE 'GMT')*1000)))AS BIGINT) AS t1, StartTime at 'gmt' as t2, trim(ClientAddr) as ClientAddr,UserName,db.QueryID,SqlTextInfo,trim(ErrorText) as ErrorText,trim(AppID) as LogonSource,trim(ClientID) as ClientID, NumResultRows as rowsReturned from dbc.qrylogv db INNER JOIN DBC.QryLogSQLV dbs ON db.QueryID = dbs.QueryID where t1 > :sql_last_value and ClientID <> 'GUC' and sqltextinfo NOT IN ('select * from DBC.Database_Default_JournalsV','COMMIT WORK','SELECT 1','SELECT USER','--isValid') and sqltextinfo not like '%LogonSource%' and sqltextinfo not like'%ApplicationName%' and sqltextinfo not like'%LOCK%' and sqltextinfo not like'%TYPE%' and sqltextinfo not like'%WORK%' order by t1;" + statement => "select SessionID, CAST((((CAST(CAST(CAST(STARTTIME At 'GMT' AS FORMAT 'YYYY-MM-DD') AS VARCHAR(14)) AS DATE FORMAT 'YYYY-MM-DD') - DATE '1970-01-01')*86400 * 1000) + ((EXTRACT(HOUR FROM StartTime at 'gmt')*3600*1000) + (EXTRACT(MINUTE FROM StartTime at 'gmt')*60*1000) + (EXTRACT(SECOND FROM db.STARTTIME AT TIME ZONE 'GMT')*1000)))AS BIGINT) AS t1, StartTime at 'gmt' as t2, trim(qca.ClientIPAddrByClient) as ClientAddr, trim(qca.ServerIPAddrByServer) as ServerAddr,db.UserName,db.QueryID,SqlTextInfo,trim(ErrorText) as ErrorText,trim(AppID) as LogonSource,trim(ClientID) as ClientID, NumResultRows as rowsReturned from dbc.qrylogv db INNER JOIN DBC.QryLogSQLV dbs ON db.QueryID = dbs.QueryID INNER JOIN DBC.QryLogClientAttrV qca ON db.QueryID = qca.QueryID where t1 > :sql_last_value and ClientID <> 'GUC' and sqltextinfo NOT IN ('select * from DBC.Database_Default_JournalsV','COMMIT WORK','SELECT 1','SELECT USER','--isValid') and sqltextinfo not like '%LogonSource%' and sqltextinfo not like'%ApplicationName%' and sqltextinfo not like'%LOCK%' and sqltextinfo not like'%TYPE%' and sqltextinfo not like'%WORK%' order by t1;" use_column_value => true tracking_column => "t1" tracking_column_type => "numeric" @@ -32,6 +32,7 @@ filter { "[Session_ID]" => "%{sessionid}" "[Time_Field]" => "%{t2}" "[Client_IP]" => "%{clientaddr}" + "[Server_IP]" => "%{serveraddr}" "[User_Name]" => "%{username}" "[Sql_Text_Info]" => "%{sqltextinfo}" "[Logon_Source]" => "%{logonsource}" @@ -44,6 +45,6 @@ filter { teradatadb_guardium_plugin_filter{} - mutate { remove_field => ["@version","type","@timestamp","Error_Text","errortext","Session_ID","sessionid","Time_Field","starttime","Client_IP","clientaddr","User_Name","username","Sql_Text_Info","sqltextinfo","Logon_Source","logonsource","Query_ID","queryid","Server_Hostname","t1","t2"] } + mutate { remove_field => ["@version","type","@timestamp","Error_Text","errortext","Session_ID","sessionid","Time_Field","starttime","Client_IP","clientaddr","Server_IP","serveraddr","User_Name","username","Sql_Text_Info","sqltextinfo","Logon_Source","logonsource","Query_ID","queryid","Server_Hostname","t1","t2"] } } } diff --git a/filter-plugin/logstash-filter-teradatadb-guardium/VERSION b/filter-plugin/logstash-filter-teradatadb-guardium/VERSION index 8cfbc905b..1b87bcd0b 100644 --- a/filter-plugin/logstash-filter-teradatadb-guardium/VERSION +++ b/filter-plugin/logstash-filter-teradatadb-guardium/VERSION @@ -1 +1 @@ -1.1.1 \ No newline at end of file +1.1.4 \ No newline at end of file diff --git a/filter-plugin/logstash-filter-teradatadb-guardium/build.gradle b/filter-plugin/logstash-filter-teradatadb-guardium/build.gradle index 4a62c047d..b5e9b241d 100644 --- a/filter-plugin/logstash-filter-teradatadb-guardium/build.gradle +++ b/filter-plugin/logstash-filter-teradatadb-guardium/build.gradle @@ -2,6 +2,30 @@ import java.nio.file.Files import static java.nio.file.StandardCopyOption.REPLACE_EXISTING apply plugin: 'java' + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + + dependencies { + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' + classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.1.0" + classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' + } +} + +def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" // =========================================================================== @@ -20,10 +44,10 @@ pluginInfo.pluginClass = "TeradatadbGuardiumPluginFilter" pluginInfo.pluginName = "teradatadb_guardium_plugin_filter" // must match the @LogstashPlugin annotation in the main plugin class // =========================================================================== -sourceCompatibility = 1.8 -targetCompatibility = 1.8 +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 -def jacocoVersion = '0.8.4' +def jacocoVersion = '0.8.11' // minimumCoverage can be set by Travis ENV def minimumCoverageStr = System.getenv("MINIMUM_COVERAGE") ?: "50.0%" @@ -33,27 +57,16 @@ if (minimumCoverageStr.endsWith("%")) { def minimumCoverage = Float.valueOf(minimumCoverageStr) / 100 -buildscript { - repositories { - maven { - url "https://plugins.gradle.org/m2/" - } - mavenCentral() - jcenter() - } - - dependencies { - classpath 'com.github.jengelman.gradle.plugins:shadow:4.0.4' - classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.0.0-RC2" - classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' - } -} - -def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); -def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) repositories { - mavenCentral() + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } } tasks.register("vendor"){ @@ -64,14 +77,13 @@ tasks.register("vendor"){ File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") projectJarFile.mkdirs() Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) - validatePluginJar(projectJarFile, project.group) } } apply plugin: 'com.github.johnrengelman.shadow' shadowJar { - classifier = null + archiveClassifier = null } dependencies { @@ -79,6 +91,7 @@ dependencies { implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: versions.dependencies.log4jCore implementation 'org.apache.commons:commons-lang3:' + versions.dependencies.commonsLang implementation 'com.google.code.gson:gson:' + versions.dependencies.gson + implementation group: 'commons-beanutils', name: 'commons-beanutils', version: versions.dependencies.commonsBeanutils implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "build/libs/logstash-core-*.*.*.jar") implementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "common-*.*.*.jar") @@ -103,6 +116,17 @@ tasks.withType(JavaCompile) { options.encoding = 'UTF-8' } +test { + jvmArgs = [ + '--add-opens=java.base/java.lang=ALL-UNNAMED', + '--add-opens=java.base/java.util=ALL-UNNAMED', + '--add-opens=java.base/java.text=ALL-UNNAMED', + '--add-opens=java.base/java.lang.reflect=ALL-UNNAMED', + '--add-opens=java.base/sun.util.resources=ALL-UNNAMED', + '--add-opens=java.base/sun.util.cldr=ALL-UNNAMED' + ] +} + tasks.register("generateRubySupportFiles") { @@ -139,18 +163,17 @@ apply plugin: "org.barfuin.gradle.jacocolog" jacoco { toolVersion = "${jacocoVersion}" - reportsDir = file("$buildDir/reports/jacoco") } jacocoTestReport { // You will see "Report -> file://...." at the end of a JaCoCo build // If no output, run this first: ./gradlew test reports { - html.enabled true - xml.enabled true - csv.enabled true - html.destination file("${buildDir}/reports/jacoco") - csv.destination file("${buildDir}/reports/jacoco/all.csv") + html.required = true + xml.required = true + csv.required = true + html.outputLocation = file("${buildDir}/reports/jacoco") + csv.outputLocation = file("${buildDir}/reports/jacoco/all.csv") } executionData.from fileTree(dir: "${buildDir}/jacoco/", includes: [ '**/*.exec' diff --git a/filter-plugin/logstash-filter-teradatadb-guardium/gradle/wrapper/gradle-wrapper.properties b/filter-plugin/logstash-filter-teradatadb-guardium/gradle/wrapper/gradle-wrapper.properties index aa991fcea..81aa1c044 100644 --- a/filter-plugin/logstash-filter-teradatadb-guardium/gradle/wrapper/gradle-wrapper.properties +++ b/filter-plugin/logstash-filter-teradatadb-guardium/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/filter-plugin/logstash-filter-teradatadb-guardium/src/main/java/com/ibm/guardium/teradatadb/Constants.java b/filter-plugin/logstash-filter-teradatadb-guardium/src/main/java/com/ibm/guardium/teradatadb/Constants.java index a25deddaf..802c433c9 100644 --- a/filter-plugin/logstash-filter-teradatadb-guardium/src/main/java/com/ibm/guardium/teradatadb/Constants.java +++ b/filter-plugin/logstash-filter-teradatadb-guardium/src/main/java/com/ibm/guardium/teradatadb/Constants.java @@ -6,12 +6,14 @@ public interface Constants { public static final String DEFAULT_IP = "0.0.0.0"; public static final int DEFAULT_PORT = -1; public static final String UNKNOWN_STRING = ""; - public static final String SERVER_TYPE_STRING = "TERADATA"; - public static final String DATA_PROTOCOL_STRING = "TeradataDB native audit"; + public static final String NOT_AVAILABLE = "N.A."; + public static final String SERVER_TYPE_STRING = "TRD"; + public static final String DATA_PROTOCOL_STRING = "TeradataDB"; public static final String TERADATA_LANGUAGE="TRD"; public static final String SESSION_ID = "Session_ID"; public static final String TIME_FIELD = "Time_Field"; public static final String CLIENT_IP = "Client_IP"; + public static final String SERVER_IP = "Server_IP"; public static final String USER_NAME = "User_Name"; public static final String SQL_TEXT_INFO = "Sql_Text_Info"; public static final String SQL_ERROR = "SQL_ERROR"; diff --git a/filter-plugin/logstash-filter-teradatadb-guardium/src/main/java/com/ibm/guardium/teradatadb/Parser.java b/filter-plugin/logstash-filter-teradatadb-guardium/src/main/java/com/ibm/guardium/teradatadb/Parser.java index a02be5f07..31da281a6 100644 --- a/filter-plugin/logstash-filter-teradatadb-guardium/src/main/java/com/ibm/guardium/teradatadb/Parser.java +++ b/filter-plugin/logstash-filter-teradatadb-guardium/src/main/java/com/ibm/guardium/teradatadb/Parser.java @@ -21,7 +21,7 @@ public static Record parseRecord(final Event e) throws ParseException { record.setSessionId(e.getField(Constants.SESSION_ID).toString()); - record.setDbName(Constants.UNKNOWN_STRING); + record.setDbName(Constants.NOT_AVAILABLE); record.setAppUserName(Constants.UNKNOWN_STRING); @@ -66,7 +66,7 @@ public static SessionLocator parseSessionLocator(final Event e) { sessionLocator.setClientIp(e.getField(Constants.CLIENT_IP).toString()); sessionLocator.setClientPort(Constants.DEFAULT_PORT); - sessionLocator.setServerIp(Constants.DEFAULT_IP); + sessionLocator.setServerIp(e.getField(Constants.SERVER_IP).toString()); sessionLocator.setServerPort(Constants.DEFAULT_PORT); sessionLocator.setIpv6(false); sessionLocator.setClientIpv6(Constants.UNKNOWN_STRING); @@ -90,7 +90,7 @@ public static Accessor parseAccessor(final Event e) { accessor.setSourceProgram(e.getField(Constants.LOGON_SOURCE).toString()); accessor.setClient_mac(Constants.UNKNOWN_STRING); accessor.setServerDescription(Constants.UNKNOWN_STRING); - accessor.setServiceName(Constants.UNKNOWN_STRING); + accessor.setServiceName(Constants.NOT_AVAILABLE); accessor.setServerOs(Constants.UNKNOWN_STRING); accessor.setServerHostName(e.getField(Constants.SERVER_HOSTNAME).toString()); accessor.setOsUser(e.getField(Constants.OS_USER).toString()); diff --git a/filter-plugin/logstash-filter-teradatadb-guardium/src/test/java/com/ibm/guardium/teradatadb/ParserTest.java b/filter-plugin/logstash-filter-teradatadb-guardium/src/test/java/com/ibm/guardium/teradatadb/ParserTest.java index e2a101d40..4e97583a5 100644 --- a/filter-plugin/logstash-filter-teradatadb-guardium/src/test/java/com/ibm/guardium/teradatadb/ParserTest.java +++ b/filter-plugin/logstash-filter-teradatadb-guardium/src/test/java/com/ibm/guardium/teradatadb/ParserTest.java @@ -20,119 +20,120 @@ public class ParserTest { Parser parser = new Parser(); Event e = new org.logstash.Event(); - - Event intitalizeEventObject() { - - e.setField(Constants.SESSION_ID, "6968"); - e.setField(Constants.TIME_FIELD, "2021-11-16T07:49:41.220Z"); - e.setField(Constants.CLIENT_IP, "9.211.127.160"); - e.setField(Constants.USER_NAME, "SYSDBA"); - e.setField(Constants.SERVER_HOSTNAME, "1.1.1.1"); - e.setField(Constants.SQL_TEXT_INFO, "select * from employee;"); - e.setField(Constants.ERROR_TEXT, null); - e.setField(Constants.LOGON_SOURCE, "(TCP/IP) c089 194.2.127.16 DBS-TERADATA1620.COM;DB-TERA CID=2D39989 " - + "AVT666744 JDBC17.10.00.14;1.8.0_202 01 LSS"); + + Event intitalizeEventObject() { + + e.setField(Constants.SESSION_ID, "6968"); + e.setField(Constants.TIME_FIELD, "2021-11-16T07:49:41.220Z"); + e.setField(Constants.CLIENT_IP, "9.211.127.160"); + e.setField(Constants.SERVER_IP, "1.1.1.1"); + e.setField(Constants.USER_NAME, "SYSDBA"); + e.setField(Constants.SERVER_HOSTNAME, "1.1.1.1"); + e.setField(Constants.SQL_TEXT_INFO, "select * from employee;"); + e.setField(Constants.ERROR_TEXT, null); + e.setField(Constants.LOGON_SOURCE, "(TCP/IP) c089 194.2.127.16 DBS-TERADATA1620.COM;DB-TERA CID=2D39989 " + + "AVT666744 JDBC17.10.00.14;1.8.0_202 01 LSS"); e.setField(Constants.OS_USER, "TESTUSER"); - return e; - } - - @Test + return e; + } + + @Test public void createQuery() throws ParseException { - - Event e=intitalizeEventObject(); - e.setField(Constants.SQL_TEXT_INFO, "CREATE SET TABLE EMPLOYEE,FALLBACK ( EmployeeNo INTEGER, FirstName VARCHAR(30), " - + "LastName VARCHAR(30), DOB DATE FORMAT 'YYYY-MM-DD', JoinedDate DATE FORMAT 'YYYY-MM-DD', " - + "DepartmentNo BYTEINT ) UNIQUE PRIMARY INDEX ( EmployeeNo );"); - + + Event e = intitalizeEventObject(); + e.setField(Constants.SQL_TEXT_INFO, "CREATE SET TABLE EMPLOYEE,FALLBACK ( EmployeeNo INTEGER, FirstName VARCHAR(30), " + + "LastName VARCHAR(30), DOB DATE FORMAT 'YYYY-MM-DD', JoinedDate DATE FORMAT 'YYYY-MM-DD', " + + "DepartmentNo BYTEINT ) UNIQUE PRIMARY INDEX ( EmployeeNo );"); + final Record record = Parser.parseRecord(e); Assert.assertEquals(record.getData().getConstruct(), null); } - - @Test + + @Test public void insertQuery() throws ParseException { - - Event e=intitalizeEventObject(); - e.setField(Constants.SQL_TEXT_INFO, "INSERT INTO Employee (EmployeeNo, FirstName, LastName, DOB, JoinedDate, DepartmentNo )" - + "VALUES ( 101, 'Mike', 'James', '1980-01-05', '2005-03-27', 01);"); - + + Event e = intitalizeEventObject(); + e.setField(Constants.SQL_TEXT_INFO, "INSERT INTO Employee (EmployeeNo, FirstName, LastName, DOB, JoinedDate, DepartmentNo )" + + "VALUES ( 101, 'Mike', 'James', '1980-01-05', '2005-03-27', 01);"); + final Record record = Parser.parseRecord(e); Assert.assertEquals(record.getData().getConstruct(), null); } - - @Test + + @Test public void selectQuery() throws ParseException { - - Event e=intitalizeEventObject(); - e.setField(Constants.SQL_TEXT_INFO, "SELECT A.EmployeeNo, A.DepartmentNo, B.NetPay FROM Employee A " - + "INNER JOIN Salary B ON (A.EmployeeNo = B. EmployeeNo);"); - + + Event e = intitalizeEventObject(); + e.setField(Constants.SQL_TEXT_INFO, "SELECT A.EmployeeNo, A.DepartmentNo, B.NetPay FROM Employee A " + + "INNER JOIN Salary B ON (A.EmployeeNo = B. EmployeeNo);"); + final Record record = Parser.parseRecord(e); Assert.assertEquals(record.getData().getConstruct(), null); Assert.assertEquals(record.getDbName(), record.getAccessor().getServiceName()); } - - - @Test + + + @Test public void updateQuery() throws ParseException { - - Event e=intitalizeEventObject(); - e.setField(Constants.SQL_TEXT_INFO, "UPDATE Employee SET DepartmentNo = 03 WHERE EmployeeNo = 101;"); - + + Event e = intitalizeEventObject(); + e.setField(Constants.SQL_TEXT_INFO, "UPDATE Employee SET DepartmentNo = 03 WHERE EmployeeNo = 101;"); + final Record record = Parser.parseRecord(e); Assert.assertEquals(record.getData().getConstruct(), null); } - - - @Test + + + @Test public void deleteQuery() throws ParseException { - - Event e=intitalizeEventObject(); - e.setField(Constants.SQL_TEXT_INFO, "DELETE FROM Employee WHERE EmployeeNo = 101;"); - + + Event e = intitalizeEventObject(); + e.setField(Constants.SQL_TEXT_INFO, "DELETE FROM Employee WHERE EmployeeNo = 101;"); + final Record record = Parser.parseRecord(e); Assert.assertEquals(record.getData().getConstruct(), null); } - + @Test public void testParseSessionLocator() throws ParseException { - Event e=intitalizeEventObject(); - SessionLocator sessionLocator = Parser.parseSessionLocator(e); - + Event e = intitalizeEventObject(); + SessionLocator sessionLocator = Parser.parseSessionLocator(e); + Assert.assertEquals("9.211.127.160", sessionLocator.getClientIp()); Assert.assertEquals(-1, sessionLocator.getClientPort()); Assert.assertEquals(false, sessionLocator.isIpv6()); - } - + } + @Test public void testParseAccessor() throws ParseException { - Event e=intitalizeEventObject(); - Accessor accessor = Parser.parseAccessor(e); - + Event e = intitalizeEventObject(); + Accessor accessor = Parser.parseAccessor(e); + Assert.assertEquals(Constants.DATA_PROTOCOL_STRING, accessor.getDbProtocol()); Assert.assertEquals(Constants.SERVER_TYPE_STRING, accessor.getServerType()); Assert.assertEquals(Constants.TERADATA_LANGUAGE, accessor.getLanguage()); Assert.assertEquals("TESTUSER", accessor.getOsUser()); } - + @Test public void testParseTimestamp() throws ParseException { - - Event e=intitalizeEventObject(); - Time time = Parser.parseTimestamp(e); - Assert.assertEquals(1637048981220L,time.getTimstamp()); - } - - @Test + + Event e = intitalizeEventObject(); + Time time = Parser.parseTimestamp(e); + Assert.assertEquals(1637048981220L, time.getTimstamp()); + } + + @Test public void testErrors() throws ParseException { - Event e=intitalizeEventObject(); + Event e = intitalizeEventObject(); - e.setField(Constants.SQL_TEXT_INFO, "select * from DBC.QryLog;"); - e.setField(Constants.ERROR_TEXT, "The user does not have SELECT access to DBC.QryLog."); - - final Record record = Parser.parseRecord(e); + e.setField(Constants.SQL_TEXT_INFO, "select * from DBC.QryLog;"); + e.setField(Constants.ERROR_TEXT, "The user does not have SELECT access to DBC.QryLog."); + + final Record record = Parser.parseRecord(e); - Assert.assertEquals(Constants.SQL_ERROR,record.getException().getExceptionTypeId()); + Assert.assertEquals(Constants.SQL_ERROR, record.getException().getExceptionTypeId()); Assert.assertEquals("The user does not have SELECT access to DBC.QryLog." - ,record.getException().getDescription()); + , record.getException().getDescription()); } } \ No newline at end of file diff --git a/filter-plugin/logstash-filter-teradatadb-guardium/src/test/java/com/ibm/guardium/teradatadb/TeradatadbGuardiumPluginFilterTest.java b/filter-plugin/logstash-filter-teradatadb-guardium/src/test/java/com/ibm/guardium/teradatadb/TeradatadbGuardiumPluginFilterTest.java index 8d37dcfb5..0f02cf379 100644 --- a/filter-plugin/logstash-filter-teradatadb-guardium/src/test/java/com/ibm/guardium/teradatadb/TeradatadbGuardiumPluginFilterTest.java +++ b/filter-plugin/logstash-filter-teradatadb-guardium/src/test/java/com/ibm/guardium/teradatadb/TeradatadbGuardiumPluginFilterTest.java @@ -27,11 +27,12 @@ public class TeradatadbGuardiumPluginFilterTest { final static TeradatadbGuardiumPluginFilter filter = new TeradatadbGuardiumPluginFilter("test-id", null, context); Event e = new org.logstash.Event(); - Event intitalizeEventObject() { - + Event intitalizeEventObject() { + e.setField(Constants.SESSION_ID, "6968"); e.setField(Constants.TIME_FIELD, "2021-11-16T07:49:41.220Z"); e.setField(Constants.CLIENT_IP, "194.2.127.16"); + e.setField(Constants.SERVER_IP, "10.0.0.1"); e.setField(Constants.USER_NAME, "SYSDBA"); e.setField(Constants.SERVER_HOSTNAME, "server.com"); e.setField(Constants.SQL_TEXT_INFO, "select * from employee;"); @@ -40,7 +41,7 @@ Event intitalizeEventObject() { + "AVT666744 JDBC17.10.00.14;1.8.0_202 01 LSS"); e.setField(Constants.OS_USER, "TESTUSER"); return e; - } + } /** * To feed Guardium universal connector, a "GuardRecord" field must exist. @@ -67,6 +68,31 @@ public void testGuardRecord() { Assert.assertEquals(1, matchListener.getMatchCount()); } + /** + * Test to verify that Server IP is properly parsed from the event + * and included in the SessionLocator of the GuardRecord. + * This test validates the ServerIPAddrByServer field retrieval + * from DBC.QryLogClientAttrV view as per Teradata support recommendation. + */ + @Test + public void testServerIPParsing() throws Exception { + Event e = intitalizeEventObject(); + + // Parse the record using the Parser + com.ibm.guardium.universalconnector.commons.structures.Record record = Parser.parseRecord(e); + + // Verify SessionLocator contains the correct Server IP + Assert.assertNotNull("SessionLocator should not be null", record.getSessionLocator()); + Assert.assertEquals("Server IP should match the value from ServerIPAddrByServer field", + "10.0.0.1", + record.getSessionLocator().getServerIp()); + + // Verify Client IP is also correctly set + Assert.assertEquals("Client IP should match the value from ClientIPAddrByClient field", + "194.2.127.16", + record.getSessionLocator().getClientIp()); + } + } class TestMatchListener implements FilterMatchListener { diff --git a/filter-plugin/logstash-filter-trino-guardium/CHANGELOG.md b/filter-plugin/logstash-filter-trino-guardium/CHANGELOG.md index 143dd1602..efb141172 100644 --- a/filter-plugin/logstash-filter-trino-guardium/CHANGELOG.md +++ b/filter-plugin/logstash-filter-trino-guardium/CHANGELOG.md @@ -4,7 +4,9 @@ Notable changes will be documented in this file. -## [] +## [1.0.0] + +- Fixed service name mismatch with database name ### Added - Initial release, in parallel to Guardium . diff --git a/filter-plugin/logstash-filter-trino-guardium/README.md b/filter-plugin/logstash-filter-trino-guardium/README.md index 61951d716..0052a9d3e 100644 --- a/filter-plugin/logstash-filter-trino-guardium/README.md +++ b/filter-plugin/logstash-filter-trino-guardium/README.md @@ -6,7 +6,9 @@ * Environment: Trino DB * Supported inputs: http (pull) * Supported Guardium versions: - * Guardium Data Protection 11.4 and above + * Guardium Data Protection 12.0 patch 5005 and above + * Guardium Data Protection 12.1 patch 5005 and above + * Guardium Data Protection 12.2 and above This is a [Logstash](https://github.com/elastic/logstash) filter plug-in for the universal connector that is featured in IBM Security Guardium. It parses events and messages from the Trino audit log into a Guardium Record. @@ -77,25 +79,22 @@ enforcements. * Configure the policies you require. See [policies](/docs/#policies) for more information. * You must have permission for the S-Tap Management role. The admin user includes this role by default * Download - the [logstash-filter-trino_guardium_filter](../../filter-plugin/logstash-filter-trino-guardium/logstash-filter-trino_guardium_plugin_filter.zip) - plug-in. -* Verify that the http input plugin is available on the GDP system. If the plugin is missing, download and install - the [logstash-input-http](../../input-plugin/logstash-input-http/logstash-input-http_guardium_filter.zip) + the [logstash-filter-trino_guardium_filter](https://github.com/IBM/universal-connectors/releases/download/v1.7.0/logstash-filter-trino_guardium_filter.zip) plug-in. +* Verify that the http input plugin is available on the GDP system. ### Procedure 1. On the collector, go to ```Setup``` > ```Tools and Views``` > ```Configure Universal Connector```. 2. Enable the universal connector if it is disabled. 3. Click ```Upload File``` and select the - offline [logstash-filter-trino_guardium_filter](../../filter-plugin/logstash-filter-trino-guardium/logstash-filter-trino_guardium_plugin_filter.zip) + offline [logstash-filter-trino_guardium_filter](https://github.com/IBM/universal-connectors/releases/download/v1.7.0/logstash-filter-trino_guardium_filter.zip) plug-in. After it is uploaded, click ```OK```. -4. Click ```Upload File``` and select the key.json file. After it is uploaded, click ```OK```. 5. Click the Plus sign to open the Connector Configuration dialog box. 6. Type a name in the Connector name field. -7. Update the input section to add the details from the [TrinoSyslog.conf](../../filter-plugin/logstash-filter-trino-guardium/TrinoOverSyslogPackage/TrinoSyslog.conf) +7. Update the input section to add the details from the [TrinoOverHttp.conf](./TrinoOverHttpPackage/TrinoOverHttp.conf) file's input part, omitting the keyword "input{" at the beginning and its corresponding "}" at the end. -8. Update the filter section to add the details from the [TrinoSyslog.conf](../../filter-plugin/logstash-filter-trino-guardium/TrinoOverSyslogPackage/TrinoSyslog.conf) +8. Update the filter section to add the details from the [TrinoOverHttp.conf](./TrinoOverHttpPackage/TrinoOverHttp.conf) file's filter part, omitting the keyword "filter{" at the beginning and its corresponding "}" at the end. 9. The 'type' fields should match in the input and filter configuration sections. This field should be unique for every individual connector added. @@ -103,3 +102,5 @@ enforcements. 11. After the offline plug-in is installed and the configuration is uploaded and saved in the Guardium machine, restart the Universal Connector using the ```Disable/Enable``` button. +#### Limitations + • Client Hostname and Source Program will be seen as blank in report. diff --git a/filter-plugin/logstash-filter-trino-guardium/TrinoOverSyslogPackage/TrinoSyslog.conf b/filter-plugin/logstash-filter-trino-guardium/TrinoOverHttpPackage/TrinoOverHttp.conf similarity index 100% rename from filter-plugin/logstash-filter-trino-guardium/TrinoOverSyslogPackage/TrinoSyslog.conf rename to filter-plugin/logstash-filter-trino-guardium/TrinoOverHttpPackage/TrinoOverHttp.conf diff --git a/filter-plugin/logstash-filter-trino-guardium/build.gradle b/filter-plugin/logstash-filter-trino-guardium/build.gradle index 8e301cfab..c8b5ff31c 100644 --- a/filter-plugin/logstash-filter-trino-guardium/build.gradle +++ b/filter-plugin/logstash-filter-trino-guardium/build.gradle @@ -3,6 +3,31 @@ import static java.nio.file.StandardCopyOption.REPLACE_EXISTING apply plugin: 'java' apply plugin: 'jacoco' + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + + dependencies { + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' + classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.1.0" + classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' + } + +} + +def universalConnectorsDir = project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load(new File("${universalConnectorsDir}/versions.yml").newInputStream()) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" apply plugin: "com.github.johnrengelman.shadow" apply plugin: "eclipse" @@ -24,10 +49,10 @@ pluginInfo.pluginClass = "TrinoGuardiumFilter" pluginInfo.pluginName = "trino_guardium_filter" // must match the @LogstashPlugin annotation in the main plugin class // =========================================================================== -sourceCompatibility = 1.8 -targetCompatibility = 1.8 +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 -def jacocoVersion = '0.8.9' +def jacocoVersion = '0.8.11' // minimumCoverage can be set by Travis ENV def minimumCoverageStr = System.getenv("MINIMUM_COVERAGE") ?: "50.0%" if (minimumCoverageStr.endsWith("%")) { @@ -35,29 +60,16 @@ if (minimumCoverageStr.endsWith("%")) { } def minimumCoverage = Float.valueOf(minimumCoverageStr) / 100 -buildscript { - repositories { - maven { - url "https://plugins.gradle.org/m2/" - } - mavenCentral() - jcenter() - } - - dependencies { - classpath 'com.github.jengelman.gradle.plugins:shadow:4.0.4' - classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.0.0-RC2" - classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' - } - -} - -def universalConnectorsDir = project.projectDir.parentFile?.parentFile.toString(); -def versions = new org.yaml.snakeyaml.Yaml().load(new File("${universalConnectorsDir}/versions.yml").newInputStream()) - repositories { - mavenCentral() + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } } tasks.register("vendor") { @@ -68,14 +80,13 @@ tasks.register("vendor") { File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") projectJarFile.mkdirs() Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) - validatePluginJar(projectJarFile, project.group) } } apply plugin: 'com.github.johnrengelman.shadow' shadowJar { - classifier = null + archiveClassifier = null } @@ -83,6 +94,7 @@ dependencies { implementation 'com.google.code.gson:gson:' + versions.dependencies.gson implementation 'org.apache.commons:commons-lang3:' + versions.dependencies.commonsLang implementation 'commons-validator:commons-validator:' + versions.dependencies.commonsValidator + implementation group: 'commons-beanutils', name: 'commons-beanutils', version: versions.dependencies.commonsBeanutils implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.17.2' implementation group: 'com.github.jsqlparser', name: 'jsqlparser', version: '5.2' @@ -154,17 +166,16 @@ apply plugin: "org.barfuin.gradle.jacocolog" // ------------------------------------ jacoco { toolVersion = "${jacocoVersion}" - reportsDir = file("$buildDir/reports/jacoco") } jacocoTestReport { // You will see "Report -> file://...." at the end of a JaCoCo build // If no output, run this first: ./gradlew test reports { - html.enabled true - xml.enabled true - csv.enabled true - html.destination file("${buildDir}/reports/jacoco") - csv.destination file("${buildDir}/reports/jacoco/all.csv") + html.required = true + xml.required = true + csv.required = true + html.outputLocation = file("${buildDir}/reports/jacoco") + csv.outputLocation = file("${buildDir}/reports/jacoco/all.csv") } executionData.from fileTree(dir: "${buildDir}/jacoco/", includes: [ '**/*.exec' diff --git a/filter-plugin/logstash-filter-trino-guardium/gradle/wrapper/gradle-wrapper.properties b/filter-plugin/logstash-filter-trino-guardium/gradle/wrapper/gradle-wrapper.properties index aa991fcea..81aa1c044 100644 --- a/filter-plugin/logstash-filter-trino-guardium/gradle/wrapper/gradle-wrapper.properties +++ b/filter-plugin/logstash-filter-trino-guardium/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/filter-plugin/logstash-filter-trino-guardium/logstash-filter-trino_guardium_plugin_filter.zip b/filter-plugin/logstash-filter-trino-guardium/logstash-filter-trino_guardium_filter.zip similarity index 52% rename from filter-plugin/logstash-filter-trino-guardium/logstash-filter-trino_guardium_plugin_filter.zip rename to filter-plugin/logstash-filter-trino-guardium/logstash-filter-trino_guardium_filter.zip index a002b128e..92db06cc4 100644 Binary files a/filter-plugin/logstash-filter-trino-guardium/logstash-filter-trino_guardium_plugin_filter.zip and b/filter-plugin/logstash-filter-trino-guardium/logstash-filter-trino_guardium_filter.zip differ diff --git a/filter-plugin/logstash-filter-trino-guardium/src/main/java/com/ibm/guardium/trino/Constants.java b/filter-plugin/logstash-filter-trino-guardium/src/main/java/com/ibm/guardium/trino/Constants.java index f63b58540..d42d89e04 100644 --- a/filter-plugin/logstash-filter-trino-guardium/src/main/java/com/ibm/guardium/trino/Constants.java +++ b/filter-plugin/logstash-filter-trino-guardium/src/main/java/com/ibm/guardium/trino/Constants.java @@ -12,10 +12,8 @@ public class Constants { static final String Context = "context"; static final String URL = "url"; static final String URI = "uri"; - static final String ClientIP = "remoteClientAddress"; static final String ServerIP = "serverAddress"; - static final String DBUser = "user"; static final String SQLCommand = "query"; static final String QueryId = "QueryId"; @@ -25,4 +23,5 @@ public class Constants { static final int DEFAULT_PORT = -1; static final String Original = "original"; static final String FailureInfo = "failureInfo"; + static final String DEFAULT_IP = "0.0.0.0"; } diff --git a/filter-plugin/logstash-filter-trino-guardium/src/main/java/com/ibm/guardium/trino/Parser.java b/filter-plugin/logstash-filter-trino-guardium/src/main/java/com/ibm/guardium/trino/Parser.java index 0b5ea1b42..8120fb2d9 100644 --- a/filter-plugin/logstash-filter-trino-guardium/src/main/java/com/ibm/guardium/trino/Parser.java +++ b/filter-plugin/logstash-filter-trino-guardium/src/main/java/com/ibm/guardium/trino/Parser.java @@ -7,7 +7,15 @@ import com.google.gson.JsonObject; -import com.ibm.guardium.universalconnector.commons.structures.*; +import com.ibm.guardium.universalconnector.commons.structures.Accessor; +import com.ibm.guardium.universalconnector.commons.structures.Construct; +import com.ibm.guardium.universalconnector.commons.structures.Data; +import com.ibm.guardium.universalconnector.commons.structures.ExceptionRecord; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import com.ibm.guardium.universalconnector.commons.structures.Sentence; +import com.ibm.guardium.universalconnector.commons.structures.SentenceObject; +import com.ibm.guardium.universalconnector.commons.structures.SessionLocator; +import com.ibm.guardium.universalconnector.commons.structures.Time; import org.apache.commons.validator.routines.InetAddressValidator; import java.time.ZonedDateTime; @@ -18,8 +26,11 @@ import static com.ibm.guardium.trino.Constants.*; import static com.ibm.guardium.trino.Constants.DB_PROTOCOL; import static com.ibm.guardium.trino.Constants.SERVER_TYPE; +import static com.ibm.guardium.trino.Constants.DEFAULT_IP; + import static com.ibm.guardium.universalconnector.commons.custom_parsing.PropertyConstant.*; import static com.ibm.guardium.universalconnector.commons.structures.Accessor.LANGUAGE_FREE_TEXT_STRING; + import com.ibm.guardium.trino.sql_parsing.SqlParser; /** @@ -33,22 +44,23 @@ public class Parser { private static final SqlParser sqlParser = new SqlParser(); private static final InetAddressValidator inetAddressValidator = - InetAddressValidator.getInstance(); + InetAddressValidator.getInstance(); - public Parser() {} + public Parser() { + } public static Record parseRecord(JsonObject data) { Record record = new Record(); record.setSessionId(EMPTY); - record.setDbName( - data.has(Metadata) ? getDbName(data.getAsJsonObject(Metadata)) : NOT_AVAILABLE); + String DbName = data.has(Context) ? getDbName(data.getAsJsonObject(Context)) : NOT_AVAILABLE; + record.setDbName(DbName); record.setAppUserName(NOT_AVAILABLE); String sqlString = data.has(Metadata) ? getSqlString(data.getAsJsonObject(Metadata)) : EMPTY; record.setException( - data.has(FailureInfo) - ? getException(data.getAsJsonObject(FailureInfo), sqlString) - : getException(data, sqlString)); - record.setAccessor(getAccessor(data)); + data.has(FailureInfo) + ? getException(data.getAsJsonObject(FailureInfo), sqlString) + : getException(data, sqlString)); + record.setAccessor(getAccessor(data, DbName)); record.setSessionLocator(getSessionLocator(data)); record.setTime(getTimestamp(data)); if (!record.isException()) record.setData(getData(sqlString)); @@ -56,18 +68,18 @@ public static Record parseRecord(JsonObject data) { return record; } - protected static Accessor getAccessor(JsonObject data) { + protected static Accessor getAccessor(JsonObject data, String DbName) { Accessor accessor = new Accessor(); - accessor.setServiceName(TRINO); + accessor.setServiceName(DbName); accessor.setDbUser( - data.has(Context) ? getDbUser(data.getAsJsonObject(Context)) : NOT_AVAILABLE); + data.has(Context) ? getDbUser(data.getAsJsonObject(Context)) : NOT_AVAILABLE); accessor.setDbProtocolVersion(EMPTY); accessor.setDbProtocol(DB_PROTOCOL); accessor.setServerType(SERVER_TYPE); accessor.setServerOs(EMPTY); accessor.setServerDescription(EMPTY); - accessor.setServerHostName(EMPTY); + accessor.setServerHostName(NOT_AVAILABLE); accessor.setClientHostName(EMPTY); accessor.setClient_mac(EMPTY); accessor.setClientOs(EMPTY); @@ -85,11 +97,11 @@ private static String getSqlString(JsonObject metadata) { } private static String getDbName(JsonObject metadata) { - return (metadata.has("tables") - && metadata.getAsJsonArray("tables").size() > 0 - && metadata.getAsJsonArray("tables").get(0).getAsJsonObject().has("schema")) - ? metadata.getAsJsonArray("tables").get(0).getAsJsonObject().get("schema").getAsString() - : NOT_AVAILABLE; + return metadata.has("schema") + ? !metadata.get("schema").isJsonNull() ? metadata.get("schema").getAsString() + : NOT_AVAILABLE + : NOT_AVAILABLE; + } private static String getDbUser(JsonObject context) { @@ -144,8 +156,7 @@ static SessionLocator getSessionLocator(JsonObject data) { // Set port numbers sessionLocator.setClientPort(DEFAULT_PORT); - sessionLocator.setServerPort( - data.has(Metadata) ? getServerPort(data.getAsJsonObject(Metadata)) : DEFAULT_PORT); + sessionLocator.setServerPort(DEFAULT_PORT); return sessionLocator; } diff --git a/filter-plugin/logstash-filter-trino-guardium/src/test/java/com/ibm/guardium/trino/ParserTest.java b/filter-plugin/logstash-filter-trino-guardium/src/test/java/com/ibm/guardium/trino/ParserTest.java index 39cfe57bc..7e88b0604 100644 --- a/filter-plugin/logstash-filter-trino-guardium/src/test/java/com/ibm/guardium/trino/ParserTest.java +++ b/filter-plugin/logstash-filter-trino-guardium/src/test/java/com/ibm/guardium/trino/ParserTest.java @@ -15,30 +15,32 @@ class ParserTest { @Test void testRecord() { String payload = - "{\"createTime\":\"2025-06-09T18:26:41.470Z\",\"context\":{\"user\":\"trino\",\"originalUser\":\"trino\",\"principal\":\"trino\",\"enabledRoles\":[],\"groups\":[],\"remoteClientAddress\":\"127.0.0.1\",\"userAgent\":\"trino-cli\",\"clientTags\":[],\"clientCapabilities\":[\"PATH\",\"PARAMETRIC_DATETIME\",\"SESSION_AUTHORIZATION\"],\"source\":\"trino-cli\",\"timezone\":\"UTC\",\"resourceGroupId\":[\"global\"],\"sessionProperties\":{},\"resourceEstimates\":{},\"serverAddress\":\"172.19.0.6\",\"serverVersion\":\"466\",\"environment\":\"production\",\"queryType\":\"SELECT\",\"retryPolicy\":\"NONE\"},\"metadata\":{\"queryId\":\"20250609_182641_00011_mtb2e\",\"transactionId\":\"03463dad-f1e8-4915-885c-d3a1c2bdedd6\",\"query\":\"SELECT * FROM hive.test_db.test_table LIMIT 10\",\"queryState\":\"QUEUED\",\"tables\":[{\"catalog\":\"hive\",\"schema\":\"test_db\",\"table\":\"test_table\",\"authorization\":\"trino\",\"filters\":[],\"columns\":[{\"column\":\"city\"},{\"column\":\"name\"},{\"column\":\"id\"},{\"column\":\"age\"}],\"directlyReferenced\":true,\"referenceChain\":[]}],\"routines\":[],\"uri\":\"http://172.19.0.6:8080/v1/query/20250609_182641_00011_mtb2e\"}}"; + "{\"createTime\":\"2025-06-09T18:26:41.470Z\",\"context\":{\"user\":\"trino\",\"originalUser\":\"trino\",\"principal\":\"trino\",\"schema\":\"my_database1\",\"enabledRoles\":[],\"groups\":[],\"remoteClientAddress\":\"127.0.0.1\",\"userAgent\":\"trino-cli\",\"clientTags\":[],\"clientCapabilities\":[\"PATH\",\"PARAMETRIC_DATETIME\",\"SESSION_AUTHORIZATION\"],\"source\":\"trino-cli\",\"timezone\":\"UTC\",\"resourceGroupId\":[\"global\"],\"sessionProperties\":{},\"resourceEstimates\":{},\"serverAddress\":\"172.19.0.6\",\"serverVersion\":\"466\",\"environment\":\"production\",\"queryType\":\"SELECT\",\"retryPolicy\":\"NONE\"},\"metadata\":{\"queryId\":\"20250609_182641_00011_mtb2e\",\"transactionId\":\"03463dad-f1e8-4915-885c-d3a1c2bdedd6\",\"query\":\"SELECT * FROM hive.test_db.test_table LIMIT 10\",\"queryState\":\"QUEUED\",\"tables\":[{\"catalog\":\"hive\",\"schema\":\"my_database1\",\"table\":\"test_table\",\"authorization\":\"trino\",\"filters\":[],\"columns\":[{\"column\":\"city\"},{\"column\":\"name\"},{\"column\":\"id\"},{\"column\":\"age\"}],\"directlyReferenced\":true,\"referenceChain\":[]}],\"routines\":[],\"uri\":\"http://172.19.0.6:8080/v1/query/20250609_182641_00011_mtb2e\"}}"; final JsonObject data = new Gson().fromJson(payload, JsonObject.class); Record record = Parser.parseRecord(data); assertNotNull(record); - assertEquals("test_db", record.getDbName()); + assertEquals("my_database1", record.getDbName()); assertEquals(-1, record.getSessionLocator().getClientPort()); assertEquals("127.0.0.1", record.getSessionLocator().getClientIp()); assertEquals("Trino", record.getAccessor().getDbProtocol()); assertEquals("172.19.0.6", record.getSessionLocator().getServerIp()); - assertEquals(8080, record.getSessionLocator().getServerPort()); + assertEquals(-1, record.getSessionLocator().getServerPort()); assertEquals("trino", record.getAccessor().getDbUser()); assertEquals("Trino", record.getAccessor().getServerType()); assertEquals( - "SELECT * FROM hive.test_db.test_table LIMIT 10", record.getData().getOriginalSqlCommand()); + "SELECT * FROM hive.test_db.test_table LIMIT 10", record.getData().getOriginalSqlCommand()); assertEquals(1749493601470L, record.getTime().getTimstamp()); + assertEquals("my_database1", record.getAccessor().getServiceName()); + assertEquals(record.getDbName(), record.getAccessor().getServiceName()); } @Test void testSessionLocatorEmpty_case1() { String payload = - "{\"context\":{\"user\":\"trino\",\"originalUser\":\"trino\",\"principal\":\"trino\",\"enabledRoles\":[],\"groups\":[],\"userAgent\":\"trino-cli\",\"clientTags\":[],\"clientCapabilities\":[\"PATH\",\"PARAMETRIC_DATETIME\",\"SESSION_AUTHORIZATION\"],\"source\":\"trino-cli\",\"timezone\":\"UTC\",\"resourceGroupId\":[\"global\"],\"sessionProperties\":{},\"resourceEstimates\":{},\"serverVersion\":\"466\",\"environment\":\"production\",\"queryType\":\"SELECT\",\"retryPolicy\":\"NONE\"},\"url\":{\"path\":\"/\", \"domain\":\"host.docker.internal\"},\"metadata\":{\"queryId\":\"20250605_195009_00010_34b3y\",\"transactionId\":\"b69e0f30-de9a-4dd3-b05f-f75566f1d6d1\",\"query\":\"SELECT * FROM hive.test_db.test_table LIMIT 10\",\"queryState\":\"FINISHED\",\"tables\":[],\"routines\":[]}}"; + "{\"context\":{\"user\":\"trino\",\"originalUser\":\"trino\",\"principal\":\"trino\",\"enabledRoles\":[],\"groups\":[],\"userAgent\":\"trino-cli\",\"clientTags\":[],\"clientCapabilities\":[\"PATH\",\"PARAMETRIC_DATETIME\",\"SESSION_AUTHORIZATION\"],\"source\":\"trino-cli\",\"timezone\":\"UTC\",\"resourceGroupId\":[\"global\"],\"sessionProperties\":{},\"resourceEstimates\":{},\"serverVersion\":\"466\",\"environment\":\"production\",\"queryType\":\"SELECT\",\"retryPolicy\":\"NONE\"},\"url\":{\"path\":\"/\", \"domain\":\"host.docker.internal\"},\"metadata\":{\"queryId\":\"20250605_195009_00010_34b3y\",\"transactionId\":\"b69e0f30-de9a-4dd3-b05f-f75566f1d6d1\",\"query\":\"SELECT * FROM hive.test_db.test_table LIMIT 10\",\"queryState\":\"FINISHED\",\"tables\":[],\"routines\":[]}}"; final JsonObject data = new Gson().fromJson(payload, JsonObject.class); SessionLocator record = Parser.getSessionLocator(data); @@ -63,24 +65,24 @@ void testSessionLocatorEmpty_case2() { void testDBUser() { String payload = "{\"DatabaseName\":\"db_name\",\"createTime\":\"2025-06-05T19:50:09.402Z\"}"; final JsonObject data = new Gson().fromJson(payload, JsonObject.class); - Accessor record = Parser.getAccessor(data); + Accessor record = Parser.getAccessor(data, "N.A."); assertEquals("N.A.", record.getDbUser()); } @Test void testGetException() { String payload = - "{\"createTime\":\"2025-06-09T18:26:41.470Z\",\"context\":{\"user\":\"trino\",\"originalUser\":\"trino\",\"principal\":\"trino\",\"enabledRoles\":[],\"groups\":[],\"remoteClientAddress\":\"127.0.0.1\",\"userAgent\":\"trino-cli\",\"clientTags\":[],\"clientCapabilities\":[\"PATH\",\"PARAMETRIC_DATETIME\",\"SESSION_AUTHORIZATION\"],\"source\":\"trino-cli\",\"timezone\":\"UTC\",\"resourceGroupId\":[\"global\"],\"sessionProperties\":{},\"resourceEstimates\":{},\"serverAddress\":\"172.19.0.6\",\"serverVersion\":\"466\",\"environment\":\"production\",\"queryType\":\"SELECT\",\"retryPolicy\":\"NONE\"},\"metadata\":{\"queryId\":\"20250609_182641_00011_mtb2e\",\"transactionId\":\"03463dad-f1e8-4915-885c-d3a1c2bdedd6\",\"query\":\"INSERT INTO hive.test_db.test_table VALUES (17, 'John Doe', 30, 'Engineer')\",\"queryState\":\"QUEUED\",\"tables\":[{\"catalog\":\"hive\",\"schema\":\"test_db\",\"table\":\"test_table\",\"authorization\":\"trino\",\"filters\":[],\"columns\":[{\"column\":\"city\"},{\"column\":\"name\"},{\"column\":\"id\"},{\"column\":\"age\"}],\"directlyReferenced\":true,\"referenceChain\":[]}],\"routines\":[],\"uri\":\"http://172.19.0.6:8080/v1/query/20250609_182641_00011_mtb2e\"},\"failureInfo\":{\"errorCode\":{\"code\":16777232,\"name\":\"HIVE_FILESYSTEM_ERROR\",\"type\":\"EXTERNAL\",\"fatal\":false},\"failureType\":\"io.trino.spi.TrinoException\",\"failureMessage\":\"Error moving data files from hdfs://hadoop-hive:9000/tmp/presto-trino/eb708b7a-5685-4a07-9d72-26d1760567b5/20250610_022922_00018_mtb2e_9e3c81b8-10b3-45fd-8103-6a45610ee8ee.gz to final location hdfs://hadoop-hive:9000/user/hive/warehouse/test_db.db/test_table/20250610_022922_00018_mtb2e_9e3c81b8-10b3-45fd-8103-6a45610ee8ee.gz\"}}"; + "{\"createTime\":\"2025-06-09T18:26:41.470Z\",\"context\":{\"user\":\"trino\",\"originalUser\":\"trino\",\"principal\":\"trino\",\"enabledRoles\":[],\"groups\":[],\"remoteClientAddress\":\"127.0.0.1\",\"userAgent\":\"trino-cli\",\"clientTags\":[],\"clientCapabilities\":[\"PATH\",\"PARAMETRIC_DATETIME\",\"SESSION_AUTHORIZATION\"],\"source\":\"trino-cli\",\"timezone\":\"UTC\",\"resourceGroupId\":[\"global\"],\"sessionProperties\":{},\"resourceEstimates\":{},\"serverAddress\":\"172.19.0.6\",\"serverVersion\":\"466\",\"environment\":\"production\",\"queryType\":\"SELECT\",\"retryPolicy\":\"NONE\"},\"metadata\":{\"queryId\":\"20250609_182641_00011_mtb2e\",\"transactionId\":\"03463dad-f1e8-4915-885c-d3a1c2bdedd6\",\"query\":\"INSERT INTO hive.test_db.test_table VALUES (17, 'John Doe', 30, 'Engineer')\",\"queryState\":\"QUEUED\",\"tables\":[{\"catalog\":\"hive\",\"schema\":\"test_db\",\"table\":\"test_table\",\"authorization\":\"trino\",\"filters\":[],\"columns\":[{\"column\":\"city\"},{\"column\":\"name\"},{\"column\":\"id\"},{\"column\":\"age\"}],\"directlyReferenced\":true,\"referenceChain\":[]}],\"routines\":[],\"uri\":\"http://172.19.0.6:8080/v1/query/20250609_182641_00011_mtb2e\"},\"failureInfo\":{\"errorCode\":{\"code\":16777232,\"name\":\"HIVE_FILESYSTEM_ERROR\",\"type\":\"EXTERNAL\",\"fatal\":false},\"failureType\":\"io.trino.spi.TrinoException\",\"failureMessage\":\"Error moving data files from hdfs://hadoop-hive:9000/tmp/presto-trino/eb708b7a-5685-4a07-9d72-26d1760567b5/20250610_022922_00018_mtb2e_9e3c81b8-10b3-45fd-8103-6a45610ee8ee.gz to final location hdfs://hadoop-hive:9000/user/hive/warehouse/test_db.db/test_table/20250610_022922_00018_mtb2e_9e3c81b8-10b3-45fd-8103-6a45610ee8ee.gz\"}}"; final JsonObject data = new Gson().fromJson(payload, JsonObject.class); Record record = Parser.parseRecord(data); assertNotNull(record.getException()); assertEquals( - "INSERT INTO hive.test_db.test_table VALUES (17, 'John Doe', 30, 'Engineer')", - record.getException().getSqlString()); + "INSERT INTO hive.test_db.test_table VALUES (17, 'John Doe', 30, 'Engineer')", + record.getException().getSqlString()); assertEquals( - "Error moving data files from hdfs://hadoop-hive:9000/tmp/presto-trino/eb708b7a-5685-4a07-9d72-26d1760567b5/20250610_022922_00018_mtb2e_9e3c81b8-10b3-45fd-8103-6a45610ee8ee.gz to final location hdfs://hadoop-hive:9000/user/hive/warehouse/test_db.db/test_table/20250610_022922_00018_mtb2e_9e3c81b8-10b3-45fd-8103-6a45610ee8ee.gz", - record.getException().getDescription()); + "Error moving data files from hdfs://hadoop-hive:9000/tmp/presto-trino/eb708b7a-5685-4a07-9d72-26d1760567b5/20250610_022922_00018_mtb2e_9e3c81b8-10b3-45fd-8103-6a45610ee8ee.gz to final location hdfs://hadoop-hive:9000/user/hive/warehouse/test_db.db/test_table/20250610_022922_00018_mtb2e_9e3c81b8-10b3-45fd-8103-6a45610ee8ee.gz", + record.getException().getDescription()); assertEquals("SQL_ERROR", record.getException().getExceptionTypeId()); } } diff --git a/filter-plugin/logstash-filter-yugabyte-guardium/README.md b/filter-plugin/logstash-filter-yugabyte-guardium/README.md index 2c523eca0..01a46b03d 100644 --- a/filter-plugin/logstash-filter-yugabyte-guardium/README.md +++ b/filter-plugin/logstash-filter-yugabyte-guardium/README.md @@ -127,9 +127,10 @@ tags : ["Yugabyte"] The Guardium universal connector is the Guardium entry point for native audit logs. The universal connector identifies and parses received events, and then converts them to a standard Guardium format. The output of the universal connector is forwarded to the Guardium sniffer on the collector, for policy and auditing enforcements. Configure Guardium to read the native audit logs by customizing the Yugabyte template. ### Limitations -* When the universal collector starts to collect data, it may show two S-TAP statuses in the pattern "postgres_" and "cassandra_" based on the types of logs it collects.. +* When the universal collector starts to collect data, it may show two S-TAP statuses in the pattern "postgres_" and "cassandra_" based on the types of logs it collects. * The ClientHostName is not available in the YugabyteDB audit logs. - +* When Yugabyte logs SQL errors and SQL statements as separate audit records, Guardium may display the SQL statement separately in Full SQL report, instead of embedding it in the SQL Error record. +* ### Before you begin - Configure the policies you require. See [policies](/docs/#policies) for more information. diff --git a/filter-plugin/logstash-filter-yugabyte-guardium/YugabyteOverSyslogPackage/YugabyteDB/filter.conf b/filter-plugin/logstash-filter-yugabyte-guardium/YugabyteOverSyslogPackage/YugabyteDB/filter.conf index bc767fd22..e102b89d9 100644 --- a/filter-plugin/logstash-filter-yugabyte-guardium/YugabyteOverSyslogPackage/YugabyteDB/filter.conf +++ b/filter-plugin/logstash-filter-yugabyte-guardium/YugabyteOverSyslogPackage/YugabyteDB/filter.conf @@ -90,18 +90,23 @@ filter { } } } else if [message] =~ /STATEMENT:/ { - # Error logs containing the lastly fired query, Example, Table not found error - # The regex starts looking from the matching timestamp in the log event. It will extract the - # required data by saperating the log logline initially by spaces or tabs up to occurance of the - # word "STATEMENT" and then later, the text will be considered as fired query. + # Error logs containing the lastly fired query, Example, Table not found error. + # Use a tolerant multiline match so valid Yugabyte ERROR + STATEMENT logs + # still populate query even when formatting varies slightly. grok { match => { - "message" => "%{IPORHOST:server_hostname},%{IPV4:server_ip},(%{NUMBER:timestamp}[\s\t]+)(%{IP:client_ip}[\s\t]*)\(%{NUMBER:client_port}\)[\s\t]+\[(%{NUMBER:process_id})\][\s\t]+(%{GREEDYDATA:application_name})[\s\t]+(%{USERNAME:username})[\s\t]+(%{WORD:db_name})[\s\t]+(?%{BASE16NUM}.%{BASE16NUM})[\s\t]+(%{NUMBER:trasaction_id})[\s\t]+(ERROR:)[\s\t]+(%{GREEDYDATA:error_description}[\s\t\\n]+)(%{NUMBER}[\s\t]+)(%{IP})\(%{NUMBER}\)[\s\t]+\[(%{NUMBER})\][\s\t]+(%{GREEDYDATA})[\s\t]+(%{USERNAME})[\s\t]+(%{WORD})[\s\t]+(%{BASE16NUM}.%{BASE16NUM})[\s\t]+(%{NUMBER})[\s\t]+(STATEMENT:)[\t\s]+(%{GREEDYDATA:query})" + "message" => [ + "%{IPORHOST:server_hostname},%{IPV4:server_ip},(%{NUMBER:timestamp}[\s\t]+)(%{IP:client_ip}[\s\t]*)\(%{NUMBER:client_port}\)[\s\t]+\[(%{NUMBER:process_id})\][\s\t]+(%{GREEDYDATA:application_name})[\s\t]+(%{USERNAME:username})[\s\t]+(%{WORD:db_name})[\s\t]+(?%{BASE16NUM}.%{BASE16NUM})[\s\t]+(%{NUMBER:trasaction_id})[\s\t]+(ERROR:)[\s\t]+(?.*?)[\s\t]+(%{NUMBER}[\s\t]+)(%{IP})\(%{NUMBER}\)[\s\t]+\[(%{NUMBER})\][\s\t]+(%{GREEDYDATA})[\s\t]+(%{USERNAME})[\s\t]+(%{WORD})[\s\t]+(%{BASE16NUM}.%{BASE16NUM})[\s\t]+(%{NUMBER})[\s\t]+(STATEMENT:)[\t\s]*(?[\s\S]*)", + "%{IPORHOST:server_hostname},%{IPV4:server_ip},(%{NUMBER:timestamp}[\s\t]+)(%{IP:client_ip}[\s\t]*)\(%{NUMBER:client_port}\)[\s\t]+\[(%{NUMBER:process_id})\][\s\t]+(%{GREEDYDATA:application_name})[\s\t]+(%{USERNAME:username})[\s\t]+(%{WORD:db_name})[\s\t]+(?%{BASE16NUM}.%{BASE16NUM})[\s\t]+(%{NUMBER:trasaction_id})[\s\t]+(STATEMENT:)[\t\s]*(?[\s\S]*)" + ] } add_field => { event_type => "SQL_ERROR" } } + mutate { + strip => [ "query", "error_description" ] + } } else { # Error logs containing only error messages, Example, Syntax errors # the regex looks for the keyword "Error" diff --git a/filter-plugin/logstash-filter-yugabyte-guardium/build.gradle b/filter-plugin/logstash-filter-yugabyte-guardium/build.gradle index 1b9bafda1..fca95a0cf 100644 --- a/filter-plugin/logstash-filter-yugabyte-guardium/build.gradle +++ b/filter-plugin/logstash-filter-yugabyte-guardium/build.gradle @@ -2,6 +2,30 @@ import java.nio.file.Files import static java.nio.file.StandardCopyOption.REPLACE_EXISTING apply plugin: 'java' + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + + dependencies { + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' + classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.1.0" + classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' + } +} + +def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" // =========================================================================== @@ -20,10 +44,10 @@ pluginInfo.pluginClass = "YugabytedbGuardiumFilter" pluginInfo.pluginName = "yugabytedb_guardium_filter" // must match the @LogstashPlugin annotation in the main plugin class // =========================================================================== -sourceCompatibility = 1.8 -targetCompatibility = 1.8 +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 -def jacocoVersion = '0.8.4' +def jacocoVersion = '0.8.11' // minimumCoverage can be set by Travis ENV def minimumCoverageStr = System.getenv("MINIMUM_COVERAGE") ?: "50.0%" if (minimumCoverageStr.endsWith("%")) { @@ -31,27 +55,16 @@ if (minimumCoverageStr.endsWith("%")) { } def minimumCoverage = Float.valueOf(minimumCoverageStr) / 100 -buildscript { - repositories { - maven { - url "https://plugins.gradle.org/m2/" - } - mavenCentral() - jcenter() - } - - dependencies { - classpath 'com.github.jengelman.gradle.plugins:shadow:4.0.4' - classpath "org.barfuin.gradle.jacocolog:gradle-jacoco-log:3.0.0-RC2" - classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' - } -} - -def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); -def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) repositories { - mavenCentral() + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } } tasks.register("vendor"){ @@ -62,14 +75,13 @@ tasks.register("vendor"){ File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") projectJarFile.mkdirs() Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) - validatePluginJar(projectJarFile, project.group) } } apply plugin: 'com.github.johnrengelman.shadow' shadowJar { - classifier = null + archiveClassifier = null } dependencies { @@ -77,6 +89,7 @@ dependencies { implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: versions.dependencies.log4jCore implementation 'org.apache.commons:commons-lang3:' + versions.dependencies.commonsLang implementation 'com.google.code.gson:gson:' + versions.dependencies.gson + implementation group: 'commons-beanutils', name: 'commons-beanutils', version: versions.dependencies.commonsBeanutils implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "build/libs/logstash-core-*.*.*.jar") implementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "common-*.*.*.jar") @@ -100,6 +113,17 @@ tasks.withType(JavaCompile) { options.encoding = 'UTF-8' } +test { + jvmArgs = [ + '--add-opens=java.base/java.lang=ALL-UNNAMED', + '--add-opens=java.base/java.util=ALL-UNNAMED', + '--add-opens=java.base/java.text=ALL-UNNAMED', + '--add-opens=java.base/java.lang.reflect=ALL-UNNAMED', + '--add-opens=java.base/sun.util.resources=ALL-UNNAMED', + '--add-opens=java.base/sun.util.cldr=ALL-UNNAMED' + ] +} + tasks.register("generateRubySupportFiles") { @@ -142,17 +166,16 @@ apply plugin: "org.barfuin.gradle.jacocolog" // ------------------------------------ jacoco { toolVersion = "${jacocoVersion}" - reportsDir = file("$buildDir/reports/jacoco") } jacocoTestReport { // You will see "Report -> file://...." at the end of a JaCoCo build // If no output, run this first: ./gradlew test reports { - html.enabled true - xml.enabled true - csv.enabled true - html.destination file("${buildDir}/reports/jacoco") - csv.destination file("${buildDir}/reports/jacoco/all.csv") + html.required = true + xml.required = true + csv.required = true + html.outputLocation = file("${buildDir}/reports/jacoco") + csv.outputLocation = file("${buildDir}/reports/jacoco/all.csv") } executionData.from fileTree(dir: "${buildDir}/jacoco/", includes: [ '**/*.exec' diff --git a/filter-plugin/logstash-filter-yugabyte-guardium/gradle/wrapper/gradle-wrapper.properties b/filter-plugin/logstash-filter-yugabyte-guardium/gradle/wrapper/gradle-wrapper.properties index 60c76b340..ba9ccfe4c 100644 --- a/filter-plugin/logstash-filter-yugabyte-guardium/gradle/wrapper/gradle-wrapper.properties +++ b/filter-plugin/logstash-filter-yugabyte-guardium/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists \ No newline at end of file diff --git a/filter-plugin/logstash-filter-yugabyte-guardium/src/main/java/com/ibm/guardium/yugabytedb/Constants.java b/filter-plugin/logstash-filter-yugabyte-guardium/src/main/java/com/ibm/guardium/yugabytedb/Constants.java index 1de46ff5a..542128872 100644 --- a/filter-plugin/logstash-filter-yugabyte-guardium/src/main/java/com/ibm/guardium/yugabytedb/Constants.java +++ b/filter-plugin/logstash-filter-yugabyte-guardium/src/main/java/com/ibm/guardium/yugabytedb/Constants.java @@ -9,7 +9,7 @@ public class Constants { - public static final String NOT_AVAILABLE= "NA"; + public static final String NOT_AVAILABLE= "N.A."; public static final String UNKNOWN_STRING = StringUtils.EMPTY; public static final String LOGSTASH_TAG_JSON_PARSE_ERROR = "_yugabyteguardium_json_parse_error"; @@ -50,9 +50,9 @@ public class Constants { public static final String LANGUAGE_CASSANDRA = "CASS"; - public static final String DB_PROTOCOL_PG = "POSTGRESQL"; + public static final String DB_PROTOCOL_PG = "YUGABYTE"; - public static final String DB_PROTOCOL_CASSANDRA = "CASSANDRA"; + public static final String DB_PROTOCOL_CASSANDRA = "YUGABYTE_CASS"; public static final String MESSAGE = "message"; public static final String TYPE = "log_type"; diff --git a/filter-plugin/logstash-filter-yugabyte-guardium/src/main/java/com/ibm/guardium/yugabytedb/Parser.java b/filter-plugin/logstash-filter-yugabyte-guardium/src/main/java/com/ibm/guardium/yugabytedb/Parser.java index 69b0de9c8..7ac51d837 100644 --- a/filter-plugin/logstash-filter-yugabyte-guardium/src/main/java/com/ibm/guardium/yugabytedb/Parser.java +++ b/filter-plugin/logstash-filter-yugabyte-guardium/src/main/java/com/ibm/guardium/yugabytedb/Parser.java @@ -5,7 +5,15 @@ package com.ibm.guardium.yugabytedb; -import com.ibm.guardium.universalconnector.commons.structures.*; +import com.ibm.guardium.universalconnector.commons.structures.Accessor; +import com.ibm.guardium.universalconnector.commons.structures.Construct; +import com.ibm.guardium.universalconnector.commons.structures.Data; +import com.ibm.guardium.universalconnector.commons.structures.ExceptionRecord; +import com.ibm.guardium.universalconnector.commons.structures.Record; +import com.ibm.guardium.universalconnector.commons.structures.Sentence; +import com.ibm.guardium.universalconnector.commons.structures.SentenceObject; +import com.ibm.guardium.universalconnector.commons.structures.SessionLocator; +import com.ibm.guardium.universalconnector.commons.structures.Time; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/gi_plugins_templates.zip b/gi_plugins_templates.zip new file mode 100644 index 000000000..2e1f9ab22 Binary files /dev/null and b/gi_plugins_templates.zip differ diff --git a/input-plugin/logstash-input-adabas/.gitignore b/input-plugin/logstash-input-adabas/.gitignore new file mode 100644 index 000000000..371b42cb5 --- /dev/null +++ b/input-plugin/logstash-input-adabas/.gitignore @@ -0,0 +1,28 @@ +# Ignore Gradle build output directory +build +bin +*.idea +*.iml +lib/ +vendor/ +.bundle/ +build/ +out/ +.idea +.gradle +.vscode +.classpath +.project +*.code-workspace +*.DS_Store +*.iml +*.class +*.ipr +*.iws +*.gemspec +Gemfile* +.gitattributes +.settings/ +.settings +gradle.properties +*.gem \ No newline at end of file diff --git a/input-plugin/logstash-input-adabas/IMPLEMENTATION_GUIDE.md b/input-plugin/logstash-input-adabas/IMPLEMENTATION_GUIDE.md new file mode 100644 index 000000000..97b8790f6 --- /dev/null +++ b/input-plugin/logstash-input-adabas/IMPLEMENTATION_GUIDE.md @@ -0,0 +1,1262 @@ +# Adabas Universal Connector Implementation Guide for Guardium Data Protection + +## Table of Contents +1. [Overview](#overview) +2. [Architecture](#architecture) +3. [Prerequisites](#prerequisites) +4. [Implementation Steps](#implementation-steps) + - [Step 1: Configure Adabas Auditing Server](#step-1-configure-adabas-auditing-server) + - [Step 2: Obtain or Build the Plugins](#step-2-obtain-or-build-the-plugins) + - [Step 3: Install Plugins on Guardium](#step-3-install-plugins-on-guardium) + - [Step 4: Configure Universal Connector](#step-4-configure-universal-connector) + - [Step 5: Enable Universal Connector](#step-5-enable-universal-connector) + - [Step 6: Verify Data Flow](#step-6-verify-data-flow) +5. [Configuration Reference](#configuration-reference) +6. [Troubleshooting](#troubleshooting) +7. [Appendix](#appendix) + +--- + +## Overview + +This guide provides step-by-step instructions for implementing the Adabas Universal Connector with IBM Guardium Data Protection. The Adabas connector enables Guardium to monitor and audit Adabas database activity without requiring traditional S-TAP agents. + +### What is the Universal Connector? + +The Guardium Universal Connector is a framework that allows Guardium to receive and process audit data from various data sources through their native audit logs. It uses a Logstash-based pipeline with three components: + +1. **Input Plugin** - Receives audit data from the data source +2. **Filter Plugin** - Parses and transforms the data into Guardium format +3. **Output Plugin** - Sends processed data to Guardium (internal component) + +### Adabas-Specific Architecture + +The Adabas implementation uses a **custom input plugin** that connects directly to the Adabas Auditing Server via EntireX Broker, rather than using standard log forwarding methods like Syslog or Filebeat. + +**Key Components:** +- **Adabas Auditing Server** - Generates audit events from Adabas database activity +- **EntireX Broker** - Messaging middleware that facilitates communication +- **Adabas Input Plugin** (`logstash-input-adabas_auditing_input`) - Connects to the broker and retrieves audit messages +- **Adabas Filter Plugin** (`logstash-filter-adabas_guardium_filter`) - Parses audit data into Guardium format +- **Guardium Universal Connector** - Hosts the plugins and forwards data to Guardium + +--- + +## Architecture + +``` +┌─────────────────────────┐ +│ Adabas Database │ +│ │ +└───────────┬─────────────┘ + │ + │ Audit Events + ▼ +┌─────────────────────────┐ +│ Adabas Auditing Server │ +│ │ +└───────────┬─────────────┘ + │ + │ EntireX Protocol + ▼ +┌─────────────────────────┐ +│ EntireX Broker │ +│ (Messaging Layer) │ +└───────────┬─────────────┘ + │ + │ Broker Messages + ▼ +┌─────────────────────────────────────────────┐ +│ Guardium Universal Connector │ +│ │ +│ ┌─────────────────────────────────────┐ │ +│ │ Input Plugin │ │ +│ │ (adabas_auditing_input) │ │ +│ │ - Connects to EntireX Broker │ │ +│ │ - Retrieves audit messages │ │ +│ │ - Parses binary format │ │ +│ └──────────────┬──────────────────────┘ │ +│ │ │ +│ ▼ │ +│ ┌─────────────────────────────────────┐ │ +│ │ Filter Plugin │ │ +│ │ (adabas_guardium_filter) │ │ +│ │ - Transforms to Guardium format │ │ +│ │ - Extracts metadata │ │ +│ │ - Handles errors │ │ +│ └──────────────┬──────────────────────┘ │ +│ │ │ +│ ▼ │ +│ ┌─────────────────────────────────────┐ │ +│ │ Output Plugin (Internal) │ │ +│ │ - Sends to Guardium Sniffer │ │ +│ └─────────────────────────────────────┘ │ +└─────────────────┬───────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────┐ +│ Guardium Data Protection │ +│ - Policy Enforcement │ +│ - Reporting & Analytics │ +│ - Alerting │ +└─────────────────────────────────────────────┘ +``` + +--- + +## Prerequisites + +### System Requirements + +#### Guardium System +- **Guardium Data Protection Version:** 12.0 or later +- **Deployment Type:** Standalone system or Collector +- **User Permissions:** S-TAP Management Application role +- **Network Access:** Connectivity to Adabas Auditing Server's EntireX Broker + +#### Adabas Environment +- **Adabas Version:** Compatible version with Auditing Server support +- **Adabas Auditing Server:** Installed and configured +- **EntireX Broker:** Running and accessible +- **Network Configuration:** Firewall rules allowing Guardium to connect to broker port (default: 3000) + +### Software Requirements + +#### For Using Pre-Built Plugins (Recommended) +- Pre-built plugin files (`.gem` files) provided by IBM or Software AG: + - `logstash-input-adabas_auditing_input--java.gem` + - `logstash-filter-adabas_guardium_filter--java.gem` + +#### For Building Plugins from Source (Optional) +- **Java Development Kit (JDK):** Version 11 or later +- **Gradle:** Version 6.x or later (included via wrapper) +- **Logstash:** Version 7.5.2 or compatible +- **Git:** For cloning repositories +- **Build Dependencies:** + - Guardium Universal Connector Commons library + - Adabas SDK libraries (provided by Software AG) + - EntireX libraries (provided by Software AG) + +### Required Information + +Before starting, gather the following information: + +| Information | Description | Example | +|------------|-------------|---------| +| **Broker Host** | Hostname or IP of EntireX Broker | `adabas-broker.company.com` | +| **Broker Port** | Port number for broker connection | `3000` | +| **Broker Class** | Broker class identifier | `ADABAS-AUDIT` | +| **Broker Server** | Broker server name | `AUDIT-SERVER` | +| **Broker Service** | Service name for audit data | `AUDIT-SERVICE` | +| **User** | Authentication user for broker | `guardium_user` | +| **Token** | Authentication token/password | `` | +| **Guardium Collector IP** | IP address of Guardium system | `10.0.1.100` | + +### Network Requirements + +Ensure the following network connectivity: + +``` +Guardium Collector → EntireX Broker +- Protocol: TCP +- Port: 3000 (default, may vary) +- Direction: Outbound from Guardium +``` + +--- + +## Implementation Steps + +### Step 1: Configure Adabas Auditing Server + +The Adabas Auditing Server must be properly configured to generate and publish audit events. + +#### 1.1 Enable Adabas Auditing + +**Note:** This step is typically performed by your Adabas administrator. Consult Software AG documentation for detailed Adabas configuration. + +1. **Enable Auditing on Adabas Database:** + - Configure the Adabas database to generate audit records + - Specify which operations to audit (reads, writes, DDL, etc.) + - Set audit detail level + +2. **Configure Adabas Auditing Server:** + - Install the Adabas Auditing Server component + - Configure connection to the Adabas database + - Set up audit event collection parameters + +#### 1.2 Configure EntireX Broker + +The EntireX Broker acts as the messaging middleware between Adabas and Guardium. + +1. **Verify Broker is Running:** + ```bash + # Check broker status (command may vary by platform) + etbinfo -b + ``` + +2. **Configure Broker Service:** + - Create or identify the service that will publish audit events + - Note the broker class, server, and service names + - Configure authentication if required + +3. **Test Broker Connectivity:** + ```bash + # Test connection from Guardium server + telnet + ``` + +#### 1.3 Verify Audit Data Flow + +Before proceeding, verify that audit data is being generated: + +1. Perform some database operations on Adabas +2. Check that audit events are being published to the broker +3. Use EntireX tools to verify message flow + +--- + +### Step 2: Obtain or Build the Plugins + +You have two options: use pre-built plugins or build from source. + +#### Option A: Using Pre-Built Plugins (Recommended) + +If IBM or Software AG has provided pre-built plugin files: + +1. **Obtain the Plugin Files:** + - `logstash-input-adabas_auditing_input--java.gem` + - `logstash-filter-adabas_guardium_filter--java.gem` + +2. **Transfer to Guardium System:** + ```bash + # Copy files to Guardium (from your local machine) + scp logstash-input-adabas_auditing_input-*.gem guardium@:/tmp/ + scp logstash-filter-adabas_guardium_filter-*.gem guardium@:/tmp/ + ``` + +3. **Skip to Step 3** (Installation) + +#### Option B: Building Plugins from Source + +If you need to build the plugins yourself: + +##### 2.1 Prepare Build Environment + +1. **Install Java 11:** + ```bash + # Verify Java installation + java -version + # Should show Java 11 or later + ``` + +2. **Download Logstash:** + ```bash + # Download Logstash 7.5.2 or compatible version + wget https://artifacts.elastic.co/downloads/logstash/logstash-7.5.2.tar.gz + tar -xzf logstash-7.5.2.tar.gz + export LOGSTASH_HOME=/path/to/logstash-7.5.2 + ``` + +3. **Obtain Required Files:** + - Download `rubyUtils.gradle` and `versions.yml` from Logstash GitHub + - Copy to your Logstash installation directory + +4. **Fix rubyUtils.gradle Issues:** + + Edit `rubyUtils.gradle` and make these changes: + + **Issue 1 - Fix JRuby Version:** + ```gradle + // Find this line (around line 20): + classpath "org.jruby:jruby-core:${gradle.ext.versions.jruby.version}" + + // Replace with actual version from versions.yml: + classpath "org.jruby:jruby-core:9.4.13.0" + ``` + + **Issue 2 - Add YAML Parsing:** + ```gradle + // Add this code after the Ruby variables section: + + // Ruby variables + def versionsPath = project.hasProperty("LOGSTASH_CORE_PATH") ? LOGSTASH_CORE_PATH + "/../versions.yml" : "${projectDir}/versions.yml" + + // Add YAML parsing code below: + def versionsFile = new File(versionsPath) + if (!versionsFile.exists()) { + throw new GradleException("versions.yml file not found at: ${versionsPath}") + } + + def versionsData = [:] + def currentSection = null + versionsFile.eachLine { line -> + def trimmed = line.trim() + if (trimmed && !trimmed.startsWith('#')) { + if (!trimmed.startsWith(' ') && trimmed.endsWith(':')) { + currentSection = trimmed.replaceAll(':', '') + versionsData[currentSection] = [:] + } else if (trimmed.startsWith('version:') || trimmed.startsWith('sha256:')) { + def parts = trimmed.split(':', 2) + if (parts.length == 2 && currentSection) { + versionsData[currentSection][parts[0].trim()] = parts[1].trim() + } + } + } + } + + gradle.ext.versions = versionsData + versionMap = gradle.ext.versions + ``` + +##### 2.2 Build Guardium Commons Library + +1. **Clone the Commons Repository:** + ```bash + git clone https://github.com/IBM/guardium-universalconnector-commons.git + cd guardium-universalconnector-commons + ``` + +2. **Build the Commons JARs:** + ```bash + # Follow the README instructions to build + ./gradlew build + # Note the location of generated JAR files + ``` + +##### 2.3 Build Input Plugin + +1. **Clone or Navigate to Input Plugin:** + ```bash + cd /path/to/universal-connectors/input-plugin/logstash-input-adabas + ``` + +2. **Create gradle.properties:** + ```bash + cat > gradle.properties << EOF + LOGSTASH_CORE_PATH=/path/to/logstash-7.5.2/logstash-core + EOF + ``` + +3. **Build the Plugin:** + ```bash + ./gradlew assemble gem + ``` + +4. **Verify Build:** + ```bash + # Check for generated .gem file + ls -l logstash-input-adabas_auditing_input-*.gem + ``` + +##### 2.4 Build Filter Plugin + +1. **Navigate to Filter Plugin:** + ```bash + cd /path/to/universal-connectors/filter-plugin/logstash-filter-adabas-guardium + ``` + +2. **Create gradle.properties:** + ```bash + cat > gradle.properties << EOF + LOGSTASH_CORE_PATH=/path/to/logstash-7.5.2/logstash-core + GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH=/path/to/guardium-universalconnector-commons/build/libs + EOF + ``` + +3. **Build the Plugin:** + ```bash + ./gradlew assemble gem + ``` + +4. **Verify Build:** + ```bash + # Check for generated .gem file + ls -l logstash-filter-adabas_guardium_filter-*.gem + ``` + +##### 2.5 Transfer Built Plugins to Guardium + +```bash +# Copy both .gem files to Guardium +scp logstash-input-adabas_auditing_input-*.gem guardium@:/tmp/ +scp logstash-filter-adabas_guardium_filter-*.gem guardium@:/tmp/ +``` + +--- + +### Step 3: Install Plugins on Guardium + +Now install the plugins on your Guardium system. + +#### 3.1 Access Guardium CLI + +1. **SSH to Guardium:** + ```bash + ssh guardium@ + ``` + +2. **Switch to CLI Mode:** + ```bash + # If prompted, enter CLI mode + cli + ``` + +#### 3.2 Install Input Plugin + +1. **Install the Input Plugin:** + ```bash + grdapi install_universal_connector_plugin \ + plugin_file=/tmp/logstash-input-adabas_auditing_input--java.gem + ``` + +2. **Verify Installation:** + ```bash + # List installed plugins + grdapi list_universal_connector_plugins + ``` + + Expected output should include: + ``` + adabas_auditing_input + ``` + +#### 3.3 Install Filter Plugin + +1. **Install the Filter Plugin:** + ```bash + grdapi install_universal_connector_plugin \ + plugin_file=/tmp/logstash-filter-adabas_guardium_filter--java.gem + ``` + +2. **Verify Installation:** + ```bash + # List installed plugins + grdapi list_universal_connector_plugins + ``` + + Expected output should include: + ``` + adabas_auditing_input + adabas_guardium_filter + ``` + +#### 3.4 Clean Up + +```bash +# Remove temporary .gem files +rm /tmp/logstash-input-adabas_auditing_input-*.gem +rm /tmp/logstash-filter-adabas_guardium_filter-*.gem +``` + +--- + +### Step 4: Configure Universal Connector + +Configure the Universal Connector to use the Adabas plugins. + +#### 4.1 Access Universal Connector Configuration + +1. **Log in to Guardium Web UI:** + - Open browser: `https://:8443` + - Enter credentials + +2. **Navigate to Universal Connector:** + - Go to: **Setup** → **Tools and Views** → **Configure Universal Connector** + +#### 4.2 Create Connector Configuration + +1. **Click "Add" to Create New Connector** + +2. **Enter Connector Name:** + - Name: `Adabas_Production` (or your preferred name) + - Description: `Adabas database auditing via EntireX Broker` + +3. **Configure Input Section:** + + Click on the **Input** tab and enter: + + ```ruby + input { + adabas_auditing_input { + # EntireX Broker connection details + host => "adabas-broker.company.com" + port => 3000 + + # Broker service identification + brokerClass => "ADABAS-AUDIT" + brokerServer => "AUDIT-SERVER" + brokerService => "AUDIT-SERVICE" + + # Authentication + user => "guardium_user" + token => "your_secure_token" + + # Optional: Connection parameters + retryInterval => 5 + retryCount => 10 + waitTime => 30 + receiveLength => 32767 + compression => 0 + + # Optional: Metadata REST server URL + # restURL => "http://metadata-server:8080" + } + } + ``` + + **Parameter Descriptions:** + + | Parameter | Required | Description | Default | + |-----------|----------|-------------|---------| + | `host` | Yes | EntireX Broker hostname or IP | `localhost` | + | `port` | Yes | EntireX Broker port | `3000` | + | `brokerClass` | Yes | Broker class identifier | `class` | + | `brokerServer` | Yes | Broker server name | `server` | + | `brokerService` | Yes | Service name for audit data | `service` | + | `user` | Yes | Authentication username | `user` | + | `token` | Yes | Authentication token/password | `token` | + | `retryInterval` | No | Retry interval in seconds | `5` | + | `retryCount` | No | Number of retry attempts | `10` | + | `waitTime` | No | Wait time in seconds | `30` | + | `receiveLength` | No | Maximum message receive length | `32767` | + | `compression` | No | Compression level (0=none) | `0` | + | `restURL` | No | Metadata REST server URL | `""` | + +4. **Configure Filter Section:** + + Click on the **Filter** tab and enter: + + ```ruby + filter { + adabas_guardium_filter { + # Source field containing audit data + source => "adabas-auditing" + } + } + ``` + + **Note:** The `source` parameter should match the field name used by the input plugin (default: `adabas-auditing`). + +5. **Review Configuration:** + - Verify all parameters are correct + - Check for syntax errors (Guardium will validate) + +6. **Save Configuration:** + - Click **Save** + - Guardium will validate the configuration + - If validation fails, review error messages and correct + +#### 4.3 Using Secrets for Sensitive Data (Recommended) + +For production environments, use Guardium's keystore for sensitive information: + +1. **Create Secrets via CLI:** + ```bash + # SSH to Guardium + ssh guardium@ + + # Add broker credentials to keystore + grdapi universal_connector_keystore_add \ + key=ADABAS_BROKER_USER \ + password=guardium_user + + grdapi universal_connector_keystore_add \ + key=ADABAS_BROKER_TOKEN \ + password=your_secure_token + ``` + +2. **Verify Secrets:** + ```bash + grdapi universal_connector_keystore_list + ``` + +3. **Update Input Configuration to Use Secrets:** + ```ruby + input { + adabas_auditing_input { + host => "adabas-broker.company.com" + port => 3000 + brokerClass => "ADABAS-AUDIT" + brokerServer => "AUDIT-SERVER" + brokerService => "AUDIT-SERVICE" + + # Use environment variables from keystore + user => "${ADABAS_BROKER_USER}" + token => "${ADABAS_BROKER_TOKEN}" + + retryInterval => 5 + retryCount => 10 + waitTime => 30 + } + } + ``` + +4. **Save Updated Configuration** + +--- + +### Step 5: Enable Universal Connector + +#### 5.1 Enable via Web UI + +1. **In Configure Universal Connector Page:** + - Verify your connector configuration is listed + - Click **Enable** button + +2. **Wait for Startup:** + - Universal Connector will start (takes 1-2 minutes) + - Status will change from "Disabled" to "Enabled" + +#### 5.2 Enable via CLI (Alternative) + +```bash +# SSH to Guardium +ssh guardium@ + +# Enable Universal Connector +grdapi run_universal_connector +``` + +#### 5.3 Enable with Debug Logging (For Troubleshooting) + +```bash +# Enable with debug level logging +grdapi run_universal_connector debug_level=2 +``` + +**Debug Levels:** +- `0` - Errors only (default) +- `1` - Warnings and errors +- `2` - Info, warnings, and errors +- `3` - Debug (verbose) + +#### 5.4 Verify Universal Connector Status + +1. **Check Status in Web UI:** + - The **Enable** button should change to **Disable** + - Status indicator should be green + +2. **Check Status via CLI:** + ```bash + grdapi get_universal_connector_status + ``` + + Expected output: + ``` + Status: Running + Connectors: 1 + ``` + +--- + +### Step 6: Verify Data Flow + +#### 6.1 Generate Test Activity + +1. **Perform Operations on Adabas:** + - Connect to your Adabas database + - Execute some queries (SELECT, INSERT, UPDATE) + - Perform administrative operations if applicable + +2. **Wait for Processing:** + - Allow 1-2 minutes for data to flow through the pipeline + +#### 6.2 Check S-TAP Status Page + +1. **Navigate to S-TAP Status:** + - Go to: **Monitor** → **S-TAP Status** + +2. **Look for Adabas Connector:** + - S-TAP Host format: `::UC` + - S-TAP Version: `Universal connector V` + - Status should be **Green** (active) + +3. **Verify Data Flow:** + - Check "Last Contact" timestamp (should be recent) + - Check "Messages" count (should be increasing) + +#### 6.3 Check Guardium Logs + +1. **Access Universal Connector Logs:** + ```bash + # SSH to Guardium + ssh guardium@ + + # View recent log entries + grdapi tail_universal_connector_log lines=100 + ``` + +2. **Look for Success Indicators:** + - Connection to broker established + - Messages being received + - No error messages + +3. **Check for Errors:** + ```bash + # Search for errors in logs + grdapi tail_universal_connector_log lines=500 | grep -i error + ``` + +#### 6.4 Verify Data in Guardium Reports + +1. **Run Activity Report:** + - Go to: **Reports** → **Activity** → **Activity Report** + - Set time range to last hour + - Filter by database type: Adabas + +2. **Check for Audit Records:** + - Verify records are appearing + - Check that user names are populated + - Verify SQL/commands are captured + - Confirm timestamps are correct + +3. **Review Session Details:** + - Go to: **Monitor** → **Sessions** + - Look for Adabas sessions + - Verify session information is complete + +#### 6.5 Test Policy Enforcement + +1. **Create Test Policy:** + - Go to: **Policy** → **Policy Builder** + - Create a simple policy (e.g., alert on SELECT statements) + - Apply to Adabas data source + +2. **Trigger Policy:** + - Execute operations that match the policy + - Wait for policy evaluation + +3. **Check Alerts:** + - Go to: **Monitor** → **Alerts** + - Verify alerts are generated for Adabas activity + +--- + +## Configuration Reference + +### Complete Configuration Example + +Here's a complete, production-ready configuration: + +```ruby +# ============================================ +# INPUT SECTION +# ============================================ +input { + adabas_auditing_input { + # Broker Connection + host => "adabas-broker.company.com" + port => 3000 + + # Service Identification + brokerClass => "ADABAS-AUDIT" + brokerServer => "AUDIT-SERVER" + brokerService => "AUDIT-SERVICE" + + # Authentication (using keystore) + user => "${ADABAS_BROKER_USER}" + token => "${ADABAS_BROKER_TOKEN}" + + # Connection Tuning + retryInterval => 5 # Retry every 5 seconds on failure + retryCount => 10 # Retry up to 10 times + waitTime => 30 # Wait 30 seconds for messages + receiveLength => 32767 # Maximum message size + compression => 0 # No compression + + # Optional: Metadata Server + # restURL => "http://metadata-server:8080" + } +} + +# ============================================ +# FILTER SECTION +# ============================================ +filter { + adabas_guardium_filter { + source => "adabas-auditing" + } + + # Optional: Add custom fields + mutate { + add_field => { + "environment" => "production" + "data_center" => "DC1" + } + } +} +``` + +### Environment Variables Reference + +When using the keystore, these environment variables are available: + +| Variable Name | Purpose | Example Value | +|--------------|---------|---------------| +| `ADABAS_BROKER_USER` | Broker authentication username | `guardium_user` | +| `ADABAS_BROKER_TOKEN` | Broker authentication token | `` | +| `ADABAS_BROKER_HOST` | Broker hostname (optional) | `adabas-broker.company.com` | +| `ADABAS_METADATA_URL` | Metadata REST server URL (optional) | `http://metadata:8080` | + +### Guardium Record Structure + +The filter plugin transforms Adabas audit data into this Guardium structure: + +```json +{ + "sessionId": "mV20eHvvRha2ELTeqJxQJg==", + "dbName": "ADABAS-DB-001", + "appUserName": "APP_USER", + "time": { + "timestamp": 1705751051070, + "minOffsetFromGMT": -240, + "minDst": 0 + }, + "accessor": { + "dbUser": "NATUID_VALUE", + "serverType": "Adabas", + "serverHostName": "LPAR_NAME", + "sourceProgram": "NATPROG_NAME", + "language": "FREE_TEXT", + "dataType": "CONSTRUCT", + "dbProtocol": "Adabas native audit" + }, + "data": { + "construct": { + "sentences": [ + { + "verb": "READ", + "fields": ["FIELD1", "FIELD2", "FIELD3"] + } + ] + } + }, + "exception": { + "exceptionTypeId": "SQL_ERROR", + "description": "Response Code 3 (0) received.", + "sqlString": "READ with ISN 12345" + } +} +``` + +--- + +## Troubleshooting + +### Common Issues and Solutions + +#### Issue 1: Universal Connector Won't Start + +**Symptoms:** +- Status remains "Disabled" after clicking Enable +- Error in logs: "Failed to start Universal Connector" + +**Solutions:** + +1. **Check Plugin Installation:** + ```bash + grdapi list_universal_connector_plugins + ``` + Verify both `adabas_auditing_input` and `adabas_guardium_filter` are listed. + +2. **Check Configuration Syntax:** + - Review input and filter sections for typos + - Ensure all required parameters are present + - Verify quotes and brackets are balanced + +3. **Check Logs for Details:** + ```bash + grdapi tail_universal_connector_log lines=200 + ``` + +4. **Restart with Overwrite:** + ```bash + grdapi run_universal_connector overwrite_old_instance="true" + ``` + +#### Issue 2: Cannot Connect to EntireX Broker + +**Symptoms:** +- Log message: "Failed to connect to broker" +- Log message: "Connection refused" +- No data flowing to Guardium + +**Solutions:** + +1. **Verify Network Connectivity:** + ```bash + # From Guardium CLI + telnet + ``` + +2. **Check Firewall Rules:** + - Ensure Guardium can reach broker port (default: 3000) + - Check both outbound (Guardium) and inbound (broker) rules + +3. **Verify Broker is Running:** + ```bash + # On broker server + etbinfo -b + ``` + +4. **Check Broker Configuration:** + - Verify broker class, server, and service names + - Ensure service is registered and active + +5. **Test with Broker Tools:** + - Use EntireX tools to verify broker accessibility + - Test authentication credentials + +#### Issue 3: Authentication Failures + +**Symptoms:** +- Log message: "Authentication failed" +- Log message: "Invalid credentials" +- Connection established but no data received + +**Solutions:** + +1. **Verify Credentials:** + - Check username and token are correct + - Ensure no extra spaces or special characters + +2. **Check Keystore Values:** + ```bash + grdapi universal_connector_keystore_list + ``` + Verify keys exist and are spelled correctly. + +3. **Test Credentials Manually:** + - Use EntireX tools to test authentication + - Verify user has permissions to access audit service + +4. **Update Credentials:** + ```bash + # Remove old key + grdapi universal_connector_keystore_remove key=ADABAS_BROKER_TOKEN + + # Add new key + grdapi universal_connector_keystore_add \ + key=ADABAS_BROKER_TOKEN \ + password=new_token + + # Restart UC + grdapi run_universal_connector overwrite_old_instance="true" + ``` + +#### Issue 4: No Audit Data Appearing + +**Symptoms:** +- Universal Connector is running +- Connection to broker successful +- No records in Guardium reports + +**Solutions:** + +1. **Verify Audit Data is Being Generated:** + - Check Adabas Auditing Server is running + - Perform database operations + - Verify audit events are published to broker + +2. **Check Filter Processing:** + ```bash + # Enable debug logging + grdapi run_universal_connector debug_level=3 + + # Check logs for filter activity + grdapi tail_universal_connector_log lines=500 | grep -i filter + ``` + +3. **Look for Skipped Events:** + ```bash + # Check for skip tags + grdapi tail_universal_connector_log lines=500 | grep -i skip + ``` + +4. **Verify Source Field:** + - Ensure filter `source` parameter matches input plugin output + - Default is `adabas-auditing` + +5. **Check for Parsing Errors:** + ```bash + # Look for parsing errors + grdapi tail_universal_connector_log lines=500 | grep -i "parse\|error" + ``` + +#### Issue 5: Incomplete or Missing Data Fields + +**Symptoms:** +- Records appear in Guardium but missing information +- User names showing as empty +- Database names not populated + +**Solutions:** + +1. **Check Adabas Audit Configuration:** + - Verify audit level captures required fields + - Ensure user context is included in audit events + +2. **Review Audit Event Structure:** + - Check that CLNT (client) data is present + - Verify ACBX (control block) data is included + +3. **Check Metadata Configuration:** + - If using metadata REST server, verify it's accessible + - Check `restURL` parameter is correct + +4. **Review Parser Logic:** + - Check filter plugin logs for warnings + - Verify all expected fields are being extracted + +#### Issue 6: Performance Issues + +**Symptoms:** +- High CPU usage on Guardium +- Slow report generation +- Delayed data processing + +**Solutions:** + +1. **Adjust Connection Parameters:** + ```ruby + input { + adabas_auditing_input { + # Increase wait time to reduce polling frequency + waitTime => 60 + + # Increase receive length for batch processing + receiveLength => 65535 + } + } + ``` + +2. **Enable Compression:** + ```ruby + input { + adabas_auditing_input { + compression => 1 # Enable compression + } + } + ``` + +3. **Filter Audit Events at Source:** + - Configure Adabas to audit only necessary operations + - Reduce audit detail level if appropriate + +4. **Check Guardium Resources:** + - Monitor CPU and memory usage + - Consider adding more collectors for load balancing + +#### Issue 7: Universal Connector Stops After Reboot + +**Symptoms:** +- Universal Connector not running after Guardium restart +- Must manually enable after reboot + +**Solution:** + +After each Guardium reboot, restart the Universal Connector: + +```bash +# SSH to Guardium +ssh guardium@ + +# Start Universal Connector +grdapi run_universal_connector +``` + +**Note:** This is expected behavior. Universal Connector must be manually started after system reboots. + +### Diagnostic Commands + +#### Check Universal Connector Status +```bash +grdapi get_universal_connector_status +``` + +#### View Recent Logs +```bash +# Last 100 lines +grdapi tail_universal_connector_log lines=100 + +# Last 500 lines with timestamps +grdapi tail_universal_connector_log lines=500 +``` + +#### List Installed Plugins +```bash +grdapi list_universal_connector_plugins +``` + +#### List Connector Configurations +```bash +# Via Web UI: Setup → Tools and Views → Configure Universal Connector +``` + +#### Check Keystore Contents +```bash +grdapi universal_connector_keystore_list +``` + +#### Restart Universal Connector +```bash +# Normal restart +grdapi run_universal_connector + +# Force restart (overwrite existing instance) +grdapi run_universal_connector overwrite_old_instance="true" + +# Restart with debug logging +grdapi run_universal_connector debug_level=3 +``` + +#### Stop Universal Connector +```bash +grdapi stop_universal_connector +``` + +### Log File Locations + +When creating a MustGather, these log files are included: + +- **Universal Connector Log:** `uc-logstash.log` +- **Logstash Standard Output:** `logstash_stdout_stderr.log` +- **Guardium System Logs:** Various system logs + +### Getting Help + +If you continue to experience issues: + +1. **Create a MustGather:** + - Go to: **Setup** → **Tools and Views** → **MustGather** + - Select all relevant options + - Include Universal Connector logs + +2. **Contact IBM Support:** + - Provide MustGather output + - Include configuration details (sanitize sensitive data) + - Describe the issue and steps to reproduce + +3. **Check Documentation:** + - [IBM Guardium Documentation](https://www.ibm.com/docs/en/guardium) + - [Universal Connectors GitHub](https://github.com/IBM/universal-connectors) + - Software AG Adabas documentation + +--- + +## Appendix + +### A. Glossary + +| Term | Definition | +|------|------------| +| **Adabas** | A high-performance database management system by Software AG | +| **Adabas Auditing Server** | Component that captures and publishes Adabas audit events | +| **EntireX Broker** | Messaging middleware for communication between Adabas and Guardium | +| **Universal Connector** | Guardium framework for ingesting data from various sources | +| **Input Plugin** | Component that receives data from external sources | +| **Filter Plugin** | Component that parses and transforms data | +| **S-TAP** | Software Tap - traditional Guardium monitoring agent | +| **Guardium Record** | Standardized data structure for audit events in Guardium | +| **ACBX** | Adabas Control Block Extended - contains command details | +| **CLNT** | Client information block in Adabas audit data | + +### B. Port Reference + +| Port | Protocol | Purpose | Direction | +|------|----------|---------|-----------| +| 3000 | TCP | EntireX Broker (default) | Guardium → Broker | +| 8443 | HTTPS | Guardium Web UI | Admin → Guardium | +| 22 | SSH | Guardium CLI access | Admin → Guardium | + +### C. File Locations + +| File/Directory | Purpose | +|----------------|---------| +| `/tmp/` | Temporary location for plugin .gem files | +| Guardium internal | Universal Connector logs (accessed via grdapi) | +| Guardium internal | Plugin installation directory (managed by Guardium) | + +### D. Related Documentation + +- [Guardium Data Protection Documentation](https://www.ibm.com/docs/en/guardium) +- [Universal Connectors GitHub Repository](https://github.com/IBM/universal-connectors) +- [Developing Plugins for Guardium](https://github.com/IBM/universal-connectors/blob/main/docs/Guardium%20Data%20Protection/developing_plugins_gdp.md) +- [Configuring Universal Connector](https://github.com/IBM/universal-connectors/blob/main/docs/Guardium%20Data%20Protection/uc_config_gdp.md) +- Software AG Adabas Documentation +- Software AG EntireX Documentation + +### E. Quick Reference Card + +#### Essential Commands + +```bash +# Enable Universal Connector +grdapi run_universal_connector + +# Check status +grdapi get_universal_connector_status + +# View logs +grdapi tail_universal_connector_log lines=100 + +# List plugins +grdapi list_universal_connector_plugins + +# Stop Universal Connector +grdapi stop_universal_connector + +# Restart with overwrite +grdapi run_universal_connector overwrite_old_instance="true" +``` + +#### Configuration Template + +```ruby +input { + adabas_auditing_input { + host => "BROKER_HOST" + port => BROKER_PORT + brokerClass => "BROKER_CLASS" + brokerServer => "BROKER_SERVER" + brokerService => "BROKER_SERVICE" + user => "${ADABAS_BROKER_USER}" + token => "${ADABAS_BROKER_TOKEN}" + } +} + +filter { + adabas_guardium_filter { + source => "adabas-auditing" + } +} +``` + +### F. Version History + +| Version | Date | Changes | +|---------|------|---------| +| 1.0 | 2025-01 | Initial implementation guide | + +--- + +## Support and Feedback + +For questions, issues, or feedback regarding this implementation guide: + +- **IBM Support:** Contact your IBM support representative +- **GitHub Issues:** [Universal Connectors Issues](https://github.com/IBM/universal-connectors/issues) +- **Documentation Updates:** Submit pull requests to improve this guide + +--- + +**Document Information:** +- **Title:** Adabas Universal Connector Implementation Guide +- **Audience:** Guardium administrators, Database administrators +- **Prerequisites:** Basic knowledge of Guardium, Adabas, and networking +- **Estimated Implementation Time:** 2-4 hours (excluding Adabas configuration) + +--- + +*This guide is provided as-is and may be updated as new versions of Guardium or the Adabas plugins are released. Always refer to the latest official IBM and Software AG documentation for the most current information.* \ No newline at end of file diff --git a/input-plugin/logstash-input-adabas/LICENSE b/input-plugin/logstash-input-adabas/LICENSE new file mode 100644 index 000000000..a80a3fd53 --- /dev/null +++ b/input-plugin/logstash-input-adabas/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2020 Elastic and contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/input-plugin/logstash-input-adabas/README.md b/input-plugin/logstash-input-adabas/README.md new file mode 100644 index 000000000..002eb003a --- /dev/null +++ b/input-plugin/logstash-input-adabas/README.md @@ -0,0 +1,125 @@ +# Logstash Adabas Auditing Input Plugin + +This is a Java plugin for [Logstash](https://github.com/elastic/logstash). + +## Build +The build of this plugin requires the access to an installation of Logstash. + +1. Download Logstash from https://www.elastic.co/downloads/logstash +2. Copy the files **rubyUtils.gradle** and **versions.yml** from Github repository https://github.com/elastic/logstash to directory where you installed Logstash + + **Note:** We've identified issues with the `rubyUtils.gradle` file from the Logstash GitHub repository that may cause build failures for this project. Please make the following modifications to the `rubyUtils.gradle` file: + + - Issue 1: JRuby Version Resolution + + **Problem:** Dynamic version reference fails during build + ```gradle + // Original (causes build failure) + classpath "org.jruby:jruby-core:${gradle.ext.versions.jruby.version}" + ``` + **Solution:** Use the actual version number of jruby from versions.yml, for example: + ```gradle + // Fixed version + classpath "org.jruby:jruby-core:9.4.13.0" + ``` + + - Issue 2: YAML Parsing + + **Problem:** Missing YAML parsing logic causes version resolution to fail + + **Solution:** Add the following YAML parsing code after the Ruby variables section: + + ```gradle + // Ruby variables + def versionsPath = project.hasProperty("LOGSTASH_CORE_PATH") ? LOGSTASH_CORE_PATH + "/../versions.yml" : "${projectDir}/versions.yml" + + // ⚠️Add this YAML parsing code below: + // Read and parse versions.yml without external dependencies + def versionsFile = new File(versionsPath) + if (!versionsFile.exists()) { + throw new GradleException("versions.yml file not found at: ${versionsPath}") + } + + // Simple YAML parsing for versions.yml structure + def versionsData = [:] + def currentSection = null + versionsFile.eachLine { line -> + def trimmed = line.trim() + if (trimmed && !trimmed.startsWith('#')) { + if (!trimmed.startsWith(' ') && trimmed.endsWith(':')) { + // Top level section + currentSection = trimmed.replaceAll(':', '') + versionsData[currentSection] = [:] + } else if (trimmed.startsWith('version:') || trimmed.startsWith('sha256:')) { + // Property in current section + def parts = trimmed.split(':', 2) + if (parts.length == 2 && currentSection) { + versionsData[currentSection][parts[0].trim()] = parts[1].trim() + } + } + } + } + + // Set gradle.ext.versions + gradle.ext.versions = versionsData + versionMap = gradle.ext.versions + ``` +3. Clone this repository +4. Set the property variable **LOGSTASH_CORE_PATH**. This could be done in gradle.properties file +5. Assemble plugin with the command `./gradlew assemble gem` + +After that successful build a file **logstash-input-adabas_auditing_input--java.gem** is created in the root directory of the project. + +See also [How to write a Java input plugin](https://www.elastic.co/guide/en/logstash/current/java-input-plugin.html). + +## Install Plugin +To install the plugin use the command +``` +logstash-plugin install --no-verify --local /logstash-input-adabas_auditing_input--java.gem +``` + +## Run Logstash +Execute the command `logstash -f ` where ``is your Logstash configuration file. An example is below. + +## Plugin Configuration Example +This configuration reads the data from the Adabas Auditing Server and write the data to `elasticsearch` and `stdout`. + +``` +input { + adabas_auditing_input { + brokerClass => "class" + brokerServer => "server" + brokerService => "service" + host => "host" + port => 3000 + token => "token" + user => "user" + } +} +output { + stdout { + codec => rubydebug + } +} +``` + +## Plugin Parameter +| Parameter | Description | Type | Default Value | +| ------------- | --------------------------- | ------ | ---------------- | +| host | Broker host | String | "localhost" | +| port | Broker port | Number | 3000 | +| brokerClass | Broker class name | String | "class" | +| brokerServer | Broker server name | String | "server" | +| brokerService | Broker service name | String | "service" | +| user | User | String | "user" | +| token | Token | String | "token" | +| retryInterval | Retry interval in seconds | Number | 5 | +| retryCount | Retry count | Number | 10 | +| waitTime | Wait time in seconds | Number | 30 | +| receiveLength | Receive length | Number | 32767 | +| compression | Compression | Number | 0 | +| restURL | URL of metadata REST server | String | "" | +| Hosts | Elasticsearch host | String | "localhost:9200" | + +## Environment Variable +Use the environment variable `REST_PATH` set the directory for the metadata outside of Logstash. diff --git a/input-plugin/logstash-input-adabas/VERSION b/input-plugin/logstash-input-adabas/VERSION new file mode 100644 index 000000000..afaf360d3 --- /dev/null +++ b/input-plugin/logstash-input-adabas/VERSION @@ -0,0 +1 @@ +1.0.0 \ No newline at end of file diff --git a/input-plugin/logstash-input-adabas/build.gradle b/input-plugin/logstash-input-adabas/build.gradle new file mode 100644 index 000000000..30414c095 --- /dev/null +++ b/input-plugin/logstash-input-adabas/build.gradle @@ -0,0 +1,137 @@ +import java.nio.file.Files +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + +plugins { + id 'com.github.johnrengelman.shadow' version '8.1.1' + id 'java' +} + +ext { + snakeYamlVersion = '2.2' + shadowGradlePluginVersion = '8.1.1' +} + + +def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + + +apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" + +// =========================================================================== +// plugin info +// =========================================================================== +group 'com.softwareag.adabas.auditing.logstash' // must match the package of the main plugin class +version "${file("VERSION").text.trim()}" // read from required VERSION file +description = "Adabas Auditing Input Plugin for Logstash" +pluginInfo.licenses = ['Apache-2.0'] // list of SPDX license IDs +pluginInfo.longDescription = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using \$LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" +pluginInfo.authors = ['Software GmbH'] +pluginInfo.email = ['support@softwareag.com'] +pluginInfo.homepage = "https://www.softwareag.com" +pluginInfo.pluginType = "input" +pluginInfo.pluginClass = "AdabasAuditingInput" +pluginInfo.pluginName = "adabas_auditing_input" // must match the @LogstashPlugin annotation in the main plugin class +// =========================================================================== + +java { + sourceCompatibility = JavaVersion.VERSION_17 + targetCompatibility = JavaVersion.VERSION_17 +} + + +repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } +} + + +shadowJar { + archiveClassifier.set('') +} + +dependencies { + // Software AG dependencies + implementation fileTree(dir: 'libs') + + // Other dependencies + implementation 'com.google.code.gson:gson:2.10.1' + + implementation 'org.apache.commons:commons-lang3:3.7' + implementation 'org.apache.logging.log4j:log4j-api:2.17.0' // provided by Logstash + implementation 'org.apache.logging.log4j:log4j-core:2.17.0' // provided by Logstash + + implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "**/logstash-core.jar") + + testImplementation 'junit:junit:4.12' + testImplementation 'org.jruby:jruby-complete:9.4.7.0' +} + +clean { + delete "${projectDir}/Gemfile" + delete "${projectDir}/" + pluginInfo.pluginFullName() + ".gemspec" + delete "${projectDir}/lib/" + delete "${projectDir}/vendor/" + new FileNameFinder().getFileNames(projectDir.toString(), pluginInfo.pluginFullName() + "-?.?.?.gem").each { filename -> + delete filename + } +} + +tasks.withType(JavaCompile) { + options.encoding = 'UTF-8' +} + +tasks.register("vendor"){ + dependsOn shadowJar + doLast { + String vendorPathPrefix = "vendor/jar-dependencies" + String projectGroupPath = project.group.replaceAll('\\.', '/') + File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") + projectJarFile.mkdirs() + Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) + validatePluginJar(projectJarFile, project.group) + } +} + +tasks.register("generateRubySupportFiles") { + doLast { + generateRubySupportFilesForPlugin(project.description, project.group, version) + } +} + +tasks.register("removeObsoleteJars") { + doLast { + new FileNameFinder().getFileNames( + projectDir.toString(), + "vendor/**/" + pluginInfo.pluginFullName() + "*.jar", + "vendor/**/" + pluginInfo.pluginFullName() + "-" + version + ".jar").each { f -> + delete f + } + } +} + +tasks.register("gem"){ + dependsOn = [downloadAndInstallJRuby, removeObsoleteJars, vendor, generateRubySupportFiles] + doLast { + buildGem(projectDir, buildDir, pluginInfo.pluginFullName() + ".gemspec") + } +} diff --git a/input-plugin/logstash-input-adabas/gradle/wrapper/gradle-wrapper.jar b/input-plugin/logstash-input-adabas/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 000000000..41d9927a4 Binary files /dev/null and b/input-plugin/logstash-input-adabas/gradle/wrapper/gradle-wrapper.jar differ diff --git a/input-plugin/logstash-input-adabas/gradle/wrapper/gradle-wrapper.properties b/input-plugin/logstash-input-adabas/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 000000000..48c0a02ca --- /dev/null +++ b/input-plugin/logstash-input-adabas/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,5 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/input-plugin/logstash-input-adabas/gradlew b/input-plugin/logstash-input-adabas/gradlew new file mode 100755 index 000000000..1b6c78733 --- /dev/null +++ b/input-plugin/logstash-input-adabas/gradlew @@ -0,0 +1,234 @@ +#!/bin/sh + +# +# Copyright © 2015-2021 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# +############################################################################## + +# Attempt to set APP_HOME + +# Resolve links: $0 may be a link +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit + +APP_NAME="Gradle" +APP_BASE_NAME=${0##*/} + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + +# Collect all arguments for the java command; +# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of +# shell script including quotes and variable substitutions, so put them in +# double quotes to make sure that they get re-expanded; and +# * put everything else in single quotes, so that it's not re-expanded. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/input-plugin/logstash-input-adabas/gradlew.bat b/input-plugin/logstash-input-adabas/gradlew.bat new file mode 100644 index 000000000..ac1b06f93 --- /dev/null +++ b/input-plugin/logstash-input-adabas/gradlew.bat @@ -0,0 +1,89 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto execute + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/input-plugin/logstash-input-adabas/libs/adamf-metadata-rest-0.0.2.jar b/input-plugin/logstash-input-adabas/libs/adamf-metadata-rest-0.0.2.jar new file mode 100644 index 000000000..f8168c555 Binary files /dev/null and b/input-plugin/logstash-input-adabas/libs/adamf-metadata-rest-0.0.2.jar differ diff --git a/input-plugin/logstash-input-adabas/libs/auditing-parser-0.0.1.jar b/input-plugin/logstash-input-adabas/libs/auditing-parser-0.0.1.jar new file mode 100644 index 000000000..cdb216709 Binary files /dev/null and b/input-plugin/logstash-input-adabas/libs/auditing-parser-0.0.1.jar differ diff --git a/input-plugin/logstash-input-adabas/libs/buffer-parser-0.0.1.jar b/input-plugin/logstash-input-adabas/libs/buffer-parser-0.0.1.jar new file mode 100644 index 000000000..281405479 Binary files /dev/null and b/input-plugin/logstash-input-adabas/libs/buffer-parser-0.0.1.jar differ diff --git a/input-plugin/logstash-input-adabas/libs/eap-sdk-4.0.0.jar b/input-plugin/logstash-input-adabas/libs/eap-sdk-4.0.0.jar new file mode 100644 index 000000000..4e5edc179 Binary files /dev/null and b/input-plugin/logstash-input-adabas/libs/eap-sdk-4.0.0.jar differ diff --git a/input-plugin/logstash-input-adabas/libs/entirex-10.9.jar b/input-plugin/logstash-input-adabas/libs/entirex-10.9.jar new file mode 100644 index 000000000..6c70ddff6 Binary files /dev/null and b/input-plugin/logstash-input-adabas/libs/entirex-10.9.jar differ diff --git a/input-plugin/logstash-input-adabas/src/main/java/com/softwareag/adabas/auditing/logstash/AdabasAuditingInput.java b/input-plugin/logstash-input-adabas/src/main/java/com/softwareag/adabas/auditing/logstash/AdabasAuditingInput.java new file mode 100644 index 000000000..68908bfde --- /dev/null +++ b/input-plugin/logstash-input-adabas/src/main/java/com/softwareag/adabas/auditing/logstash/AdabasAuditingInput.java @@ -0,0 +1,323 @@ +/* + * Copyright © 2025 Software GmbH, Darmstadt, Germany and/or its licensors + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.softwareag.adabas.auditing.logstash; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.function.Consumer; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +// import org.apache.logging.log4j.LogManager; +// import org.apache.logging.log4j.Logger; + +import com.softwareag.adabas.auditingparser.ALAParse; +import com.softwareag.adabas.collector.sdk.DataObject; +import com.softwareag.entirex.aci.Broker; +import com.softwareag.entirex.aci.BrokerException; +import com.softwareag.entirex.aci.BrokerMessage; +import com.softwareag.entirex.aci.BrokerService; +import com.softwareag.entirex.aci.UnitofWork; + +import co.elastic.logstash.api.Configuration; +import co.elastic.logstash.api.Context; +import co.elastic.logstash.api.Input; +import co.elastic.logstash.api.LogstashPlugin; +import co.elastic.logstash.api.PluginConfigSpec; + +// class name must match plugin name +@LogstashPlugin(name = "adabas_auditing_input") +public class AdabasAuditingInput implements Input { + + // log4j2 logger + private static final Logger logger = LogManager.getLogger(AdabasAuditingInput.class); + + public static final PluginConfigSpec HOST_CONFIG = PluginConfigSpec.stringSetting("host", "localhost"); + public static final PluginConfigSpec PORT_CONFIG = PluginConfigSpec.numSetting("port", 3000); + public static final PluginConfigSpec BROKER_CLASS_CONFIG = PluginConfigSpec.stringSetting("brokerClass", + "class"); + public static final PluginConfigSpec BROKER_SERVER_CONFIG = PluginConfigSpec.stringSetting("brokerServer", + "server"); + public static final PluginConfigSpec BROKER_SERVICE_CONFIG = PluginConfigSpec.stringSetting("brokerService", + "service"); + public static final PluginConfigSpec USER_CONFIG = PluginConfigSpec.stringSetting("user", "user"); + public static final PluginConfigSpec TOKEN_CONFIG = PluginConfigSpec.stringSetting("token", "token"); + public static final PluginConfigSpec RETRY_INTERVAL_CONFIG = PluginConfigSpec.numSetting("retryInterval", + 5); + public static final PluginConfigSpec RETRY_COUNT_CONFIG = PluginConfigSpec.numSetting("retryCount", 10); + public static final PluginConfigSpec WAIT_TIME_CONFIG = PluginConfigSpec.numSetting("waitTime", 30); + public static final PluginConfigSpec RECEIVE_LENGTH_CONFIG = PluginConfigSpec.numSetting("receiveLength", + 32767); + public static final PluginConfigSpec COMPESSION_CONFIG = PluginConfigSpec.numSetting("compression", 0); + public static final PluginConfigSpec REST_URL_CONFIG = PluginConfigSpec.stringSetting("restURL", ""); + + private String id; + + // input plugin parameters + private String host; + private int port; + private String brokerClass; + private String brokerServer; + private String brokerService; + private String user; + private String token; + private String retryInterval; + private int retryCount; + private String waitTime; + private int receiveLength; + private int compression; + private String restURL; + + // EntireX Broker + private Broker broker; + private BrokerService service; + private UnitofWork uow; + + // Metadata API + private MetadataThread metadataApi; + + private final CountDownLatch done = new CountDownLatch(1); + private volatile boolean stopped; + + // all plugins must provide a constructor that accepts id, Configuration, and + // Context + public AdabasAuditingInput(String id, Configuration config, Context context) { + // constructors should validate configuration options + this.id = id; + + host = config.get(HOST_CONFIG); + port = Long.valueOf(config.get(PORT_CONFIG)).intValue(); + brokerClass = config.get(BROKER_CLASS_CONFIG); + brokerServer = config.get(BROKER_SERVER_CONFIG); + brokerService = config.get(BROKER_SERVICE_CONFIG); + user = config.get(USER_CONFIG); + token = config.get(TOKEN_CONFIG); + retryInterval = config.get(RETRY_INTERVAL_CONFIG) + "s"; + retryCount = Long.valueOf(config.get(RETRY_COUNT_CONFIG)).intValue(); + waitTime = config.get(WAIT_TIME_CONFIG) + "s"; + receiveLength = Long.valueOf(config.get(RECEIVE_LENGTH_CONFIG)).intValue(); + compression = Long.valueOf(config.get(COMPESSION_CONFIG)).intValue(); + restURL = config.get(REST_URL_CONFIG); + } + + @Override + public void start(Consumer> consumer) { + + logger.info("Starting Adabas Auditing input plugin"); + logger.info("Host ............ {}", host); + logger.info("Port ............ {}", port); + logger.info("Broker Class .... {}", brokerClass); + logger.info("Broker Server ... {}", brokerServer); + logger.info("Broker Service .. {}", brokerService); + logger.info("User ............ {}", user); + logger.info("Token ........... {}", token); + logger.info("Retry Interval .. {}", retryInterval); + logger.info("Retry Count ..... {}", retryCount); + logger.info("Wait Time ....... {}", waitTime); + logger.info("Receive Length .. {}", receiveLength); + logger.info("Compression ..... {}", compression); + logger.info("REST URL ........ {}", restURL); + + // The start method should push Map instances to the supplied + // QueueWriter + // instance. Those will be converted to Event instances later in the Logstash + // event + // processing pipeline. + // + // Inputs that operate on unbounded streams of data or that poll indefinitely + // for new + // events should loop indefinitely until they receive a stop request. Inputs + // that produce + // a finite sequence of events should loop until that sequence is exhausted or + // until they + // receive a stop request, whichever comes first. + + if (restURL.equals("")) { + restURL = "http://localhost:8080/metadata/JSON"; + } + if (restURL.contains("localhost")) { // starts local REST API + String regexPort = ":[0-9]+"; + Pattern pattern = Pattern.compile(regexPort); + Matcher matcher = pattern.matcher(restURL); + matcher.find(); + String url = matcher.group().split(":")[1]; + int port = Integer.valueOf(url); + metadataApi = new MetadataThread(port); + metadataApi.run(); + } + + try { + service = new BrokerService(getBroker(), + brokerClass + "/" + brokerServer + "/" + brokerService); + service.register(); + service.setDefaultWaittime(waitTime); + service.setMaxReceiveLen(receiveLength); + service.setAdjustReceiveLen(true); + getBroker().logon(); + } catch (BrokerException e) { + e.printStackTrace(); + } + + try { + while (!stopped) { + byte[] message = receive(); + if (message != null) { + ALAParse parser = ALAParse.getInstance(); + parser.setRestURL(restURL); + ArrayList parsedMessage = parser.parseBytesAsIndividualUABIs(message); + for (DataObject obj : parsedMessage) { + consumer.accept(Collections.singletonMap("adabas-auditing", convertToHashMap(obj))); + } + commit(); + } + } + } catch (Exception e) { + e.printStackTrace(); + } finally { + stopped = true; + done.countDown(); + } + } + + @Override + public void stop() { + stopped = true; // set flag to request cooperative stop of input + close(); + } + + @Override + public void awaitStop() throws InterruptedException { + done.await(); // blocks until input has stopped + } + + @Override + public Collection> configSchema() { + // should return a list of all configuration options for this plugin + return Arrays.asList(HOST_CONFIG, PORT_CONFIG, BROKER_CLASS_CONFIG, + BROKER_SERVER_CONFIG, BROKER_SERVICE_CONFIG, USER_CONFIG, TOKEN_CONFIG, RETRY_INTERVAL_CONFIG, + RETRY_COUNT_CONFIG, WAIT_TIME_CONFIG, RECEIVE_LENGTH_CONFIG, COMPESSION_CONFIG, + REST_URL_CONFIG); + } + + @Override + public String getId() { + return this.id; + } + + private void close() { + try { + if (service != null) { + service.deregisterImmediate(); + } + broker.logoff(); + broker.disconnect(); + } catch (Exception e) { + } + } + + private Broker getBroker() { + if (broker == null) { + broker = new Broker(host + ":" + port, user, token, Integer.MAX_VALUE); + } + return broker; + } + + /** + * Receive a message from Broker. + * + * @return Raw message from Broker as a byte array. + */ + private byte[] receive() { + + // logger.traceEntry(); + + BrokerMessage brokerMessage = null; + + if (uow == null) { + uow = new UnitofWork(service); + } + try { + brokerMessage = uow.receive(); + } catch (BrokerException ex) { + if (ex.getErrorClass() == 74 && ex.getErrorCode() == 74) { // simple timeout loop + // logger.debug("BrokerException!: {}: Error class = {}: Error code = {}", + // ex.getMessage(), ex.getErrorClass(), ex.getErrorCode()); + } else { // some real messaging error + try { + logger.error("BrokerException: {}: Error class = {}: Error code = {}", + ex.getMessage(), ex.getErrorClass(), ex.getErrorCode()); + Thread.sleep(5 * 1000); + } catch (InterruptedException e) { + } // wait before return + // connected = false; // all other errors assume connection lost + } + } + // logger.traceExit(); + + if (brokerMessage != null) + return brokerMessage.getMessage(); + else + return null; + } + + private void commit() throws BrokerException { + if (uow.getStatus().equals("RECV_ONLY") || + uow.getStatus().equals("RECV_LAST")) { + uow.commitEndConversation(); + uow = null; + } + } + + private HashMap convertToHashMap(DataObject object) { + HashMap map = new HashMap<>(); + // iterate over hashmap + for (Map.Entry entry : object.getList().entrySet()) { + String key = entry.getKey(); + Object value = entry.getValue(); + if (value instanceof DataObject) { + Object obj = convertToHashMap((DataObject) value); + map.put(key, obj); + } else { + if (value instanceof ArrayList) { + ArrayList list = new ArrayList<>(); + for (Object obj : (ArrayList) value) { + if (obj instanceof DataObject) { + list.add(convertToHashMap((DataObject) obj)); + } else { + list.add(obj); + } + } + map.put(key, list); + } else { + map.put(key, value); + } + } + } + return map; + } +} diff --git a/input-plugin/logstash-input-adabas/src/main/java/com/softwareag/adabas/auditing/logstash/MetadataThread.java b/input-plugin/logstash-input-adabas/src/main/java/com/softwareag/adabas/auditing/logstash/MetadataThread.java new file mode 100644 index 000000000..e8d7bfc54 --- /dev/null +++ b/input-plugin/logstash-input-adabas/src/main/java/com/softwareag/adabas/auditing/logstash/MetadataThread.java @@ -0,0 +1,45 @@ +/* + * Copyright © 2025 Software GmbH, Darmstadt, Germany and/or its licensors + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.softwareag.adabas.auditing.logstash; + +import com.softwareag.adabas.adamfmetadatarest.web.MetaDataController; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.IOException; + +public class MetadataThread extends Thread { + + private int apiPort; + private static final Logger logger = LogManager.getLogger(); + + public MetadataThread(int apiPort) { + this.apiPort = apiPort; + } + + @Override + public void run() { + try { + MetaDataController.startAPI(apiPort, false); + } catch (IOException e) { + logger.error("", e); + } + } +} diff --git a/input-plugin/logstash-input-adabas/src/test/java/com/softwareag/adabas/auditing/logstash/AdabasAuditingInputTest.java b/input-plugin/logstash-input-adabas/src/test/java/com/softwareag/adabas/auditing/logstash/AdabasAuditingInputTest.java new file mode 100644 index 000000000..a8e82990c --- /dev/null +++ b/input-plugin/logstash-input-adabas/src/test/java/com/softwareag/adabas/auditing/logstash/AdabasAuditingInputTest.java @@ -0,0 +1,52 @@ +package com.softwareag.adabas.auditing.logstash; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; + +import org.apache.commons.lang3.StringUtils; +import org.junit.Assert; +import org.junit.Test; +import org.logstash.plugins.ConfigurationImpl; + +import co.elastic.logstash.api.Configuration; + +public class AdabasAuditingInputTest { + + @Test + public void testAdabasAuditingInputPlugin() { + String prefix = "This is message"; + long eventCount = 5; + Map configValues = new HashMap<>(); + Configuration config = new ConfigurationImpl(configValues); + AdabasAuditingInput input = new AdabasAuditingInput("test-id", config, null); + TestConsumer testConsumer = new TestConsumer(); + input.start(testConsumer); + + List> events = testConsumer.getEvents(); + Assert.assertEquals(eventCount, events.size()); + for (int k = 1; k <= events.size(); k++) { + Assert.assertEquals(prefix + " " + StringUtils.center(k + " of " + eventCount, 20), + events.get(k - 1).get("message")); + } + } + + private static class TestConsumer implements Consumer> { + + private List> events = new ArrayList<>(); + + @Override + public void accept(Map event) { + synchronized (this) { + events.add(event); + } + } + + public List> getEvents() { + return events; + } + } + +} diff --git a/input-plugin/logstash-input-azure-event-hubs/README.md b/input-plugin/logstash-input-azure-event-hubs/README.md index 33d59766a..bc67e012c 100644 --- a/input-plugin/logstash-input-azure-event-hubs/README.md +++ b/input-plugin/logstash-input-azure-event-hubs/README.md @@ -1,22 +1,33 @@ ## azure_event_hubs input plug-in + ### Meet Azure event hubs + * Tested versions: 1.4.3 * Developed by Elastic -* Configuration instructions can be found on every relevant filter plugin readme page. For example: [Azure PostgresSQL](../../filter-plugin/logstash-filter-azure-postgresql-guardium/README.md#procedure) -* Supported Guardium versions: +* Configuration instructions can be found on every relevant filter plugin readme page. For + example: [Azure PostgresSQL](../../filter-plugin/logstash-filter-azure-postgresql-guardium/README.md#procedure) +* Supported Guardium versions: * Guardium Data Protection: 11.4 and above -This is a [Logstash](https://github.com/elastic/logstash) input plug-in for the universal connector that is featured in IBM Security Guardium. It pulls events from the Azure Event Hub. The events are then sent over to corresponding filter plugin which transforms these audit logs into a [Guardium record](https://github.com/IBM/universal-connectors/blob/main/common/src/main/java/com/ibm/guardium/universalconnector/commons/structures/Record.java) instance (which is a standard structure made out of several parts). The information is then sent over to Guardium. Guardium records include the accessor (the person who tried to access the data), the session, data, and exceptions. If there are no errors, the data contains details about the query "construct". The construct details the main action (verb) and collections (objects) involved. - +This is a [Logstash](https://github.com/elastic/logstash) input plug-in for the universal connector that is featured in +IBM Security Guardium. It pulls events from the Azure Event Hub. The events are then sent over to corresponding filter +plugin which transforms these audit logs into +a [Guardium record](https://github.com/IBM/universal-connectors/blob/main/common/src/main/java/com/ibm/guardium/universalconnector/commons/structures/Record.java) +instance (which is a standard structure made out of several parts). The information is then sent over to Guardium. +Guardium records include the accessor (the person who tried to access the data), the session, data, and exceptions. If +there are no errors, the data contains details about the query "construct". The construct details the main action (verb) +and collections (objects) involved. ## Purpose: -This plugin consumes events from Azure Event Hubs, a highly scalable data streaming platform and event ingestion service. Event producers send events to the Azure Event Hub, and this plugin consumes those events for use with Logstash. +This plugin consumes events from Azure Event Hubs, a highly scalable data streaming platform and event ingestion +service. Event producers send events to the Azure Event Hub, and this plugin consumes those events for use with +Logstash. ## Usage: ### Parameters: - + | Parameter | Input Type | Required | Default | |-----------|------------|----------|---------| | config_mode | String (basic or advanced) | | Basic | @@ -26,35 +37,55 @@ This plugin consumes events from Azure Event Hubs, a highly scalable data stream | decorate_events | Boolean | No | | | consumer_group | string | No | $Default | - - #### `config_mode` -The `config_mode` setting allows specifying configuration to either [Basic](https://www.elastic.co/guide/en/logstash/current/plugins-inputs-azure_event_hubs.html#plugins-inputs-azure_event_hubs-eh_basic_config) configuration (default) or [Advanced](https://www.elastic.co/guide/en/logstash/current/plugins-inputs-azure_event_hubs.html#plugins-inputs-azure_event_hubs-eh_advanced_config) configuration. + +The `config_mode` setting allows specifying configuration to +either [Basic](https://www.elastic.co/guide/en/logstash/current/plugins-inputs-azure_event_hubs.html#plugins-inputs-azure_event_hubs-eh_basic_config) +configuration (default) +or [Advanced](https://www.elastic.co/guide/en/logstash/current/plugins-inputs-azure_event_hubs.html#plugins-inputs-azure_event_hubs-eh_advanced_config) +configuration. #### `event_hub_connections` -The `event_hub_connections` setting allows specifying the list of connection strings that identifies the Event Hubs to be read. Connection strings include the EntityPath for the Event Hub. + +The `event_hub_connections` setting allows specifying the list of connection strings that identifies the Event Hubs to +be read. + +Each connection string must include the following mandatory components: Endpoint, SharedAccessKeyName, SharedAccessKey, +EntityPath (for the Event Hub) The event_hub_connections option is defined per Event Hub. All other configuration options are shared among Event Hubs. #### `initial_position` + The `initial_position` setting allows specifying when first reading from an Event Hub, start from this position: Valid options for `start_position` are: + * `beginning` - reads all pre-existing events in the Event Hub (default) * `end` - does not read any pre-existing events in the Event Hub -* look_back reads end minus a number of seconds worth of pre-existing events. You control the number of seconds using the initial_position_look_back option. +* look_back reads end minus a number of seconds worth of pre-existing events. You control the number of seconds using + the initial_position_look_back option. #### `threads` -The `threads` setting allows setting total number of threads used to process events. The value you set here applies to all Event Hubs. Even with advanced configuration, this value is a global setting, and can’t be set per event hub. + +The `threads` setting allows setting total number of threads used to process events. The value you set here applies to +all Event Hubs. Even with advanced configuration, this value is a global setting, and can’t be set per event hub. #### `decorate_events` -The `decorate_events` setting allows adding metadata about the Event Hub, including Event Hub name, consumer_group, processor_host, partition, offset, sequence, timestamp, and event_size. + +The `decorate_events` setting allows adding metadata about the Event Hub, including Event Hub name, consumer_group, +processor_host, partition, offset, sequence, timestamp, and event_size. #### `consumer_group` -The `consumer_group` setting allows specifying the Consumer group used to read the Event Hub(s). Create a consumer group specifically for Logstash. Then ensure that all instances of Logstash use that consumer group so that they can work together properly. + +The `consumer_group` setting allows specifying the Consumer group used to read the Event Hub(s). Create a consumer group +specifically for Logstash. Then ensure that all instances of Logstash use that consumer group so that they can work +together properly. #### Logstash Default config params + Other standard logstash parameters are available such as: + * `add_field` * `type` * `tags` @@ -65,7 +96,7 @@ Other standard logstash parameters are available such as: azure_event_hubs { config_mode => "basic" - event_hub_connections => [] + event_hub_connections => ["Endpoint=;SharedAccessKeyName=;SharedAccessKey=;EntityPath="] initial_position => "end" threads => 8 decorate_events => true diff --git a/input-plugin/logstash-input-beats/FilebeatInputPackage/Filebeat/input.conf b/input-plugin/logstash-input-beats/FilebeatInputPackage/Filebeat/input.conf index 70952721e..331f92df8 100644 --- a/input-plugin/logstash-input-beats/FilebeatInputPackage/Filebeat/input.conf +++ b/input-plugin/logstash-input-beats/FilebeatInputPackage/Filebeat/input.conf @@ -1,7 +1,7 @@ input{ beats { port => guc_input_param_port - ssl => guc_input_param_is_ssl + ssl_enabled => guc_input_param_is_ssl ssl_certificate_authorities => SSL_CERT_AUTH ssl_certificate => "/service/certs/external/tls.crt" ssl_key => "/service/certs/external/tls.key" diff --git a/input-plugin/logstash-input-beats/README.md b/input-plugin/logstash-input-beats/README.md index 9772bd1cc..cecbf58d8 100644 --- a/input-plugin/logstash-input-beats/README.md +++ b/input-plugin/logstash-input-beats/README.md @@ -91,7 +91,8 @@ https://www.elastic.co/guide/en/beats/filebeat/current/directory-layout.html • Locate "filebeat.inputs" in the filebeat.yml file, then add the following parameters. filebeat.inputs: - - type: log + - type: filestream + - id: - enabled: true paths: - diff --git a/input-plugin/logstash-input-cloudwatch-logs/README.md b/input-plugin/logstash-input-cloudwatch-logs/README.md index dfc7a1944..b229e1692 100644 --- a/input-plugin/logstash-input-cloudwatch-logs/README.md +++ b/input-plugin/logstash-input-cloudwatch-logs/README.md @@ -69,6 +69,11 @@ The `region` setting allows specify the region in which the Cloudwatch log group #### `codec` The `codec` setting allows specify, the codec used for input data. Input codecs are a convenient method for decoding the data before it enters the input, without needing a separate filter in the Logstash pipeline. + ##### `codec pattern` + The `codec pattern` is a regular expression that Logstash uses to identify lines that are either the start of a new multiline event or a continuation of a previous one. + For the Redshift and Postgres plug-ins, update the value of the pattern parameter from the inputs section as specified in the codec pattern. + For Redshift, add pattern from [here] ( https://github.com/IBM/universal-connectors/blob/main/filter-plugin/logstash-filter-redshift-aws-guardium/redshift-over-cloudwatch.conf ) + For Postgres, add pattern from [here] ( https://github.com/IBM/universal-connectors/blob/main/filter-plugin/logstash-filter-postgres-guardium/PostgresOverCloudWatchPackage/postgresCloudwatch.conf ) #### `role_arn` The role_arn setting allows you to specify which AWS IAM Role to assume, if any. This is used to generate temporary credentials, typically for cross-account access. To understand more about the settings to be followed while using this parameter, click [here]( ./SettingsForRoleArn.md ) @@ -108,4 +113,45 @@ Other standard logstash parameters are available such as: ``` grdapi add_domain_to_universal_connector_allowed_domains domain=amazonaws.com grdapi add_domain_to_universal_connector_allowed_domains domain=amazon.com -``` \ No newline at end of file +``` +``` + + +## Troubleshooting: +### Using VPC Endpoints for AWS Connectivity + +If Logstash is unable to connect to AWS CloudWatch Logs due to **network restrictions** (e.g. Traffic restricted to private subnets), you may need to route the connection through an **AWS VPC Endpoint**. + +### Solution: Use a VPC Endpoint with a Custom Endpoint URL + +When running Logstash inside a **VPC** (Virtual Private Cloud), especially in private subnets, the AWS SDK cannot reach the public `logs.{region}.amazonaws.com` endpoint unless explicitly allowed. +To solve this, configure the plugin to use your VPC Endpoint along with AWS's bundled Certificate Authority (CA) for secure communication. + +#### Example Configuration + +```logstash +input { + cloudwatch_logs { + # Default Configuration + log_group => "/aws/lambda/my-lambda-function" + region => "us-east-1" + access_key_id => "YOUR_ACCESS_KEY" + secret_access_key => "YOUR_SECRET_KEY" + + # Use your private VPC Endpoint instead of the public AWS endpoint + endpoint => "https://vpce-xxxxxxxxabcdef.logs.us-east-1.vpce.amazonaws.com" + # Ensures the connection uses AWS's trusted root certificates + use_aws_bundled_ca => true + } +} +``` + +> **Note**: Replace the `endpoint` URL with your actual VPC Endpoint DNS name. You can find this in the AWS Console under **VPC > Endpoints**. + +### Additional Notes + +- Make sure the VPC Endpoint is created for the **CloudWatch Logs interface service** (`com.amazonaws.us-east-1.logs`). +- Ensure your **subnet** and **security group** allow HTTPS traffic to the endpoint. +- If you're using **IAM roles** (e.g., EC2 instance roles), you can omit the access keys. + +For more information, see the official AWS docs: [VPC Interface Endpoints for CloudWatch Logs](https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/CloudWatchLogs-and-InterfaceVPC.html). diff --git a/input-plugin/logstash-input-couchbase-capella/README.md b/input-plugin/logstash-input-couchbase-capella/README.md index 5f289f116..6c5291c26 100644 --- a/input-plugin/logstash-input-couchbase-capella/README.md +++ b/input-plugin/logstash-input-couchbase-capella/README.md @@ -4,70 +4,78 @@ * Tested versions: 1.0.0 * Developed by IBM * Configuration instructions can be found on [Guardium Couchbase Capella documentation](../../input-plugin/logstash-input-couchbase-capella/README.md#setup-couchbase-capella-cluster) -* Supported Guardium versions: - * Guardium Data Protection: 12.0 and above - -This is a java [Logstash](https://github.com/elastic/logstash) input plug-in for the universal connector that is featured in IBM Security Guardium. It reads events and messages from the Mongo Atlas audit log into a [Guardium record](https://github.com/IBM/universal-connectors/blob/main/common/src/main/java/com/ibm/guardium/universalconnector/commons/structures/Record.java) instance (which is a standard structure made out of several parts). The information is then sent over to Guardium. Guardium records include the accessor (the person who tried to access the data), the session, data, and exceptions. If there are no errors, the data contains details about the query "construct". The construct details the main action (verb) and collections (objects) involved. - -# Setup Couchbase Capella Cluster -1. Login to Capella using https://cloud.couchbase.com. -2. Click ```Create Cluster```. -3. Select My First Project as the project for your cluster. -4. Select one of the ```Cluster Option```. -5. In the Name field, enter a name for your cluster or accept the default option. -6. Select one of the available cloud service providers: ```AWS```, ```Google Cloud```, ```Azure```. +* Supported Guardium versions: Guardium Data Protection: 12.0 or later + +This is a java [Logstash](https://github.com/elastic/logstash) input plug-in for the universal connector that is featured in IBM Security Guardium. It reads events and messages from the Mongo Atlas audit log into a [Guardium record](https://github.com/IBM/universal-connectors/blob/main/common/src/main/java/com/ibm/guardium/universalconnector/commons/structures/Record.java) instance, which is a standard structure made out of several parts. Then the information is sent to Guardium. Guardium records include the accessor (the person who tries to access the data), the session, data, and exceptions. If there are no errors, the data contains details about the query `construct`. The construct details the main action (verb) and collections (objects) involved. + +## Setting up a Couchbase Capella Cluster +1. Login to Capella by using https://cloud.couchbase.com. +2. Click **Create Cluster**. +3. Select **My First Project** as the project for your cluster. +4. Select a **Cluster Option**. +5. In the **Name** field, enter a name for your cluster or accept the default option. +6. Select one of the available cloud service providers: **AWS**, **Google Cloud**, or **Azure**. 7. Select an available geographic region for your cluster. -8. Enter a CIDR Block for your cluster, or accept the default. For more information about how to configure a CIDR block, see https://docs.couchbase.com/cloud/clusters/databases.html#cloud-provider -9. Click ```Create Cluster``` to deploy your free tier operational cluster with Capella. - -# Get Access Bearer Token -1. Login to Capella using https://cloud.couchbase.com. -2. Select ```Settings``` -3. Select ```API Keys``` -4. Select ```Generate Key``` on the upper left side. -5. Enter ```Key Name```, select one of the Organization Roles -6. Click ```Generate key``` -Note: You can also create API Key through the endpoint, the detail information see https://docs.couchbase.com/cloud/management-api-reference/index.html - -# Setup Couchbase Capella Cluster Auditing -1. Open the Audit tab of the Security settings by selecting Security -> Audit. +8. Enter a CIDR block for your cluster, or accept the default. For more information about configuring a CIDR block, see https://docs.couchbase.com/cloud/clusters/databases.html#cloud-provider. +9. Click **Create Cluster** to deploy your free tier operational cluster with Capella. + +## Obtaining an access bearer token +1. Login to Capella by using https://cloud.couchbase.com. +2. Click **Settings** > **API Keys** > **Generate Key**. +3. Enter a **Key Name**. Then select one of the organization roles. +4. Click **Generate key**. + + **Note:** You can also create an API Key through the endpoint. For more information, see [Create API Key](https://docs.couchbase.com/cloud/management-api-reference/index.html#tag/Api-Keys/operation/postOrganizationAPIKeys). + +## Setting up Couchbase Capella cluster auditing +1. Go to **Security** > **Audit** tab. 2. Turn on the Audit events & write them to a log toggle. -3. For more information, -https://docs.couchbase.com/server/current/manage/manage-security/manage-auditing.html + For more information, [Managing Auditing](https://docs.couchbase.com/server/current/manage/manage-security/manage-auditing.html). + +## Configuring the input Capella plugin in Guardium -# Configuring the Input Capella plugin in Guardium ### Before you begin -* Configure the policies you require. See [policies](/docs/#policies) for more information. -* You must have permission for the S-Tap Management role. The admin user includes this role by default +* Configure the policies you need. For more information, see [Policies](/docs/#policies). +* You must have permissions for the S-Tap Management role. By default, the admin user is assigned the S-Tap Management role. * Download the [logstash-input-couchbase_capella_input](logstash-input-couchbase_capella_input.zip) plug-in. ### Procedure -1. On the collector, go to ```Setup``` > ```Tools and Views``` > ```Configure Universal Connector```. +1. On the collector, go to **Setup** > **Tools and Views** > **Configure Universal Connector**. 2. Enable the universal connector if it is disabled. -3. Click ```Upload File``` and select the offline [logstash-input-couchbase_capella_input](logstash-input-couchbase_capella_input.zip) plug-in. After it is uploaded, click ```OK```. -4. Click the Plus sign to open the Connector Configuration dialog box. -5. Type a name in the Connector name field. -6. Update the input section to add the details from the [capellaCouchbase.conf](../../filter-plugin/logstash-filter-capella-guardium/capellaCouchbaseOverCapellaPackage/capella/capellaCouchbase.conf) file's input part, omitting the keyword "input{" at the beginning and its corresponding "}" at the end. -7. Update the filter section to add the details from the [capellaCouchbase.conf](../../filter-plugin/logstash-filter-capella-guardium/capellaCouchbaseOverCapellaPackage/capella/capellaCouchbase.conf) file's filter part, omitting the keyword "filter{" at the beginning and its corresponding "}" at the end. -8. The 'type' fields should match in the input and filter configuration sections. This field should be unique for every individual connector added. -9. Click ```Save```. Guardium validates the new connector and displays it in the Configure Universal Connector page. -10. After the offline plug-in is installed and the configuration is uploaded and saved in the Guardium machine, restart the Universal Connector using the ```Disable/Enable``` button. - -# Usage +3. Click **Upload File** and select the offline [logstash-input-couchbase_capella_input](logstash-input-couchbase_capella_input.zip) plug-in. After it is uploaded, click **OK**. +4. Click the **Plus** icon to open the Connector Configuration dialog box. +5. In the **Connector name** field, enter a name. +6. Update the input section to add the details from the [capellaCouchbase.conf](../../filter-plugin/logstash-filter-capella-guardium/capellaCouchbaseOverCapellaPackage/capellaCouchbase.conf) file's ``input`` section, omitting the keyword ``input{`` at the beginning and its corresponding ``}`` at the end. +7. Update the filter section to add the details from the [capellaCouchbase.conf](../../filter-plugin/logstash-filter-capella-guardium/capellaCouchbaseOverCapellaPackage/capellaCouchbase.conf) file's ``filter`` section, omitting the keyword ``filter{`` at the beginning and its corresponding ``}`` at the end. +8. Make sure that the ``type`` fields in the ``input`` and ``filter`` configuration sections align. This field must be unique for each connector added to the system. +9. Click **Save**. Guardium validates the new connector and displays it in the Configure Universal Connector page. +10. After the offline plug-in is installed and the configuration is uploaded and saved in the Guardium machine, restart the universal connector by using the **Disable/Enable** button. + +## Limitations +* No more than three historical export requests are permitted over 24-hour period. +* The original Capella audit log contains no values for the following fields: Database Name, Service Name. + +Notes: +* It may take approximately 30 minutes for data to appear in the Full SQL report. + +## Usage + ### Parameters | Parameter | Input Type | Required | Default | |----------------|------------|----------|--------------| | query-interval | number | Yes | 8*3600 | | query_length | number | Yes | 3600 | -| api_base_url | string | Yes |`https://cloudapi.cloud.couchbase.com/v4` | +| api_base_url | string | Yes |https://cloudapi.cloud.couchbase.com/v4 | | organization_id | string | Yes | | | project_id | string | Yes | | | cluster_id | string | Yes | | | auth_token | string | Yes | | -# Couchbase Capella Cluster Audit Log Event -## Sample log: -{ +## Couchbase Capella cluster audit log events + +### Sample log + +```{ "description": "Successful login to couchbase cluster", "id": 8192, "local": { @@ -89,13 +97,8 @@ https://docs.couchbase.com/server/current/manage/manage-security/manage-auditing "sessionid": "ba2760cee506d0293a8b4a0bf83687b807329667", "timestamp": "2021-02-09T14:44:17.938Z" } +``` -## Supported audit messages -For more information, reference here https://docs.couchbase.com/server/current/audit-event-reference/audit-event-reference.html - -## Limitations -* ## Limitations -* No more than three historical export requests are permitted over 24-hour period. +For more information about supported audit messages, see [Audit Event Reference](https://docs.couchbase.com/server/current/audit-event-reference/audit-event-reference.html).

-* ## Suggestion -* In the configuration file, query_interval and query_length have no restrictions, with both fields defaulting to 1 hour. However, we recommend using shorter intervals rather than longer ones, as a larger interval may result in unnecessary waiting time before the next cycle, leading to resource inefficiency. \ No newline at end of file +**Tip:** In the configuration file, `query_interval` and `query_length` have no restrictions, and both fields are set to **1 hour** by default. To improve resource efficiency, use shorter intervals as larger intervals may result in unnecessary waiting time before the next cycle. diff --git a/input-plugin/logstash-input-couchbase-capella/build.gradle b/input-plugin/logstash-input-couchbase-capella/build.gradle index 977160c2b..5ef213f2a 100644 --- a/input-plugin/logstash-input-couchbase-capella/build.gradle +++ b/input-plugin/logstash-input-couchbase-capella/build.gradle @@ -3,12 +3,19 @@ import static java.nio.file.StandardCopyOption.REPLACE_EXISTING buildscript { repositories { + mavenLocal() maven { - url "https://plugins.gradle.org/m2/" + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } } - mavenCentral() gradlePluginPortal() - jcenter() + } + + dependencies { + classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' } } @@ -24,6 +31,10 @@ ext { shadowGradlePluginVersion = '8.1.1' } +def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" // =========================================================================== @@ -44,8 +55,8 @@ pluginInfo.pluginName = "couchbase_capella_input" // must match the @Logsta // =========================================================================== java { - sourceCompatibility = JavaVersion.VERSION_11 - targetCompatibility = JavaVersion.VERSION_11 + sourceCompatibility = JavaVersion.VERSION_17 + targetCompatibility = JavaVersion.VERSION_17 } def jacocoVersion = '0.8.4' @@ -59,7 +70,14 @@ def minimumCoverage = Float.valueOf(minimumCoverageStr) / 100 repositories { - mavenCentral() + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } } shadowJar { @@ -70,11 +88,12 @@ dependencies { implementation 'org.apache.commons:commons-lang3:3.7' compileOnly 'org.apache.logging.log4j:log4j-api:2.17.0' // provided by Logstash implementation 'org.apache.logging.log4j:log4j-core:2.17.0' // provided by Logstash + implementation group: 'commons-beanutils', name: 'commons-beanutils', version: versions.dependencies.commonsBeanutils implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "**/logstash-core.jar") implementation 'org.apache.httpcomponents:httpclient:4.5.13' implementation 'com.google.code.gson:gson:2.8.9' - implementation 'com.google.guava:guava:30.1-jre' + implementation 'com.google.guava:guava:' + versions.dependencies.guava testImplementation 'junit:junit:4.12' @@ -106,7 +125,6 @@ tasks.register("vendor"){ File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") projectJarFile.mkdirs() Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) - validatePluginJar(projectJarFile, project.group) } } @@ -164,8 +182,8 @@ jacocoTestReport { csv { enabled true } - html.destination file("${buildDir}/reports/jacoco") - csv.destination file("${buildDir}/reports/jacoco/all.csv") + html.outputLocation = file("${buildDir}/reports/jacoco") + csv.outputLocation = file("${buildDir}/reports/jacoco/all.csv") } executionData.from fileTree(dir: "${buildDir}/jacoco/", includes: [ '**/*.exec' diff --git a/input-plugin/logstash-input-couchbase-capella/logstash-input-couchbase_capella_input.zip b/input-plugin/logstash-input-couchbase-capella/logstash-input-couchbase_capella_input.zip index 1cd342369..4b7a53763 100644 Binary files a/input-plugin/logstash-input-couchbase-capella/logstash-input-couchbase_capella_input.zip and b/input-plugin/logstash-input-couchbase-capella/logstash-input-couchbase_capella_input.zip differ diff --git a/input-plugin/logstash-input-couchbase-capella/src/main/java/org/logstashplugins/CouchbaseCapellaInput.java b/input-plugin/logstash-input-couchbase-capella/src/main/java/org/logstashplugins/CouchbaseCapellaInput.java index cd88efe42..e9cecc48b 100644 --- a/input-plugin/logstash-input-couchbase-capella/src/main/java/org/logstashplugins/CouchbaseCapellaInput.java +++ b/input-plugin/logstash-input-couchbase-capella/src/main/java/org/logstashplugins/CouchbaseCapellaInput.java @@ -230,6 +230,13 @@ public void start(Consumer> consumer) { log.debug("invalid json exception: {}", originalLine); continue; } + + // Drop system generated logs + if (line.contains("\"user\":\"@")){ + log.debug("Drop logs: {}", line); + continue; + } + HashMap map = new HashMap(); map.put(Constants.OUTPUT_KEY_MESSAGE, line); map.put(Constants.OUTPUT_KEY_ORG, organizationID); diff --git a/input-plugin/logstash-input-couchbase-capella/src/test/java/org/logstashplugins/CouchbaseCapellaInputTest.java b/input-plugin/logstash-input-couchbase-capella/src/test/java/org/logstashplugins/CouchbaseCapellaInputTest.java index c847edc82..35e1cc3d9 100644 --- a/input-plugin/logstash-input-couchbase-capella/src/test/java/org/logstashplugins/CouchbaseCapellaInputTest.java +++ b/input-plugin/logstash-input-couchbase-capella/src/test/java/org/logstashplugins/CouchbaseCapellaInputTest.java @@ -18,6 +18,7 @@ import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; +import java.util.TimeZone; import static org.mockserver.integration.ClientAndServer.startClientAndServer; import static org.mockserver.stop.Stop.stopQuietly; @@ -60,7 +61,7 @@ public void testCouchbaseCapellaInputTest() { var baseUrl = String.format("http://%s:%d/%s", mockServerHost, mockServerPort, mockServerApiBasePath); Map configValues = new HashMap<>(); configValues.put(CouchbaseCapellaInput.INTERVAL_CONFIG.name(), 2L); - configValues.put(CouchbaseCapellaInput.QUERY_LENGTH_CONFIG.name(), 1L); + configValues.put(CouchbaseCapellaInput.QUERY_LENGTH_CONFIG.name(), 20 * 60L); configValues.put(CouchbaseCapellaInput.API_BASE_URL_CONFIG.name(), baseUrl); configValues.put(CouchbaseCapellaInput.ORG_ID_CONFIG.name(), "success-org"); configValues.put(CouchbaseCapellaInput.PROJECT_ID_CONFIG.name(), "success-project"); @@ -106,21 +107,28 @@ public void run() { @Test public void testCouchbaseCapellaInput_EpochToISO8601() { - var baseUrl = String.format("http://%s:%d/%s", mockServerHost, mockServerPort, mockServerApiBasePath); - Map configValues = new HashMap<>(); - configValues.put(CouchbaseCapellaInput.INTERVAL_CONFIG.name(), 2L); - configValues.put(CouchbaseCapellaInput.QUERY_LENGTH_CONFIG.name(), 1L); - configValues.put(CouchbaseCapellaInput.API_BASE_URL_CONFIG.name(), baseUrl); - configValues.put(CouchbaseCapellaInput.ORG_ID_CONFIG.name(), "success-org"); - configValues.put(CouchbaseCapellaInput.PROJECT_ID_CONFIG.name(), "success-project"); - configValues.put(CouchbaseCapellaInput.CLUSTER_ID_CONFIG.name(), "success-cluster"); - configValues.put(CouchbaseCapellaInput.AUTH_TOKEN_CONFIG.name(), "Bearer good_token"); - - Configuration config = new ConfigurationImpl(configValues); - CouchbaseCapellaInput input = new CouchbaseCapellaInput("test-id", config, null); + TimeZone originalTimeZone = TimeZone.getDefault(); - var dateTimeStr = input.epochSecToISO8601DateTimeString(1747702429L); - Assert.assertEquals("2025-05-19T20:53:49Z", dateTimeStr); + try { + TimeZone.setDefault(TimeZone.getTimeZone("UTC")); + var baseUrl = String.format("http://%s:%d/%s", mockServerHost, mockServerPort, mockServerApiBasePath); + Map configValues = new HashMap<>(); + configValues.put(CouchbaseCapellaInput.INTERVAL_CONFIG.name(), 2L); + configValues.put(CouchbaseCapellaInput.QUERY_LENGTH_CONFIG.name(), 20 * 60L); + configValues.put(CouchbaseCapellaInput.API_BASE_URL_CONFIG.name(), baseUrl); + configValues.put(CouchbaseCapellaInput.ORG_ID_CONFIG.name(), "success-org"); + configValues.put(CouchbaseCapellaInput.PROJECT_ID_CONFIG.name(), "success-project"); + configValues.put(CouchbaseCapellaInput.CLUSTER_ID_CONFIG.name(), "success-cluster"); + configValues.put(CouchbaseCapellaInput.AUTH_TOKEN_CONFIG.name(), "Bearer good_token"); + + Configuration config = new ConfigurationImpl(configValues); + CouchbaseCapellaInput input = new CouchbaseCapellaInput("test-id", config, null); + + var dateTimeStr = input.epochSecToISO8601DateTimeString(1747702429L); + Assert.assertEquals("2025-05-20T00:53:49Z", dateTimeStr); + } finally { + TimeZone.setDefault(originalTimeZone); // Restore after test + } } @Test(expected = IllegalArgumentException.class) @@ -128,7 +136,7 @@ public void testCouchbaseCapellaInput_EmptyBaseUrl() { Map configValues = new HashMap<>(); configValues.put(CouchbaseCapellaInput.INTERVAL_CONFIG.name(), 2L); - configValues.put(CouchbaseCapellaInput.QUERY_LENGTH_CONFIG.name(), 1L); + configValues.put(CouchbaseCapellaInput.QUERY_LENGTH_CONFIG.name(), 20 * 60L); configValues.put(CouchbaseCapellaInput.API_BASE_URL_CONFIG.name(), ""); configValues.put(CouchbaseCapellaInput.ORG_ID_CONFIG.name(), "success-org"); configValues.put(CouchbaseCapellaInput.PROJECT_ID_CONFIG.name(), "success-project"); @@ -147,7 +155,7 @@ public void testCouchbaseCapellaInput_EmptyOrgID() { var baseUrl = String.format("http://%s:%d/%s", mockServerHost, mockServerPort, mockServerApiBasePath); Map configValues = new HashMap<>(); configValues.put(CouchbaseCapellaInput.INTERVAL_CONFIG.name(), 2L); - configValues.put(CouchbaseCapellaInput.QUERY_LENGTH_CONFIG.name(), 1L); + configValues.put(CouchbaseCapellaInput.QUERY_LENGTH_CONFIG.name(), 20 * 60L); configValues.put(CouchbaseCapellaInput.API_BASE_URL_CONFIG.name(), baseUrl); configValues.put(CouchbaseCapellaInput.ORG_ID_CONFIG.name(), ""); configValues.put(CouchbaseCapellaInput.PROJECT_ID_CONFIG.name(), "success-project"); @@ -166,7 +174,7 @@ public void testCouchbaseCapellaInput_EmptyProjectID() { var baseUrl = String.format("http://%s:%d/%s", mockServerHost, mockServerPort, mockServerApiBasePath); Map configValues = new HashMap<>(); configValues.put(CouchbaseCapellaInput.INTERVAL_CONFIG.name(), 2L); - configValues.put(CouchbaseCapellaInput.QUERY_LENGTH_CONFIG.name(), 1L); + configValues.put(CouchbaseCapellaInput.QUERY_LENGTH_CONFIG.name(), 20 * 60L); configValues.put(CouchbaseCapellaInput.API_BASE_URL_CONFIG.name(), baseUrl); configValues.put(CouchbaseCapellaInput.ORG_ID_CONFIG.name(), "success-org"); configValues.put(CouchbaseCapellaInput.PROJECT_ID_CONFIG.name(), ""); @@ -185,7 +193,7 @@ public void testCouchbaseCapellaInput_EmptyClusterID() { var baseUrl = String.format("http://%s:%d/%s", mockServerHost, mockServerPort, mockServerApiBasePath); Map configValues = new HashMap<>(); configValues.put(CouchbaseCapellaInput.INTERVAL_CONFIG.name(), 2L); - configValues.put(CouchbaseCapellaInput.QUERY_LENGTH_CONFIG.name(), 1L); + configValues.put(CouchbaseCapellaInput.QUERY_LENGTH_CONFIG.name(), 20 * 60L); configValues.put(CouchbaseCapellaInput.API_BASE_URL_CONFIG.name(), baseUrl); configValues.put(CouchbaseCapellaInput.ORG_ID_CONFIG.name(), "success-org"); configValues.put(CouchbaseCapellaInput.PROJECT_ID_CONFIG.name(), "success-project"); @@ -204,7 +212,7 @@ public void testCouchbaseCapellaInput_EmptyAuthToken() { var baseUrl = String.format("http://%s:%d/%s", mockServerHost, mockServerPort, mockServerApiBasePath); Map configValues = new HashMap<>(); configValues.put(CouchbaseCapellaInput.INTERVAL_CONFIG.name(), 2L); - configValues.put(CouchbaseCapellaInput.QUERY_LENGTH_CONFIG.name(), 1L); + configValues.put(CouchbaseCapellaInput.QUERY_LENGTH_CONFIG.name(), 20 * 60L); configValues.put(CouchbaseCapellaInput.API_BASE_URL_CONFIG.name(), baseUrl); configValues.put(CouchbaseCapellaInput.ORG_ID_CONFIG.name(), "success-org"); configValues.put(CouchbaseCapellaInput.PROJECT_ID_CONFIG.name(), "success-project"); diff --git a/input-plugin/logstash-input-couchbase-capella/src/test/resources/mocks/mock-audit-log.tar.gz b/input-plugin/logstash-input-couchbase-capella/src/test/resources/mocks/mock-audit-log.tar.gz index 010185ed7..438d2d38a 100644 Binary files a/input-plugin/logstash-input-couchbase-capella/src/test/resources/mocks/mock-audit-log.tar.gz and b/input-plugin/logstash-input-couchbase-capella/src/test/resources/mocks/mock-audit-log.tar.gz differ diff --git a/input-plugin/logstash-input-google-pubsub/README.md b/input-plugin/logstash-input-google-pubsub/README.md index 09afa32f3..d6aca0080 100644 --- a/input-plugin/logstash-input-google-pubsub/README.md +++ b/input-plugin/logstash-input-google-pubsub/README.md @@ -114,7 +114,7 @@ The `create_subscription` setting, if set true, will have the input plugin creat - This requires additional permissions to be granted to the client (i.e. the Service Account) and is not recommended for most use-cases. If you still need to use it, grant the Service Account the "Cloud Pub/Sub Service Agent" Role in *IAM & Admin > Service Accounts > Grant Access* #### `max_messages` -The `max_messages` setting, helps to mitigate the issues caused due to subscriber client processing and acknowledging the messages more slowly than Pub/Sub sending them to the client. This option helps to control the rate at which the subscriber receives messages. +The `max_messages` setting, helps to mitigate the issues caused due to subscriber client processing and acknowledging the messages more slowly than Pub/Sub sending them to the client. This option helps to control the rate at which the subscriber receives messages. The value needs to adjusted according to traffic load. #### Logstash Default config params diff --git a/input-plugin/logstash-input-http/README.md b/input-plugin/logstash-input-http/README.md deleted file mode 100644 index 0201ccf0a..000000000 --- a/input-plugin/logstash-input-http/README.md +++ /dev/null @@ -1,41 +0,0 @@ -## http input plug-in -### Meet http -* Tested versions: 3.10.2 -* Developed by Elastic -* Configuration instructions can be found on [HTTP](https://github.com/IBM/universal-connectors/blob/main/filter-plugin/logstash-filter-trino-guardium) -* Supported Guardium versions: - * Guardium Data Protection: 11.3 and above - -This is a [Logstash](https://github.com/elastic/logstash) input plug-in for the universal connector that is featured in IBM Security Guardium. It enables Logstash to receive events from the http framework. The events are then sent over to corresponding filter plugin which transforms these audit logs into a [Guardium record](https://github.com/IBM/universal-connectors/blob/main/common/src/main/java/com/ibm/guardium/universalconnector/commons/structures/Record.java) instance (which is a standard structure made out of several parts). The information is then sent over to Guardium. Guardium records include the accessor (the person who tried to access the data), the session, data, and exceptions. If there are no errors, the data contains details about the query "construct". The construct details the main action (verb) and collections (objects) involved. - -## Purpose: - -Specify a port, and this plugin will poll the same port on the Logstash host for any new log events. - - -## Usage: - -### Parameters: - -| Parameter | Input Type | Required | Default | -|-----------|------------|----------|---------| -| port | number | Yes | | - -#### `port` -The `port` setting allows specifying a port on which the Logstash host listens to and pull the log events written there. - - -#### Logstash Default config params -Other standard logstash parameters are available such as: -* `add_field` -* `type` -* `tags` - -### Example - - input { - http { - port => 5060 - type => "http" - } - } diff --git a/input-plugin/logstash-input-http/httpInputPackage/http/input.conf b/input-plugin/logstash-input-http/httpInputPackage/http/input.conf deleted file mode 100644 index 63424102c..000000000 --- a/input-plugin/logstash-input-http/httpInputPackage/http/input.conf +++ /dev/null @@ -1,6 +0,0 @@ -input{ - http { - port => guc_input_param_port - type => "http" - } -} \ No newline at end of file diff --git a/input-plugin/logstash-input-http/logstash-input-http_guardium_filter.zip b/input-plugin/logstash-input-http/logstash-input-http_guardium_filter.zip deleted file mode 100644 index f058403d6..000000000 Binary files a/input-plugin/logstash-input-http/logstash-input-http_guardium_filter.zip and /dev/null differ diff --git a/input-plugin/logstash-input-http/logstash-offline-input-http-plugins.zip b/input-plugin/logstash-input-http/logstash-offline-input-http-plugins.zip deleted file mode 100644 index 8bbf08257..000000000 Binary files a/input-plugin/logstash-input-http/logstash-offline-input-http-plugins.zip and /dev/null differ diff --git a/input-plugin/logstash-input-mongo-atlas/CHANGELOG.md b/input-plugin/logstash-input-mongo-atlas/CHANGELOG.md index 44dc6c7c0..5df85af5b 100644 --- a/input-plugin/logstash-input-mongo-atlas/CHANGELOG.md +++ b/input-plugin/logstash-input-mongo-atlas/CHANGELOG.md @@ -1,3 +1,9 @@ +## 1.0.7 - 01/05/2026 +- GRD-114688: Update input configuration to include mongo-api-url + +## 1.0.7 +- GRD-114688: Guardium Implementation for Mongo Atlas. + ## 1.0.1 - Fix gradle incompatibility issue with Logstash 7.9 and above. diff --git a/input-plugin/logstash-input-mongo-atlas/InputMongoAtlasPackage/MongoAtlas/input.conf b/input-plugin/logstash-input-mongo-atlas/InputMongoAtlasPackage/MongoAtlas/input.conf index 3f87c4388..838dcbed1 100644 --- a/input-plugin/logstash-input-mongo-atlas/InputMongoAtlasPackage/MongoAtlas/input.conf +++ b/input-plugin/logstash-input-mongo-atlas/InputMongoAtlasPackage/MongoAtlas/input.conf @@ -4,6 +4,7 @@ input { public-key => "${GUC_CRED_PUBLIC_KEY}" private-key => "${GUC_CRED_PRIVATE_KEY}" group-id => "guc_input_param_group_id" + mongo-api-url => "guc_input_param_mongo_api_url" hostname => "guc_input_param_hostname" type => "guc_input_param_type" } diff --git a/input-plugin/logstash-input-mongo-atlas/README.md b/input-plugin/logstash-input-mongo-atlas/README.md index 6ea6ab854..07165afa0 100644 --- a/input-plugin/logstash-input-mongo-atlas/README.md +++ b/input-plugin/logstash-input-mongo-atlas/README.md @@ -37,7 +37,7 @@ In order to support a few features one zip has to be added with the name "guardi 2. Click ```Create API Key```. 3. Enter the API Key Information. a.Enter a Description. - b.In the Organization Permissions menu, select the new role or roles for the API key. + b.In the Organization Permissions menu, select the new role or roles for the API key. Minimum permission: ```Project Data Access Read Only``` (For more information, https://www.mongodb.com/docs/atlas/reference/user-roles/#mongodb-authrole-Project-Data-Access-Read-Only). 4. Click ```Next```. 5. Copy and save the Public Key. 6. Copy and save the Private Key. @@ -82,7 +82,7 @@ grdapi add_domain_to_universal_connector_allowed_domains domain=cloud.mongodb.co ## Procedure 1. On the collector, go to ```Setup``` > ```Tools and Views``` > ```Configure Universal Connector```. 2. Enable the Guardium Universal Connector if is in disabled state before uploading the UC plug-in. -3. Click ```Upload File``` and select the offline [logstash-input-mongo_atlas_input.zip](https://github.com/IBM/universal-connectors/releases/download/v1.5.2/logstash-input-mongo_atlas_input.zip) plug-in. After it is uploaded, click ```OK```. +3. Click ```Upload File``` and select the offline [logstash-input-mongo_atlas_input.zip](https://github.com/IBM/universal-connectors/releases/download/main_dev/logstash-input-mongo_atlas_input.zip) plug-in. After it is uploaded, click ```OK```. 4. Click the Plus sign to open the Connector Configuration dialog box. 5. Type a name in the ```Connector name``` field. 6. Update the input section to add the details from the [input-mongo-atlas.conf](https://github.com/IBM/universal-connectors/blob/main/input-plugin/logstash-input-mongo-atlas/input-mongo-atlas.conf) file input section, omitting the keyword "input{" at the beginning and its corresponding "}" at the end. diff --git a/input-plugin/logstash-input-mongo-atlas/VERSION b/input-plugin/logstash-input-mongo-atlas/VERSION index af0b7ddbf..238d6e882 100644 --- a/input-plugin/logstash-input-mongo-atlas/VERSION +++ b/input-plugin/logstash-input-mongo-atlas/VERSION @@ -1 +1 @@ -1.0.6 +1.0.7 diff --git a/input-plugin/logstash-input-mongo-atlas/build.gradle b/input-plugin/logstash-input-mongo-atlas/build.gradle index a4856a199..16fe2bb83 100644 --- a/input-plugin/logstash-input-mongo-atlas/build.gradle +++ b/input-plugin/logstash-input-mongo-atlas/build.gradle @@ -2,6 +2,29 @@ import java.nio.file.Files import static java.nio.file.StandardCopyOption.REPLACE_EXISTING apply plugin: 'java' + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + + dependencies { + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' + classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' + } +} + +def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" // =========================================================================== @@ -20,37 +43,25 @@ pluginInfo.pluginClass = "MongoAtlasInput" pluginInfo.pluginName = "mongo_atlas_input" // must match the @LogstashPlugin annotation in the main plugin class // =========================================================================== -sourceCompatibility = 1.8 -targetCompatibility = 1.8 +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 -buildscript { - buildscript { - repositories { - maven { - url "https://plugins.gradle.org/m2/" - } - mavenCentral() - jcenter() - } - dependencies { - classpath 'com.github.jengelman.gradle.plugins:shadow:4.0.4' - classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' +repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" } } } -def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); -def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) - -repositories { - mavenCentral() -} - apply plugin: 'com.github.johnrengelman.shadow' shadowJar { - classifier = null + archiveClassifier = null } dependencies { @@ -59,6 +70,7 @@ dependencies { implementation 'org.apache.httpcomponents:httpclient:4.5.13' implementation 'org.apache.commons:commons-lang3:' + versions.dependencies.commonsLang implementation 'com.google.code.gson:gson:' + versions.dependencies.gson + implementation group: 'commons-beanutils', name: 'commons-beanutils', version: versions.dependencies.commonsBeanutils implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "build/libs/logstash-core.jar") implementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "guardium-universalconnector-commons-*.*.*.jar") @@ -90,7 +102,6 @@ tasks.register("vendor"){ File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") projectJarFile.mkdirs() Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) - validatePluginJar(projectJarFile, project.group) } } diff --git a/input-plugin/logstash-input-mongo-atlas/gradle/wrapper/gradle-wrapper.properties b/input-plugin/logstash-input-mongo-atlas/gradle/wrapper/gradle-wrapper.properties index aa991fcea..81aa1c044 100644 --- a/input-plugin/logstash-input-mongo-atlas/gradle/wrapper/gradle-wrapper.properties +++ b/input-plugin/logstash-input-mongo-atlas/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/input-plugin/logstash-input-mongo-atlas/input-mongo-atlas.conf b/input-plugin/logstash-input-mongo-atlas/input-mongo-atlas.conf index 381812962..60cf23b5c 100644 --- a/input-plugin/logstash-input-mongo-atlas/input-mongo-atlas.conf +++ b/input-plugin/logstash-input-mongo-atlas/input-mongo-atlas.conf @@ -5,6 +5,9 @@ input { private-key => "" group-id => "" hostname => "" + # For MongoDB Atlas deployments on custom domains or private cloud installations (other than cloud.mongodb.com), + # uncomment the following line and specify your MongoDB Atlas API base URL. + # mongo-api-url => "" # Example: https://cloud.mongodb.com/api/atlas/v1.0/groups/ type => "mongodbatlas" } } diff --git a/input-plugin/logstash-input-mongo-atlas/src/main/java/com/ibm/guardium/mongodb/MongoApi.java b/input-plugin/logstash-input-mongo-atlas/src/main/java/com/ibm/guardium/mongodb/MongoApi.java index 24631b309..6da597ad6 100644 --- a/input-plugin/logstash-input-mongo-atlas/src/main/java/com/ibm/guardium/mongodb/MongoApi.java +++ b/input-plugin/logstash-input-mongo-atlas/src/main/java/com/ibm/guardium/mongodb/MongoApi.java @@ -80,16 +80,23 @@ private static CloseableHttpClient buildHttpClient(URL url){ * @param privateKey * @param groupId * @param hostname + * @param mongoApiUrl * @param fileName * @param startDateInEpoch * @param endDateInEpoch * @return */ - public static String getResponseFromJsonURL(String publicKey, String privateKey, String groupId, String hostname, String fileName, long startDateInEpoch, long endDateInEpoch) { + public static String getResponseFromJsonURL(String publicKey, String privateKey, String groupId, String mongoApiUrl, String hostname, String fileName, long startDateInEpoch, long endDateInEpoch) { + // Use default API URL if mongoApiUrl is null or empty + if (mongoApiUrl == null || mongoApiUrl.isEmpty()) { + log.info("MongoDB API URL is not set, using default: {}", MONGO_API_URL); + mongoApiUrl = MONGO_API_URL; + } + //build url - String url = MONGO_API_URL + groupId + "/clusters/" + hostname + "/logs/"+fileName+"?startDate=" + startDateInEpoch + "&endDate=" + endDateInEpoch; + String url = mongoApiUrl + groupId + "/clusters/" + hostname + "/logs/"+fileName+"?startDate=" + startDateInEpoch + "&endDate=" + endDateInEpoch; if (log.isDebugEnabled()) { - log.debug(url); + log.debug("Constructed MongoDB Atlas API URL: {}", url); } try { diff --git a/input-plugin/logstash-input-mongo-atlas/src/main/java/org/logstashplugins/MongoAtlasInput.java b/input-plugin/logstash-input-mongo-atlas/src/main/java/org/logstashplugins/MongoAtlasInput.java index 17fe81951..28a102618 100644 --- a/input-plugin/logstash-input-mongo-atlas/src/main/java/org/logstashplugins/MongoAtlasInput.java +++ b/input-plugin/logstash-input-mongo-atlas/src/main/java/org/logstashplugins/MongoAtlasInput.java @@ -52,6 +52,8 @@ public class MongoAtlasInput implements Input { PluginConfigSpec.stringSetting("group-id", "message"); public static final PluginConfigSpec HOSTNAME_CONFIG = PluginConfigSpec.stringSetting("hostname", "message"); + public static final PluginConfigSpec MONGO_API_URL_CONFIG = + PluginConfigSpec.stringSetting("mongo-api-url", "https://cloud.mongodb.com/api/atlas/v1.0/groups/"); public static final PluginConfigSpec FILE_NAME_CONFIG = PluginConfigSpec.stringSetting("filename", "mongodb-audit-log.gz"); private String id; @@ -60,6 +62,7 @@ public class MongoAtlasInput implements Input { private String publicKey; private String groupId; private String hostname; + private String mongoApiUrl; private String fileName; private String pluginType; private final CountDownLatch done = new CountDownLatch(1); @@ -74,6 +77,7 @@ public MongoAtlasInput(String id, Configuration config, Context context) { publicKey = config.get(PUBLIC_KEY_CONFIG); groupId = config.get(GROUP_ID_CONFIG); hostname = config.get(HOSTNAME_CONFIG); + mongoApiUrl = config.get(MONGO_API_URL_CONFIG); fileName = config.get(FILE_NAME_CONFIG); pluginType = config.get(TYPE_CONFIG); } @@ -95,7 +99,7 @@ public void start(Consumer> consumer) { try { while (!stopped) { loopStartTime = System.currentTimeMillis() / 1000L; - String allText = MongoApi.getResponseFromJsonURL(this.publicKey, this.privateKey,this.groupId,this.hostname,this.fileName, lasttime, loopStartTime); + String allText = MongoApi.getResponseFromJsonURL(this.publicKey, this.privateKey, this.groupId, this.mongoApiUrl, this.hostname,this.fileName, lasttime, loopStartTime); if (allText != null) { lasttime = loopStartTime; String lines[] = allText.split("\\r?\\n"); @@ -103,6 +107,7 @@ public void start(Consumer> consumer) { ) { HashMap map = new HashMap(); map.put("message", line); + map.put("mongoApiUrl", mongoApiUrl); map.put("hostname", hostname); map.put("groupId", groupId); map.put("type", pluginType); @@ -142,7 +147,7 @@ public void awaitStop() throws InterruptedException { @Override public Collection> configSchema() { // should return a list of all configuration options for this plugin - return Arrays.asList(INTERVAL_CONFIG,PUBLIC_KEY_CONFIG,PRIVATE_KEY_CONFIG,GROUP_ID_CONFIG,HOSTNAME_CONFIG,TYPE_CONFIG); + return Arrays.asList(INTERVAL_CONFIG,PUBLIC_KEY_CONFIG,PRIVATE_KEY_CONFIG,GROUP_ID_CONFIG,MONGO_API_URL_CONFIG,HOSTNAME_CONFIG,TYPE_CONFIG); } @Override diff --git a/input-plugin/logstash-input-mongo-atlas/src/test/java/org/logstashplugins/JavaInputExampleTest.java b/input-plugin/logstash-input-mongo-atlas/src/test/java/org/logstashplugins/JavaInputExampleTest.java index d97038d29..a812c9a82 100644 --- a/input-plugin/logstash-input-mongo-atlas/src/test/java/org/logstashplugins/JavaInputExampleTest.java +++ b/input-plugin/logstash-input-mongo-atlas/src/test/java/org/logstashplugins/JavaInputExampleTest.java @@ -19,6 +19,7 @@ public void testActualPullingFromMongoAtlas() throws InterruptedException { configValues.put(MongoAtlasInput.GROUP_ID_CONFIG.name(), "123456789abcefg"); configValues.put(MongoAtlasInput.HOSTNAME_CONFIG.name(), "cluster1-shard-12-34.i1234.mongodb.net"); configValues.put(MongoAtlasInput.TYPE_CONFIG.name(), "mongodbatlas"); + configValues.put(MongoAtlasInput.MONGO_API_URL_CONFIG.name(), "https://cloud.mongodb.com/api/atlas/v1.0/groups/"); Configuration config = new ConfigurationImpl(configValues); ByteArrayOutputStream baos = new ByteArrayOutputStream(); diff --git a/input-plugin/logstash-input-mongo-atlas/src/test/java/org/logstashplugins/JavaInputExampleTestLocalProxy.java b/input-plugin/logstash-input-mongo-atlas/src/test/java/org/logstashplugins/JavaInputExampleTestLocalProxy.java index d8e4fa6e9..5a87736de 100644 --- a/input-plugin/logstash-input-mongo-atlas/src/test/java/org/logstashplugins/JavaInputExampleTestLocalProxy.java +++ b/input-plugin/logstash-input-mongo-atlas/src/test/java/org/logstashplugins/JavaInputExampleTestLocalProxy.java @@ -28,6 +28,7 @@ public void testAtlasInputUsingProxy(){ configValues.put(MongoAtlasInput.GROUP_ID_CONFIG.name(), "123456789abcefg"); configValues.put(MongoAtlasInput.HOSTNAME_CONFIG.name(), "cluster1-shard-12-34.i1234.mongodb.net"); configValues.put(MongoAtlasInput.TYPE_CONFIG.name(), "mongodbatlas"); + configValues.put(MongoAtlasInput.MONGO_API_URL_CONFIG.name(), "https://cloud.mongodb.com/api/atlas/v1.0/groups/"); Configuration config = new ConfigurationImpl(configValues); MongoAtlasInput inputSample = new MongoAtlasInput("test-id", config, null); diff --git a/input-plugin/logstash-input-s3sqs/CHANGELOG.md b/input-plugin/logstash-input-s3sqs/CHANGELOG.md new file mode 100644 index 000000000..935534a88 --- /dev/null +++ b/input-plugin/logstash-input-s3sqs/CHANGELOG.md @@ -0,0 +1,9 @@ +## 1.0.2 +- Fixing a bug that caused the plugin to fail when reinstall the UC. + +## 1.0.1 +- Enabling support for CSV log file format. + +## 1.0.0 +- Initial version for experimental v0 of native support for Java plugins. + diff --git a/input-plugin/logstash-input-s3sqs/InputS3SQSPackage/S3SQS/input.conf b/input-plugin/logstash-input-s3sqs/InputS3SQSPackage/S3SQS/input.conf new file mode 100644 index 000000000..1687a0ed4 --- /dev/null +++ b/input-plugin/logstash-input-s3sqs/InputS3SQSPackage/S3SQS/input.conf @@ -0,0 +1,14 @@ +input { + s3_sqs { + queue_url => "" + region => "" + access_key_id => "" + secret_access_key => "" + role_arn => "" # Leave empty if not using role-based access + max_messages => + wait_time => # Must be >= 0 and <= 20, + polling_frequency => + type => "" + add_field => { "account_id" => "" } + } +} diff --git a/input-plugin/logstash-input-s3sqs/InputS3SQSPackage/S3SQS/logstash-input-s3_sqs.zip b/input-plugin/logstash-input-s3sqs/InputS3SQSPackage/S3SQS/logstash-input-s3_sqs.zip new file mode 100644 index 000000000..ae42aa543 Binary files /dev/null and b/input-plugin/logstash-input-s3sqs/InputS3SQSPackage/S3SQS/logstash-input-s3_sqs.zip differ diff --git a/input-plugin/logstash-input-http/httpInputPackage/http/manifest.json b/input-plugin/logstash-input-s3sqs/InputS3SQSPackage/S3SQS/manifest.json similarity index 67% rename from input-plugin/logstash-input-http/httpInputPackage/http/manifest.json rename to input-plugin/logstash-input-s3sqs/InputS3SQSPackage/S3SQS/manifest.json index cf03c9166..bf6c08261 100644 --- a/input-plugin/logstash-input-http/httpInputPackage/http/manifest.json +++ b/input-plugin/logstash-input-s3sqs/InputS3SQSPackage/S3SQS/manifest.json @@ -1,15 +1,15 @@ { - "name": "http_input", - "alias": "http", + "name": "s3_sqs_input", + "alias": "s3_sqs", "type": "input", "pipeline_type": null, - "plugin_version": "4.1.2", + "plugin_version": "1.0.0", "supportedDataSources": null, "supportedInputPlugins": null, "developer": "Elastic", "license": "Apache2.0", "supported_input_plugins": [], - "description": "Listens for events in the Elastic http format. Incoming events are passed on to the filter stage.", + "description": "Listens for events in S3SQS. Incoming events are passed on to the filter stage.", "configuration_notes": "", "documentation_path": "", "pluginSHA256Checksum": -1, @@ -18,5 +18,5 @@ "pluginConfigTemplateStrings": null, "uploadDate": null, "uploadUser": null, - "auto_scaling": true -} \ No newline at end of file + "auto_scaling": false +} diff --git a/input-plugin/logstash-input-s3sqs/InputS3SQSPackage/gi_templates.json b/input-plugin/logstash-input-s3sqs/InputS3SQSPackage/gi_templates.json new file mode 100644 index 000000000..dbb5f5101 --- /dev/null +++ b/input-plugin/logstash-input-s3sqs/InputS3SQSPackage/gi_templates.json @@ -0,0 +1,11 @@ +{ + "plugin_name": "S3 SQS Input", + "help_link": "", + "input_name": "s3_sqs", + "input_parameters": [ + ], + "filter_name": "", + "filter_parameters": [], + "auth_parameters": [ + ] +} \ No newline at end of file diff --git a/input-plugin/logstash-input-s3sqs/LICENSE b/input-plugin/logstash-input-s3sqs/LICENSE new file mode 100644 index 000000000..a80a3fd53 --- /dev/null +++ b/input-plugin/logstash-input-s3sqs/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2020 Elastic and contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/input-plugin/logstash-input-s3sqs/README.md b/input-plugin/logstash-input-s3sqs/README.md new file mode 100644 index 000000000..62b421d62 --- /dev/null +++ b/input-plugin/logstash-input-s3sqs/README.md @@ -0,0 +1,65 @@ +## s3sqs input plug-in +### Meet s3sqs +* Tested versions: 1.0.0 +* Developed by [IBM](https://github.ibm.com/Activity-Insights/universal-connectors/tree/master/input-plugin/logstash-input-s3sqs) +* Supported Guardium versions: + * Guardium Data Protection: 11.4 and above + +This is a [Logstash](https://github.com/elastic/logstash) input plug-in for the universal connector that is featured in IBM Security Guardium. The more details of this plugin can be found [here](./README.md). It retrieves messages from Amazon SQS, each containing the detailed path to an audit log file stored in an S3 bucket. The file is then read and transformed into events. The events are then sent over to corresponding filter plugin which transforms these audit logs into a [Guardium record](https://github.com/IBM/universal-connectors/blob/main/common/src/main/java/com/ibm/guardium/universalconnector/commons/structures/Record.java) instance (which is a standard structure made out of several parts). The information is then sent over to Guardium. Guardium records include the accessor (the person who tried to access the data), the session, data, and exceptions. If there are no errors, the data contains details about the query "construct". The construct details the main action (verb) and collections (objects) involved. + +## Purpose: + +Specify the Queue url from where the file name in S3 is to be read. + +## Usage: + +### Parameters: +| Parameter | Input Type | Required | Default | +|-------------------|------------|----------|--------| +| queue_url | string | Yes | | +| region | string | Yes | | +| max_messages | Integer | Yes | | +| wait_time | Integer | Yes | | +| polling_frequency | Integer | Yes | | +| access_key_id | string | No | | +| secret_access_key | string | No | | +| role_arn | string | No | | +| type | string | No | | +| account_id | string | No | | + +#### `queue_url` +The `queue_url` setting specifies the SQS queue URL created to receive notifications from S3. + +#### `region` +The `region` setting allows specify the region in which the Amazon SQS Queue exists. + +#### `max_messages` +The `max_messages` setting defines the maximum number of messages to retrieve in a single SQS call. + +#### `wait_time` +The `wait_time` specifies the duration (in seconds) the call will wait for a message to appear in the queue before returning. If a message is available sooner, the call returns immediately. Ensure that `wait_time` is always less than the `polling_frequency`. + +#### `polling_frequency` +The `polling_frequency` setting specifies the delay between consecutive SQS calls. + +#### `access_key_id` +The `access_key_id` setting allows specifying the access key id of the IAM user having cloudwatch access to read the logs. + +#### `secret_access_key` +The `secret_access_key` setting allows specifying the access secret key id of the IAM user having cloudwatch access to read the logs. + +#### `role_arn` +The role_arn setting allows you to specify which AWS IAM Role to assume, if any. This is used to generate temporary credentials, typically for cross-account access. Click [here](S3SQS_RoleARN.md) to configure the RoleARN. + +#### `type` +The type you can use to specify the plugin type in the input section. + +#### `account_id` +You can use the account_id to specify the AWS account ID + +## Note: +For detailed instructions on collecting RDS audit logs with Amazon Kinesis Data Firehose, please refer "[S3SQSWithFireHose](S3SQSWithFirehose.md)". + +## Limitations: +Duplicate Event Delivery: Amazon S3 does not support sending event notifications to FIFO SQS queues. Only Standard queues are compatible with S3 event notifications. +The plugin relies on Amazon SQS Standard queues, which guarantee at-least-once delivery. As a result, the same S3 event may be delivered multiple times, leading to duplicate processing of S3 objects. \ No newline at end of file diff --git a/input-plugin/logstash-input-s3sqs/S3SQSWithFirehose.md b/input-plugin/logstash-input-s3sqs/S3SQSWithFirehose.md new file mode 100644 index 000000000..59e2350c2 --- /dev/null +++ b/input-plugin/logstash-input-s3sqs/S3SQSWithFirehose.md @@ -0,0 +1,299 @@ +# Export log to S3 bucket with help of Amazon Kinesis Data Firehose delivery stream + +## Overview + +This guide details how to collect audit logs from AWS data sources (such as RDS or other supported services), stream them through Amazon CloudWatch Logs and Amazon Kinesis Data Firehose, deliver them to Amazon S3, and notify Amazon SQS when new files are created. This enables downstream analysis by IBM Security Guardium. + +--- + +## Prerequisites + +- AWS account with required permissions +- AWS service emitting logs (e.g., RDS, Aurora, Dynamo, etc.) +- Audit logging enabled for the source service +- Amazon CloudWatch Logs enabled +- Amazon Kinesis Data Firehose delivery stream configured +- Amazon S3 bucket ready to store logs +- Amazon SQS queue for notifications + +--- + +## Configuration Steps + +### 1. Enable Audit Logging + +1. Open the console for the source service (e.g., Amazon RDS Console). +2. Choose your resource (e.g., database or instance). +3. Locate **Configuration** and click the linked **Parameter Group**. +4. Click **Edit parameters** and adjust logging/audit parameters according to your service's documentation. +5. Save changes. +6. Reboot or restart the service if required. + +--- + +### 2. Export Logs to CloudWatch + +1. In the source service's console (e.g., RDS), click **Modify**. +2. Scroll to **Log exports**. +3. Enable relevant log types (e.g., `audit`, `general`, `error`, `slowquery`). +4. Click **Continue** → **Apply immediately**. +5. Wait for the modification to complete. + +--- + +### 3. Create IAM Role: CloudWatch Logs to Firehose + +#### 3.1 Create Role + +1. Go to **IAM Console** → **Roles** → **Create role**. +2. Choose `AWS service` as trusted entity. +3. Select **CloudWatch Logs**. +4. Click **Next** (no permissions yet). + +#### 3.2 Role Details + +- Name the role: `` (e.g., `CloudWatchToFirehoseRole`) + +#### 3.3 Add Inline Policy + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "firehose:PutRecord", + "firehose:PutRecordBatch" + ], + "Resource": "arn:aws:firehose:::deliverystream/" + } + ] +} +``` + +- Name the policy `AllowFirehosePutRecord` + +--- + +## Configure Firehose to Deliver Logs to Amazon S3 + +### 1. Create IAM Role for Firehose Access to S3 + +#### Add Policy + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:PutObjectAcl", + "s3:GetBucketLocation", + "s3:ListBucket", + "logs:PutLogEvents", + "logs:CreateLogStream", + "logs:CreateLogGroup" + ], + "Resource": [ + "arn:aws:s3:::", + "arn:aws:s3:::/*", + "arn:aws:logs:::log-group:/aws/kinesisfirehose/*" + ] + } + ] +} +``` + +- Role name: `` + +### 2. Create Kinesis Firehose Delivery Stream + +1. Open **Kinesis → Delivery Streams** → **Create delivery stream**. +2. Source: `Direct PUT or other sources` +3. Destination: `Amazon S3` + - New line delimiter: `Enabled` + - Bucket: `` + - Prefix: `logs/!{timestamp:yyyy/MM/dd}/` + - Error prefix (optional): `errors/!{firehose:error-output-type}/!{timestamp:yyyy/MM/dd}/` +4. Buffering: + - Size: `128 MB` + - Interval: `60 seconds` + - Compression: `GZIP` + - It is recommended to consult with your DBA to adjust the buffering size and interval based on expected log traffic volume and desired file size in S3. +5. IAM Role: Use `` +6. Name the delivery stream: `` +7. Click **Create delivery stream** +8. Edit Delivery Stream Go To Transform and convert records and click on edit +9. Mark Decompress source records from Amazon CloudWatch Logs as `on` + +--- +### Creating the SQS queue +The SQS queue created in these steps will receive messages from the Event Notification (configured in the next section). +These messages, generated by monitoring the S3 bucket, will contain details of the recently added S3 log files. + + +#### Procedure +1. Go to https://console.aws.amazon.com/ +2. Click **Services** +3. Search for SQS and click on **Simple Queue Services** +4. Click **Create Queue**. +5. Select the type as **Standard**. +6. Enter the name for the queue +7. Keep the rest of the default settings + +### Creating a policy for the relevant IAM User +Perform the following steps for the IAM user who is accessing the SQS logs in Guardium: + +#### Procedure +1. Go to https://console.aws.amazon.com/ +2. Go to **IAM service** > **Policies** > **Create Policy**. +3. Select **service as SQS**. +4. Check the following checkboxes: + * **ListQueues** + * **DeleteMessage** + * **DeleteMessageBatch** + * **GetQueueAttributes** + * **GetQueueUrl** + * **ReceiveMessage** + * **ChangeMessageVisibility** + * **ChangeMessageVisibilityBatch** +5. In the resources, specify the ARN of the queue created in the above step. +6. Click **Review policy** and specify the policy name. +7. Click **Create policy**. +8. Assign the policy to the user + 1. Log in to the IAM console as an IAM user (https://console.aws.amazon.com/iam/). + 2. Go to **Users** on the console and select the relevant IAM user to whom you want to give permissions. + Click the **username**. + 3. In the **Permissions tab**, click **Add permissions**. + 4. Click **Attach existing policies directly**. + 5. Search for the policy created and check the checkbox next to it. + 6. Click **Next: Review** + 7. Click **Add permissions** + +### Creating the Event Notification +The Event Notification will get triggered when a new Object is added to S3 bucket and will send the events to the SQS queue. +Follow the steps below to configure the Event Notification + +#### Creating Access Policy to allow Notifications +Update the Access Policy of the SQS queue to allow the Notification Service to send messages to the Queue + +__*Procedure*__ +1. Go to https://console.aws.amazon.com/ +2. Go to **SQS** -> **Queues** +3. Click on the Queue that was created in the above step +4. Go to **Access Policy** +5. Click on **Edit** +6. Add the below details to the existing policy + +``` +{ + "Sid": "example-statement-ID", + "Effect": "Allow", + "Principal": { + "Service": "s3.amazonaws.com" + }, + "Action": "SQS:SendMessage", + "Resource": "", + "Condition": { + "StringEquals": { + "aws:SourceAccount": "" + }, + "ArnLike": { + "aws:SourceArn": "" + } + } +} +``` + + +7. Click on **Save** + + +#### Create the Event Notification +__*Procedure*__ +1. Go to https://console.aws.amazon.com/ +2. Go to **Services**. Search for **S3**. +3. Click on the S3 bucket that is associated with the CloudTrail. +4. Click **Properties** +5. Navigate to **Event Notifications** +6. Click on **Create event notification**. +7. Enter **Event name** +8. Enter the **Prefix** though this is optional, this can be set to capture the specific traffic. +9. In **Event Types** Select **All object create events**. +10. In **Destination** Select **SQS queue**. +11. In **Specify SQS Queue** either **Choose from your SQS queues** option select the Queue name from drop down list or **Enter SQS queue ARN** enter the Queue ARN manually. +12. Click on **Save Changes** + +## Create CloudWatch Log Subscription Filter + +1. Go to **CloudWatch Console** → **Logs** → **Log groups** +2. Find the log group related to your service, such as: + +``` +/aws//instance//audit +``` + +3. Click the log group +4. Click **Actions** → **Create subscription filter** + +| Field | Value | +| ------------------- |--------------------------------------------------------------| +| **Filter name** | `SubscriptionLogstoS3` | +| **Filter pattern** | *(Leave blank or add specific pattern to filter log events)* | +| **Destination** | `Kinesis Firehose` | +| **Delivery stream** | `` | +| **IAM role** | `` | + +5. Click **Start streaming** + +--- + +## Enable S3 Event Notification to SQS + +### 1. Create SQS Queue + +1. Go to **SQS Console** → **Create queue** +2. Choose queue type (Standard recommended) +3. Name the queue: `` +4. Note the ARN of the queue + +### 2. Update S3 Bucket Notification Configuration + +1. Go to **S3 Console** → Select `` +2. Navigate to **Properties** → **Event notifications** +3. Click **Create event notification** + - Name: `NotifySQSonNewFile` + - Event type: `PUT` (object created) + - Prefix: `logs/` + - Destination: SQS Queue → `` + +### 3. Update S3 Bucket Policy + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": {"Service": "s3.amazonaws.com"}, + "Action": "sqs:SendMessage", + "Resource": "arn:aws:sqs:::", + "Condition": { + "ArnLike": { + "aws:SourceArn": "arn:aws:s3:::" + } + } + } + ] +} +``` + +--- + +## Validate Delivery + +1. Go to the S3 bucket and confirm log file delivery under the expected prefix +2. Check the SQS queue for new messages when a new file lands in S3 \ No newline at end of file diff --git a/input-plugin/logstash-input-s3sqs/S3SQS_RoleARN.md b/input-plugin/logstash-input-s3sqs/S3SQS_RoleARN.md new file mode 100644 index 000000000..71698059e --- /dev/null +++ b/input-plugin/logstash-input-s3sqs/S3SQS_RoleARN.md @@ -0,0 +1,388 @@ +# AWS IAM Role Configuration Guide + +This guide outlines how to configure an **IAM Role** with necessary **permission policies**, **trust relationships**, and **cross-account access** using the AWS Console. The role is intended to allow EC2 instances to access **Amazon SQS** and **Amazon S3** securely with least privilege. + +--- + +## Purpose + +Grant an EC2 instance the ability to: + +- Read from specific **Amazon SQS queues** +- Access **CloudWatch logs** stored in **Amazon S3** +- Support cross-account usage (optional) + +--- + +## Prerequisites + +- AWS account credentials with IAM privileges +- An existing **SQS queue** +- An existing **S3 bucket** with CloudWatch logs +- An EC2 instance to assume the role + +--- + +## Instructions + +### 1. Create IAM Role for EC2 + +1. Go to **IAM > Roles** in the [AWS Console](https://console.aws.amazon.com/iam/). +2. Click **Create role**. +3. **Select Trusted Entity Type**: Choose **AWS service**. +4. Select **EC2** as the use case. +5. Click **Next**. + +--- + +### 2. Attach Policies + +#### For Single Account Setup (EC2, SQS, and S3 in the same AWS account) + +Click **Create policy** (in new tab), and create the following: + +##### a. Amazon SQS Read-Only Access Policy + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AmazonSQSReadOnlyAccess", + "Effect": "Allow", + "Action": [ + "sqs:GetQueueAttributes", + "sqs:GetQueueUrl", + "sqs:ListDeadLetterSourceQueues", + "sqs:ListQueues", + "sqs:ListMessageMoveTasks", + "sqs:ListQueueTags" + ], + "Resource": "*" + } + ] +} +``` + +> Name this policy: `AmazonSQSReadOnlyCustom` + +--- + +##### b. S3 Read-Only Access to CloudWatch Logs + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:GetObject" + ], + "Resource": [ + "arn:aws:s3:::", + "arn:aws:s3:::/*" + ] + }, + { + "Effect": "Allow", + "Action": [ + "s3:ListBucket" + ], + "Resource": "arn:aws:s3:::" + } + ] +} +``` + +> Replace `` with your actual CloudWatch log bucket name in your current AWS account. + +> Name this policy: `S3CloudWatchReadAccess` + +--- + +##### c. SQS Full Access for Specific Queue + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "sqs:ReceiveMessage", + "sqs:DeleteMessage", + "sqs:GetQueueAttributes" + ], + "Resource": "arn:aws:sqs:::" + } + ] +} +``` + +> Replace: +> - `` with your AWS region (e.g., us-east-1) +> - `` with your current AWS account ID +> - `` with your SQS queue name in your current AWS account + +> Name this policy: `SQSQueueAccess` + +--- + +##### d. Assume Role Permission (for same account) + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "VisualEditor0", + "Effect": "Allow", + "Action": "sts:AssumeRole", + "Resource": [ + "arn:aws:iam:::role//*", + "arn:aws:iam:::role/", + "arn:aws:sts:::assumed-role//*", + "arn:aws:sts:::assumed-role//" + ] + } + ] +} +``` + +> Replace: +> - `` with your current AWS account ID +> - `` with the name of the role you're creating in your current AWS account +> - `` with your EC2 instance ID in your current AWS account + +> Name this policy: `AssumeRolePermission` + +--- + +#### For Cross Account Setup (EC2 in one account, SQS/S3 in another) + +### Step-by-Step Cross-Account Configuration + +For this setup, we'll use the following terminology: +- **Account1**: The AWS account where your EC2 instance is hosted (e.g., Account ID: 111111) +- **Account2**: The AWS account where your SQS queue and S3 bucket are located (e.g., Account ID: 222222) + +#### A. In Account1 (where EC2 instance is located) + +1. Log in to your IAM console (https://console.aws.amazon.com/iam/) of Account1 where EC2 is hosted (e.g., Account ID: 111111) +2. Click the **Roles** tab under **Access Management** +3. Click the **Create Role** button +4. For **Trusted Entity Type**, select AWS Service +5. For **Use case**, select EC2 +6. Click **Next** +7. Click **Create policy** (in new tab) to create the following policy: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "VisualEditor0", + "Effect": "Allow", + "Action": "sts:AssumeRole", + "Resource": "arn:aws:iam:::role/" + } + ] +} +``` + +> Replace: +> - `` with the account ID where your SQS/S3 resources are located (e.g., 222222) +> - `` with the name of the role you'll create in Account2 (e.g., role_on_222222) + +8. Name this policy (e.g., `CrossAccountAssumeRolePolicy`) and click **Create policy** +9. Return to the role creation tab, refresh the policy list, and attach the newly created policy +10. Click **Next** +11. Name the role (e.g., `role_on_111111`) and provide a description +12. Click **Create role** +13. In the IAM Role's **Trust relationships** tab, click **Edit trust policy** +14. Ensure the trust policy looks like this: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Service": "ec2.amazonaws.com" + }, + "Action": "sts:AssumeRole" + } + ] +} +``` + +15. Click **Update Trust Policy** +16. Attach this role to your EC2 instance: + 1. Go to **EC2 > Instances** + 2. Select your instance > **Actions > Security > Modify IAM role** + 3. Choose the role you just created (e.g., `role_on_111111`) and click **Update IAM role** + +#### B. In Account2 (where SQS/S3 resources are located) + +1. Log in to your IAM console (https://console.aws.amazon.com/iam/) of Account2 where SQS/S3 resources are located (e.g., Account ID: 222222) +2. Click the **Roles** tab under **Access Management** +3. Click the **Create Role** button +4. For **Trusted Entity Type**, select **Another AWS account** +5. Enter the **Account ID of Account1** (e.g., 111111) +6. Click **Next** +7. Enter the role name (e.g., `role_on_222222`) +8. Click **Create Role** +9. Search for the created role (e.g., `role_on_222222`) and open it +10. Steps to set the Permissions Policies: + 1. In the **Permissions** tab, click the **Add Permissions** button and select **Create Inline Policy** + 2. On the **Create Policy** page, select JSON editor and add the following SQS policy: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "sqs:ReceiveMessage", + "sqs:DeleteMessage", + "sqs:DeleteMessageBatch", + "sqs:ChangeMessageVisibility", + "sqs:ChangeMessageVisibilityBatch", + "sqs:GetQueueAttributes", + "sqs:GetQueueUrl" + ], + "Resource": "arn:aws:sqs:::" + } + ] +} +``` + +11. Click **Review Policy** +12. Enter the policy name (e.g., `SQSAccessPolicy`) and click **Create Policy** +13. In the **Permissions** tab, click the **Add Permissions** button and select **Create Inline Policy** +14. On the **Create Policy** page, select JSON editor and add the following S3 policy: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:GetObject" + ], + "Resource": [ + "arn:aws:s3:::/AWSLogs//*" + ] + }, + { + "Effect": "Allow", + "Action": [ + "s3:ListBucket" + ], + "Resource": "arn:aws:s3:::", + "Condition": { + "StringLike": { "s3:prefix": ["AWSLogs//*"] } + } + } + ] +} +``` + +> Replace `` with your actual CloudWatch log bucket name in Account2. + +15. Click **Review Policy** +16. Enter the policy name (e.g., `S3AccessPolicy`) and click **Create Policy** +17. In the **Trust relationships** tab, click **Edit trust policy** +18. Replace the trust policy with: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam:::role/" + }, + "Action": "sts:AssumeRole" + } + ] +} +``` + +19. Click **Update Trust Policy** + +--- + +#### C. Configure Role ARN in Logstash Input Plugin + +In your Logstash configuration file, set the `role_arn` parameter to the ARN of the role in Account2: + +``` +input { + s3sqs { + # Other parameters... + role_arn => "arn:aws:iam:::role/" # e.g., "arn:aws:iam::222222:role/role_on_222222" + # Additional parameters... + } +} +``` + +> Replace `` and `` with your actual values. + +--- + +### 3. Modify the Trust Relationship + +#### For Single Account Setup + +1. In the IAM Role's **Trust relationships** tab, click **Edit trust relationship**. +2. Replace the trust policy with: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Service": "ec2.amazonaws.com" + }, + "Action": "sts:AssumeRole" + }, + { + "Sid": "AllowSelfAssume", + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam:::role/" + }, + "Action": "sts:AssumeRole" + } + ] +} +``` + +> Replace: +> - `` with your current AWS account ID +> - `` with the name of the role you're creating in your current AWS account + +--- + +## Attach Role to EC2 Instance + +1. Go to **EC2 > Instances** +2. Select your instance > **Actions > Security > Modify IAM role** +3. Choose the role `` and click **Update IAM role** + +--- + +## References + +- [AWS IAM User Guide: Tutorial - Delegate Access Across AWS Accounts Using IAM Roles](https://docs.aws.amazon.com/IAM/latest/UserGuide/tutorial_cross-account-with-roles.html) +- [AWS SQS Developer Guide: Cross-Account Permissions](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-customer-managed-policy-examples.html#grant-cross-account-permissions-to-role-and-user-name) +- [AWS S3 User Guide: Cross-Account Access](https://docs.aws.amazon.com/AmazonS3/latest/userguide/example-walkthroughs-managing-access-example2.html) +- [AWS STS Reference: AssumeRole](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html) diff --git a/input-plugin/logstash-input-s3sqs/VERSION b/input-plugin/logstash-input-s3sqs/VERSION new file mode 100644 index 000000000..e6d5cb833 --- /dev/null +++ b/input-plugin/logstash-input-s3sqs/VERSION @@ -0,0 +1 @@ +1.0.2 \ No newline at end of file diff --git a/input-plugin/logstash-input-s3sqs/build.gradle b/input-plugin/logstash-input-s3sqs/build.gradle new file mode 100644 index 000000000..969e0dd32 --- /dev/null +++ b/input-plugin/logstash-input-s3sqs/build.gradle @@ -0,0 +1,152 @@ +import java.nio.file.Files +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING + +apply plugin: 'java' + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + + dependencies { + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' + classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' + } +} + +def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + +apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" + +// =========================================================================== +// plugin info +// =========================================================================== +group 'org.logstashplugins' // must match the package of the main plugin class +version "${file("VERSION").text.trim()}" // read from required VERSION file +description = "S3SQS-Guardium input plugin" +pluginInfo.licenses = ['Apache-2.0'] // list of SPDX license IDs +pluginInfo.longDescription = "This gem is a Logstash S3SQS filter plugin required to be installed as part of IBM Security Guardium, Guardium Universal connector configuration. This gem is not a stand-alone program." +pluginInfo.authors = ['IBM'] +pluginInfo.email = [''] +pluginInfo.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html" +pluginInfo.pluginType = "input" +pluginInfo.pluginClass = "S3SQS" +pluginInfo.pluginName = "s3_sqs" // must match the @LogstashPlugin annotation in the main plugin class +// =========================================================================== + +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 + + +repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } +} + +apply plugin: 'com.github.johnrengelman.shadow' + +shadowJar { + archiveClassifier = null + zip64 true +} + +dependencies { + + implementation group: 'commons-io', name: 'commons-io', version: '2.18.0' + + implementation group: 'software.amazon.awssdk', name: 's3', version: '2.30.17' + + implementation group: 'software.amazon.awssdk', name: 'sqs', version: '2.30.17' + + implementation group: 'software.amazon.awssdk', name: 'sts', version: '2.30.19' + + implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.17.0' + + implementation group: 'org.json', name: 'json', version: '20250107' + + implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.17.1' + + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.15.2' + + implementation("org.apache.commons:commons-csv:1.14.0") + implementation group: 'commons-beanutils', name: 'commons-beanutils', version: versions.dependencies.commonsBeanutils + + implementation group: 'io.netty', name: 'netty-codec-http2', version: '4.2.11.Final' + implementation group: 'io.netty', name: 'netty-codec-http', version: '4.2.10.Final' + implementation group: 'io.netty', name: 'netty-codec', version: '4.1.125.Final' + + implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "**/logstash-core*.jar") + implementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "guardium-universalconnector-commons*.jar") + + + testImplementation 'junit:junit:4.13.1' + testImplementation 'org.jruby:jruby-complete:9.3.2.0' + + testImplementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "guardium-universalconnector-commons*.jar") + testImplementation("org.junit.jupiter:junit-jupiter-api:5.12.1") + +} + +clean { + delete "${projectDir}/Gemfile" + delete "${projectDir}/" + pluginInfo.pluginFullName() + ".gemspec" + delete "${projectDir}/lib/" + delete "${projectDir}/vendor/" + new FileNameFinder().getFileNames(projectDir.toString(), pluginInfo.pluginFullName() + "-?.?.?.gem").each { filename -> + delete filename + } +} + +tasks.withType(JavaCompile) { + options.encoding = 'UTF-8' +} + +tasks.register("vendor"){ + dependsOn shadowJar + doLast { + String vendorPathPrefix = "vendor/jar-dependencies" + String projectGroupPath = project.group.replaceAll('\\.', '/') + File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") + projectJarFile.mkdirs() + Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) + } +} + +tasks.register("generateRubySupportFiles") { + doLast { + generateRubySupportFilesForPlugin(project.description, project.group, version) + } +} + +tasks.register("removeObsoleteJars") { + doLast { + new FileNameFinder().getFileNames( + projectDir.toString(), + "vendor/**/" + pluginInfo.pluginFullName() + "*.jar", + "vendor/**/" + pluginInfo.pluginFullName() + "-" + version + ".jar").each { f -> + delete f + } + } +} + +tasks.register("gem"){ + dependsOn = [downloadAndInstallJRuby, removeObsoleteJars, vendor, generateRubySupportFiles] + doLast { + buildGem(projectDir, buildDir, pluginInfo.pluginFullName() + ".gemspec") + } +} diff --git a/input-plugin/logstash-input-s3sqs/gradle/wrapper/gradle-wrapper.jar b/input-plugin/logstash-input-s3sqs/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 000000000..62d4c0535 Binary files /dev/null and b/input-plugin/logstash-input-s3sqs/gradle/wrapper/gradle-wrapper.jar differ diff --git a/input-plugin/logstash-input-s3sqs/gradle/wrapper/gradle-wrapper.properties b/input-plugin/logstash-input-s3sqs/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 000000000..81aa1c044 --- /dev/null +++ b/input-plugin/logstash-input-s3sqs/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,5 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/input-plugin/logstash-input-s3sqs/gradlew b/input-plugin/logstash-input-s3sqs/gradlew new file mode 100755 index 000000000..fbd7c5158 --- /dev/null +++ b/input-plugin/logstash-input-s3sqs/gradlew @@ -0,0 +1,185 @@ +#!/usr/bin/env sh + +# +# Copyright 2015 the original author or authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin or MSYS, switch paths to Windows format before running java +if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=`expr $i + 1` + done + case $i in + 0) set -- ;; + 1) set -- "$args0" ;; + 2) set -- "$args0" "$args1" ;; + 3) set -- "$args0" "$args1" "$args2" ;; + 4) set -- "$args0" "$args1" "$args2" "$args3" ;; + 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=`save "$@"` + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +exec "$JAVACMD" "$@" diff --git a/input-plugin/logstash-input-s3sqs/gradlew.bat b/input-plugin/logstash-input-s3sqs/gradlew.bat new file mode 100644 index 000000000..5093609d5 --- /dev/null +++ b/input-plugin/logstash-input-s3sqs/gradlew.bat @@ -0,0 +1,104 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto init + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto init + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:init +@rem Get command-line arguments, handling Windows variants + +if not "%OS%" == "Windows_NT" goto win9xME_args + +:win9xME_args +@rem Slurp the command line arguments. +set CMD_LINE_ARGS= +set _SKIP=2 + +:win9xME_args_slurp +if "x%~1" == "x" goto execute + +set CMD_LINE_ARGS=%* + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/input-plugin/logstash-input-s3sqs/input-s3sqs.conf b/input-plugin/logstash-input-s3sqs/input-s3sqs.conf new file mode 100644 index 000000000..27cb09d75 --- /dev/null +++ b/input-plugin/logstash-input-s3sqs/input-s3sqs.conf @@ -0,0 +1,16 @@ +input { + s3_sqs { + queue_url => "" + region => "" + access_key_id => "" + secret_access_key => "" + role_arn => "" # Leave empty if not using role-based access + max_messages => + wait_time => # Must be >= 0 and <= 20, + polling_frequency => + type => "" + add_field => { "account_id" => "" } + } +} + + diff --git a/input-plugin/logstash-input-s3sqs/src/main/java/org/logstashplugins/S3SQS.java b/input-plugin/logstash-input-s3sqs/src/main/java/org/logstashplugins/S3SQS.java new file mode 100644 index 000000000..e6c9ace1a --- /dev/null +++ b/input-plugin/logstash-input-s3sqs/src/main/java/org/logstashplugins/S3SQS.java @@ -0,0 +1,394 @@ +package org.logstashplugins; + +import co.elastic.logstash.api.*; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; +import org.json.JSONArray; +import org.json.JSONObject; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.sqs.SqsClient; +import software.amazon.awssdk.services.sqs.model.DeleteMessageRequest; +import software.amazon.awssdk.services.sqs.model.Message; +import software.amazon.awssdk.services.sqs.model.ReceiveMessageRequest; +import software.amazon.awssdk.services.sts.StsClient; +import software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.time.Instant; +import java.util.*; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; +import java.util.zip.GZIPInputStream; + +@LogstashPlugin(name = "s3_sqs") +public class S3SQS implements Input, AutoCloseable { + private final String queueUrl; + private final int maxMessages; + private final int waitTime; + private final Context context; + private volatile boolean stopped = false; + private final CountDownLatch done = new CountDownLatch(1); + private ExecutorService executorService; + private Consumer> consumer; + private final S3Client s3Client; + private final SqsClient sqsClient; + private final String type; + private final Long pollingFrequency; + private static final String LOG_EVENTS = "logEvents"; + private static final String RECORDS = "Records"; + private Map addField = new HashMap<>(); + + private static final PluginConfigSpec QUEUE_URL = PluginConfigSpec.stringSetting("queue_url"); + private static final PluginConfigSpec REGION = PluginConfigSpec.stringSetting("region"); + private static final PluginConfigSpec ACCESS_KEY = PluginConfigSpec.stringSetting("access_key_id"); + private static final PluginConfigSpec SECRET_KEY = PluginConfigSpec.stringSetting("secret_access_key"); + private static final PluginConfigSpec ARN = PluginConfigSpec.stringSetting("role_arn"); + private static final PluginConfigSpec MAX_MESSAGES = PluginConfigSpec.numSetting("max_messages", 10); + private static final PluginConfigSpec WAIT_TIME = PluginConfigSpec.numSetting("wait_time", 20); + private static final PluginConfigSpec TYPE = PluginConfigSpec.stringSetting("type"); + private static final PluginConfigSpec SQS_POLLING_FREQUENCY = PluginConfigSpec.numSetting("polling_frequency", 10); + private static final PluginConfigSpec> ADD_FIELD = PluginConfigSpec.hashSetting("add_field", Collections.emptyMap(), false, false); + + private static final ObjectMapper mapper = new ObjectMapper(); + + + + public S3SQS(String id, Configuration config, Context context) { + this.context = context; + this.queueUrl = config.get(QUEUE_URL); + this.maxMessages = Math.toIntExact(config.get(MAX_MESSAGES)); + this.waitTime = Math.toIntExact(config.get(WAIT_TIME)); + String region = config.get(REGION); + String accessKeyId = config.get(ACCESS_KEY); + String secretAccessKey = config.get(SECRET_KEY); + String arn = config.get(ARN); + this.type = config.get(TYPE); + this.addField = config.get(ADD_FIELD); + + + // convert seconds to milliseconds + this.pollingFrequency = (1000 * config.get(SQS_POLLING_FREQUENCY)); + + AwsCredentialsProvider credentialsProvider; + if (arn != null && !arn.isEmpty()) { + credentialsProvider = StsAssumeRoleCredentialsProvider.builder() + .refreshRequest(b -> b + .roleArn(arn) + .roleSessionName((type != null && !type.isEmpty())? type : "S3SQS-session")) + .stsClient(software.amazon.awssdk.services.sts.StsClient.builder() + .region(Region.of(region)) + .credentialsProvider(DefaultCredentialsProvider.create()) + .build()) + .build(); + + } else { + credentialsProvider = StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKeyId, secretAccessKey)); + } + + this.s3Client = S3Client.builder() + .region(Region.of(region)) + .credentialsProvider(credentialsProvider) + .overrideConfiguration(ClientOverrideConfiguration.builder() + .apiCallTimeout(Duration.ofMinutes(2)) + .build()) + .build(); + + this.sqsClient = SqsClient.builder() + .region(Region.of(region)) + .credentialsProvider(credentialsProvider) + .build(); + } + + @Override + public void start(Consumer> consumer) { + this.stopped = false; + this.consumer = consumer; + this.executorService = Executors.newFixedThreadPool(1); + + executorService.submit(() -> { + try { + while (!stopped) { + processMessages(); + try { + Thread.sleep(this.pollingFrequency); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + context.getLogger(this).error("Thread interrupted: {}", e.getMessage()); + break; + } + } + } finally { + stopped = true; + context.getLogger(this).info("S3SQS input plugin stopped"); + done.countDown(); + } + }); + } + + + private void processMessages() { + try { + context.getLogger(this).debug("Polling messages from SQS..."); + ReceiveMessageRequest receiveRequest = ReceiveMessageRequest.builder() + .queueUrl(queueUrl) + .maxNumberOfMessages(maxMessages) + .waitTimeSeconds(waitTime) + .build(); + + List messages = sqsClient.receiveMessage(receiveRequest).messages(); + if (!messages.isEmpty()) { + messages.forEach(this::processSQSMessage); + } + } catch (Exception e) { + context.getLogger(this).error("Error processing messages", e); + } + } + + private void processSQSMessage(Message message) { + boolean processingSuccessful = false; + try { + JSONObject jsonMessage = new JSONObject(message.body()); + JSONArray records = jsonMessage.optJSONArray("Records"); + + if (records != null) { + for (int i = 0; i < records.length(); i++) { + JSONObject record = records.getJSONObject(i); + JSONObject s3Info = record.getJSONObject("s3"); + String bucketName = s3Info.getJSONObject("bucket").getString("name"); + String fileKey = s3Info.getJSONObject("object").getString("key"); + fetchAndProcessFile(bucketName, fileKey); + } + } + // Mark as successful only if no exceptions occurred + processingSuccessful = true; + } catch (Exception e) { + context.getLogger(this).error("Error processing SQS message - message will be retried after visibility timeout", e); + // Don't delete message - let it become visible again for retry + } finally { + // Only delete message if processing was successful + if (processingSuccessful) { + deleteMessageFromQueue(message); + context.getLogger(this).debug("Successfully processed and deleted message from queue"); + } else { + context.getLogger(this).warn("Message processing failed - message will remain in queue for retry"); + } + } + } + + private void fetchAndProcessFile(String bucketName, String fileKey) { + context.getLogger(this).debug("Fetching file from S3 - Bucket: {} | FileKey: {}", bucketName, fileKey); + + GetObjectRequest getObjectRequest = GetObjectRequest.builder() + .bucket(bucketName) + .key(fileKey) + .build(); + + try { + InputStream inputStream = s3Client.getObject(getObjectRequest); + InputStream effectiveStream = fileKey.contains(".gz") + ? new GZIPInputStream(inputStream) + : inputStream; + + try (BufferedReader reader = new BufferedReader(new InputStreamReader(effectiveStream, StandardCharsets.UTF_8))) { + if(fileKey.contains(".gz")){ + reader.lines().forEach(line -> generateEvent(bucketName, fileKey, line)); + } else if (fileKey.contains(".csv")) { + List> jsonData = processCSVFile(reader); + for (Map jsonDataMap : jsonData){ + context.getLogger(this).debug("jsonDataMap {} ", jsonDataMap); + processEventString(bucketName,fileKey,mapper.writeValueAsString(jsonDataMap)); + } + } + } catch (IOException ioException){ + generateErrorEventMap(bucketName, fileKey, ioException, "IOException Failed to fetch file from S3"); + } + + } catch (Exception e) { + if (e instanceof JsonProcessingException ) + generateErrorEventMap(bucketName, fileKey, e, "JsonProcessingException"); + else + generateErrorEventMap(bucketName, fileKey, e, "Failed to fetch file from S3"); + } + } + + private List> processCSVFile(BufferedReader reader) { + + List> jsonData = new ArrayList<>(); + try ( + CSVParser csvParser = new CSVParser(reader, CSVFormat.DEFAULT.withFirstRecordAsHeader())) { + for (CSVRecord record : csvParser) { + Map jsonObject = new LinkedHashMap<>(); + for (String header : csvParser.getHeaderMap().keySet()) { + jsonObject.put(header, record.get(header)); + } + jsonData.add(jsonObject); + + context.getLogger(this).debug("processCSVFile jsonObject {} ", mapper.writeValueAsString(jsonObject)); + } + + } catch (IOException e) { + context.getLogger(this).error("processCSVFile cause {} | message {}", e.getCause(), e.getMessage()); + } + return jsonData; + } + + private void generateEvent(String bucketName, String fileKey, String line) { + context.getLogger(this).debug("line : {}", line); + if (isValidaJSON(line)) { + try { + JsonNode rootNode = getJSON(line); + if (rootNode.isArray()) { + rootNode.forEach(node -> { + processEventArray(bucketName, fileKey, node); + }); + } else { + processEventString(bucketName, fileKey, line); + } + } catch (JsonProcessingException jsonProcessingException) { + context.getLogger(this).error("JsonProcessingException", jsonProcessingException); + generateErrorEventMap(bucketName, fileKey, jsonProcessingException,"JsonProcessingException"); + } + } + } + + private void processEventString(String bucketName, String fileKey, String jsonString) { + //ObjectMapper mapper = new ObjectMapper(); + //Map eventMap = mapper.convertValue(jsonString, Map.class); + Map eventMap = new HashMap<>(); + eventMap.put("message",jsonString); + eventMap.put("bucketName", bucketName); + eventMap.put("fileKey", fileKey); + eventMap.put("timestamp", Instant.now().toString()); + eventMap.put("type", this.type); + eventMap.putAll(addField); + consumer.accept(eventMap); + } + + private void processEventArray(String bucketName, String fileKey, JsonNode rootNode) { + //ObjectMapper objMapper = new ObjectMapper(); + rootNode.forEach(node -> { + //Map eventMap = objMapper.convertValue(node, Map.class); + Map eventMap = new HashMap<>(); + eventMap.put("message",node); + eventMap.put("bucketName", bucketName); + eventMap.put("fileKey", fileKey); + eventMap.put("timestamp", Instant.now().toString()); + eventMap.put("type", this.type); + eventMap.putAll(addField); + consumer.accept(eventMap); + }); + } + + private void generateErrorEventMap(String bucketName, String fileKey, Exception e, String exceptionDetail) { + Map errorEvent = new HashMap<>(); + errorEvent.put("error", exceptionDetail); + errorEvent.put("bucketName", bucketName); + errorEvent.put("fileKey", fileKey); + errorEvent.put("timestamp", Instant.now().toString()); + errorEvent.put("type", this.type); + errorEvent.putAll(addField); + consumer.accept(errorEvent); + context.getLogger(this).error("Error fetching or processing file from S3", e); + } + + private boolean isValidaJSON(String json) { + try { + ObjectMapper objMapper = new ObjectMapper(); + objMapper.readTree(json); + return true; + } catch (Exception e) { + context.getLogger(this).error("Not valid JSON {}", json); + return false; + } + } + + private JsonNode getJSON(String json) throws JsonProcessingException { + ObjectMapper objMapper = new ObjectMapper(); + return objMapper.readTree(json); + } + + private void deleteMessageFromQueue(Message message) { + try { + sqsClient.deleteMessage(DeleteMessageRequest.builder() + .queueUrl(queueUrl) + .receiptHandle(message.receiptHandle()) + .build()); + } catch (Exception e) { + context.getLogger(this).error("Error deleting message from SQS", e); + } + } + + + @Override + public void stop() { + context.getLogger(this).info("S3SQS stop() called - setting stopped flag to true"); + stopped = true; + if (executorService != null) { + executorService.shutdown(); + try { + if (!executorService.awaitTermination(20, TimeUnit.SECONDS)) { + executorService.shutdownNow(); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + executorService.shutdownNow(); + } + } + // Don't close AWS clients here - they need to be reused if start() is called again + // close() will be called by Logstash when the plugin is truly being destroyed + context.getLogger(this).info("S3SQS stop() completed"); + } + + @Override + public void awaitStop() throws InterruptedException { + done.await(); + } + + @Override + public void close() { + try { + if (sqsClient != null) { + sqsClient.close(); + } + if (s3Client != null) { + s3Client.close(); + } + } catch (Exception e) { + context.getLogger(this).error("Error closing AWS clients", e); + } + } + + @Override + public Collection> configSchema() { + return List.of(QUEUE_URL, REGION, ACCESS_KEY, SECRET_KEY, ARN, MAX_MESSAGES, WAIT_TIME, TYPE, SQS_POLLING_FREQUENCY, ADD_FIELD); + } + + @Override + public String getName() { + return "s3_sqs_input"; + } + + @Override + public String getId() { + return UUID.randomUUID().toString(); + } +} diff --git a/input-plugin/logstash-input-s3sqs/src/test/java/org/logstashplugins/S3SQSLogstashPluginTest.java b/input-plugin/logstash-input-s3sqs/src/test/java/org/logstashplugins/S3SQSLogstashPluginTest.java new file mode 100644 index 000000000..b45057f90 --- /dev/null +++ b/input-plugin/logstash-input-s3sqs/src/test/java/org/logstashplugins/S3SQSLogstashPluginTest.java @@ -0,0 +1,42 @@ +package org.logstashplugins; + +import org.junit.Test; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.sqs.SqsClient; +import software.amazon.awssdk.services.sqs.model.ReceiveMessageResponse; +import software.amazon.awssdk.services.sqs.model.Message; + +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +public class S3SQSLogstashPluginTest { + + @Test + public void testRetrieveMessageFromSQS() { + Message message = Message.builder().body("{\"Records\":[{\"s3\":{\"bucket\":{\"name\":\"example-bucket\"},\"object\":{\"key\":\"2025/02/27/11/example-file-name\"}}}]}" ).build(); + ReceiveMessageResponse response = ReceiveMessageResponse.builder().messages(Collections.singletonList(message)).build(); + + List messages = Collections.singletonList(message); + + assertFalse(messages.isEmpty()); + assertEquals("{\"Records\":[{\"s3\":{\"bucket\":{\"name\":\"example-bucket\"},\"object\":{\"key\":\"2025/02/27/11/example-file-name\"}}}]}", messages.get(0).body()); + } + + @Test + public void testRetrieveFileFromS3() { + String fileContent = "{\"timestamp\":\"2025-02-27T11:17:53.777587Z\",\"bucketName\":\"example-bucket\",\"Name\":\"user@example.com\",\"type\":\"S3SQS\",\"message\":\"{\"messageType\":\"DATA_MESSAGE\",\"owner\":\"123456789012\",\"logGroup\":\"aws/rds/cluster/example/audit\",\"logStream\":\"example-instance-1.audit.log.1\",\"subscriptionFilters\":[\"ExampleCloudwatchToS3\"],\"logEvents\":[{\"id\":\"38817886306025407784629710918078008262425635349182414848\",\"timestamp\":1740654223007,\"message\":\"CREATE TABLE log_20250227_110342 ( id INT AUTO_INCREMENT PRIMARY KEY, data VARCHAR(255) NOT NULL, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ) ENGINE=InnoDB\"}]}\"}\",\"@version\":\"1\",\"@timestamp\":\"2025-02-27T11:17:53.777677Z\",\"fileKey\":\"2025/02/27/11/example-file-name\"}"; + ByteArrayInputStream inputStream = new ByteArrayInputStream(fileContent.getBytes(StandardCharsets.UTF_8)); + + GetObjectRequest request = GetObjectRequest.builder().bucket("example-bucket").key("2025/02/27/11/example-file-name").build(); + GetObjectResponse response = GetObjectResponse.builder().build(); + + String content = new String(inputStream.readAllBytes(), StandardCharsets.UTF_8); + + assertEquals(fileContent, content); + } +} diff --git a/input-plugin/logstash-input-sqs-custom/AWSSQSInputPackage/SQSInput/input.conf b/input-plugin/logstash-input-sqs-custom/AWSSQSInputPackage/SQSInput/input.conf new file mode 100644 index 000000000..d98e7ab56 --- /dev/null +++ b/input-plugin/logstash-input-sqs-custom/AWSSQSInputPackage/SQSInput/input.conf @@ -0,0 +1,19 @@ +input { + custom_sqs { + queue => "guc_input_param_queue" + access_key_id => "${GUC_CRED_ACCESS_KEY_ID}" + region => "guc_input_param_region" + role_arn => "guc_input_param_role-arn" + secret_access_key => "${GUC_CRED_SECRET_ACCESS_KEY}" + # Optional: Specify a custom endpoint (e.g., proxy) + # endpoint => "https://proxy.company.com" + # Set to true to use AWS's bundled CA certificates for SSL/TLS connections + use_aws_bundled_ca => false + # Optional: Provide additional settings (e.g., custom SSL certificate bundle) + # additional_settings => { + # ssl_ca_bundle => "/usr/share/logstash/third_party/" + # } + add_field => { "account_id" => "guc_input_param_account_id" } + type => "guc_input_param_type" + } +} \ No newline at end of file diff --git a/input-plugin/logstash-input-sqs-custom/AWSSQSInputPackage/SQSInput/logstash-input-custom_sqs.zip b/input-plugin/logstash-input-sqs-custom/AWSSQSInputPackage/SQSInput/logstash-input-custom_sqs.zip new file mode 100644 index 000000000..8ba586213 Binary files /dev/null and b/input-plugin/logstash-input-sqs-custom/AWSSQSInputPackage/SQSInput/logstash-input-custom_sqs.zip differ diff --git a/input-plugin/logstash-input-sqs-custom/AWSSQSInputPackage/SQSInput/manifest.json b/input-plugin/logstash-input-sqs-custom/AWSSQSInputPackage/SQSInput/manifest.json new file mode 100644 index 000000000..08473aa04 --- /dev/null +++ b/input-plugin/logstash-input-sqs-custom/AWSSQSInputPackage/SQSInput/manifest.json @@ -0,0 +1,22 @@ +{ + "name": "AWS SQS custom input", + "alias": "AWS SQS custom input", + "type": "input", + "pipeline_type": null, + "plugin_version": "1.0.0", + "supportedDataSources": null, + "supportedInputPlugins": null, + "developer": "Elastic,IBM", + "license": "Apache2.0", + "supported_input_plugins": [], + "description": "Pull events from an Amazon Simple Queue Service (SQS) queue and passes them to the filter stage", + "configuration_notes": "", + "documentation_path": "", + "pluginSHA256Checksum": -1, + "pluginOfflinePackagePath": null, + "pluginConfigTemplateParams": null, + "pluginConfigTemplateStrings": null, + "uploadDate": null, + "uploadUser": null, + "auto_scaling": true +} \ No newline at end of file diff --git a/input-plugin/logstash-input-sqs-custom/AWSSQSInputPackage/gi_templates.json b/input-plugin/logstash-input-sqs-custom/AWSSQSInputPackage/gi_templates.json new file mode 100644 index 000000000..8df61d61f --- /dev/null +++ b/input-plugin/logstash-input-sqs-custom/AWSSQSInputPackage/gi_templates.json @@ -0,0 +1,11 @@ +{ + "plugin_name": "AWS SQS custom input", + "help_link": "", + "input_name": "AWS SQS custom input", + "input_parameters": [ + ], + "filter_name": "", + "filter_parameters": [], + "auth_parameters": [ + ] +} \ No newline at end of file diff --git a/input-plugin/logstash-input-sqs-custom/CHANGELOG.md b/input-plugin/logstash-input-sqs-custom/CHANGELOG.md new file mode 100644 index 000000000..e69de29bb diff --git a/input-plugin/logstash-input-sqs-custom/LICENSE b/input-plugin/logstash-input-sqs-custom/LICENSE new file mode 100644 index 000000000..a80a3fd53 --- /dev/null +++ b/input-plugin/logstash-input-sqs-custom/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2020 Elastic and contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/input-plugin/logstash-input-sqs-custom/README.md b/input-plugin/logstash-input-sqs-custom/README.md new file mode 100644 index 000000000..154edf34d --- /dev/null +++ b/input-plugin/logstash-input-sqs-custom/README.md @@ -0,0 +1,148 @@ +# AWS SQS Input Plugin + +## Overview +* Version: 1.0.0 +* Supported Guardium versions: 12.0 and above + +This is a [Logstash](https://github.com/elastic/logstash) input plug-in for the universal connector that is featured in IBM Security Guardium. It pulls events from the SQS from the Amazon Web Services. The events are then sent over to corresponding filter plugin which transforms these audit logs into a [Guardium record](https://github.com/IBM/universal-connectors/blob/main/common/src/main/java/com/ibm/guardium/universalconnector/commons/structures/Record.java) instance (which is a standard structure made out of several parts). The information is then sent over to Guardium. Guardium records include the accessor (the person who tried to access the data), the session, data, and exceptions. If there are no errors, the data contains details about the query "construct". The construct details the main action (verb) and collections (objects) involved. + +This implementation prevents pipeline crashes due to credential or network errors through graceful error handling. + +## Purpose: + +This plug-in pulls events from an Amazon Web Services Simple Queue Service (SQS) queue. + +SQS is a simple, scalable queue system that is part of the Amazon Web Services suite of tools. + +### Creating the SQS queue +**_Procedure_** +1. Go to https://console.aws.amazon.com/ +2. Click **Services** +3. Search for SQS and click on **Simple Queue Services** +4. Click **Create Queue**. +5. Select the type as **Standard**. +6. Enter the name for the queue. +7. Keep the rest of the default settings. + + +## Usage: + +### a. Prerequisites: + +1. Have an AWS account. + +2. Set up an SQS queue as mentioned previously. + +3. Create an identity that has access to consume messages from the queue. + +4. The "consumer" identity must have the following permissions on the queue: + + ``` + sqs:ChangeMessageVisibility + + sqs:ChangeMessageVisibilityBatch + + sqs:DeleteMessage + + sqs:DeleteMessageBatch + + sqs:GetQueueAttributes + + sqs:GetQueueUrl + + sqs:ListQueues + + sqs:ReceiveMessage + ``` + +5. Create a user and apply the below IAM Policy to the user. + + ``` + { + "Statement": [ + { + "Action": [ + "sqs:ChangeMessageVisibility", + "sqs:ChangeMessageVisibilityBatch", + "sqs:DeleteMessage", + "sqs:DeleteMessageBatch", + "sqs:GetQueueAttributes", + "sqs:GetQueueUrl", + "sqs:ListQueues", + "sqs:ReceiveMessage" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:sqs:us-east-1:123456789012:Logstash" + ] + } + ] + } + ``` + + +### b. Parameters: + +| Parameter | Input Type | Required | Default | +|-----------|------------|----------|---------| +| access_key_id | String | No | | +| secret_access_key | String | No | | +| polling_frequency | Number | No | 20 | +| queue | String | Yes | | +| region | String | No | | +| role_arn | string | No | | + + +#### `access_key_id` +The `access_key_id` setting allows to set the access key ID for the user that has access to SQS. This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order: + + 1. Static configuration, using access_key_id and secret_access_key params in logstash plugin config. + + 2. External credentials file specified by an aws_credentials_file. + + 3. Environment variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY. + + 4. Environment variables AMAZON_ACCESS_KEY_ID and AMAZON_SECRET_ACCESS_KEY. + + 5. IAM Instance Profile (available when running inside EC2). + +#### `secret_access_key` +The `secret_access_key` setting defines the AWS Secret Access Key. + +#### `polling_frequency` +The `polling_frequency` setting defines the frequency for the queue to be polled. + +#### `queue` +The `queue` setting specifies the name of the SQS queue to pull messages from. Note that this is just the name of the queue, not the URL or ARN. + +#### `region` +The `region` setting defines the region where the SQS is present. + +#### `role_arn` +The role_arn setting allows you to specify which AWS IAM Role to assume, if any. This is used to generate temporary credentials, typically for cross-account access. To understand more about the settings to be followed while using this parameter, click [here]( ./SettingsForRoleArn.md ) + + +#### Logstash Default configuration parameters +Other standard logstash parameters are available, such as: +* `add_field` +* `type` +* `tags` + +### Example + + input { + custom_sqs { + access_key_id => "" + secret_access_key => "" + queue => "" + region => "" + } + } + +### Authorizing outgoing traffic from AWS to Guardium + +#### Procedure +1. Log in to the Guardium collector's API. +2. Issue these commands: + • grdapi add_domain_to_universal_connector_allowed_domains domain=amazonaws.com + • grdapi add_domain_to_universal_connector_allowed_domains domain=amazon.com \ No newline at end of file diff --git a/input-plugin/logstash-input-sqs-custom/VERSION b/input-plugin/logstash-input-sqs-custom/VERSION new file mode 100644 index 000000000..afaf360d3 --- /dev/null +++ b/input-plugin/logstash-input-sqs-custom/VERSION @@ -0,0 +1 @@ +1.0.0 \ No newline at end of file diff --git a/input-plugin/logstash-input-sqs-custom/build.gradle b/input-plugin/logstash-input-sqs-custom/build.gradle new file mode 100644 index 000000000..807d04b26 --- /dev/null +++ b/input-plugin/logstash-input-sqs-custom/build.gradle @@ -0,0 +1,146 @@ +import java.nio.file.Files +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING + +apply plugin: 'java' +def universalConnectorsDir=project.projectDir.parentFile?.parentFile.toString(); +def versions = new org.yaml.snakeyaml.Yaml().load( new File("${universalConnectorsDir}/versions.yml").newInputStream() ) +gradle.ext.versions = new org.yaml.snakeyaml.Yaml().load( new File(LOGSTASH_CORE_PATH + "/../versions.yml").newInputStream() ) + + +apply from: LOGSTASH_CORE_PATH + "/../rubyUtils.gradle" + +// =========================================================================== +// plugin info +// =========================================================================== +group 'org.logstashplugins' // must match the package of the main plugin class +version "${file("VERSION").text.trim()}" // read from required VERSION file +description = "Custom SQS input plugin" +pluginInfo.licenses = ['Apache-2.0'] // list of SPDX license IDs +pluginInfo.longDescription = "This gem is a Logstash custom SQS input plugin with graceful error handling required to be installed as part of IBM Security Guardium, Guardium Universal connector configuration. This gem is not a stand-alone program." +pluginInfo.authors = ['IBM'] +pluginInfo.email = [''] +pluginInfo.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html" +pluginInfo.pluginType = "input" +pluginInfo.pluginClass = "CustomSQS" +pluginInfo.pluginName = "custom_sqs" // must match the @LogstashPlugin annotation in the main plugin class +// =========================================================================== + +sourceCompatibility = JavaVersion.VERSION_17 +targetCompatibility = JavaVersion.VERSION_17 + +buildscript { + repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } + } + + dependencies { + classpath 'com.github.johnrengelman.shadow:com.github.johnrengelman.shadow.gradle.plugin:8.1.1' + classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' + } + ext { + + snakeYamlVersion = '2.2' + + } +} + +repositories { + mavenLocal() + maven { + url System.getenv("ARTIFACTORY_URL") ?: "https://repo.maven.apache.org/maven2/" + credentials { + username System.getenv("ARTIFACTORY_USERNAME") ?: "" + password System.getenv("ARTIFACTORY_AUTH_TOKEN") ?: "" + } + } +} + +apply plugin: 'com.github.johnrengelman.shadow' + +shadowJar { + archiveClassifier = null + zip64 true +} + +dependencies { + + implementation group: 'software.amazon.awssdk', name: 'sqs', version: '2.30.17' + + implementation group: 'software.amazon.awssdk', name: 'sts', version: '2.30.19' + + implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.17.1' + + implementation group: 'org.apache.logging.log4j', name: 'log4j-api', version: '2.17.1' + + implementation group: 'io.netty', name: 'netty-codec-http2', version: '4.2.11.Final' + implementation group: 'io.netty', name: 'netty-codec-http', version: '4.2.10.Final' + implementation group: 'io.netty', name: 'netty-codec', version: '4.1.125.Final' + + implementation fileTree(dir: LOGSTASH_CORE_PATH, include: "build/libs/logstash-core.jar") + implementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "common-*.*.*.jar") + + + testImplementation 'junit:junit:4.13.1' + testImplementation 'org.jruby:jruby-complete:9.3.2.0' + + testImplementation fileTree(dir: GUARDIUM_UNIVERSALCONNECTOR_COMMONS_PATH, include: "common-*.*.*.jar") + testImplementation("org.junit.jupiter:junit-jupiter-api:5.12.1") + +} + +clean { + delete "${projectDir}/Gemfile" + delete "${projectDir}/" + pluginInfo.pluginFullName() + ".gemspec" + delete "${projectDir}/lib/" + delete "${projectDir}/vendor/" + new FileNameFinder().getFileNames(projectDir.toString(), pluginInfo.pluginFullName() + "-?.?.?.gem").each { filename -> + delete filename + } +} + +tasks.withType(JavaCompile) { + options.encoding = 'UTF-8' +} + +tasks.register("vendor"){ + dependsOn shadowJar + doLast { + String vendorPathPrefix = "vendor/jar-dependencies" + String projectGroupPath = project.group.replaceAll('\\.', '/') + File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar") + projectJarFile.mkdirs() + Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) + } +} + +tasks.register("generateRubySupportFiles") { + doLast { + generateRubySupportFilesForPlugin(project.description, project.group, version) + } +} + +tasks.register("removeObsoleteJars") { + doLast { + new FileNameFinder().getFileNames( + projectDir.toString(), + "vendor/**/" + pluginInfo.pluginFullName() + "*.jar", + "vendor/**/" + pluginInfo.pluginFullName() + "-" + version + ".jar").each { f -> + delete f + } + } +} + +tasks.register("gem"){ + dependsOn = [downloadAndInstallJRuby, removeObsoleteJars, vendor, generateRubySupportFiles] + doLast { + buildGem(projectDir, buildDir, pluginInfo.pluginFullName() + ".gemspec") + } +} + diff --git a/input-plugin/logstash-input-sqs-custom/gradle/wrapper/gradle-wrapper.jar b/input-plugin/logstash-input-sqs-custom/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 000000000..62d4c0535 Binary files /dev/null and b/input-plugin/logstash-input-sqs-custom/gradle/wrapper/gradle-wrapper.jar differ diff --git a/input-plugin/logstash-input-sqs-custom/gradle/wrapper/gradle-wrapper.properties b/input-plugin/logstash-input-sqs-custom/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 000000000..81aa1c044 --- /dev/null +++ b/input-plugin/logstash-input-sqs-custom/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,5 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/input-plugin/logstash-input-sqs-custom/gradlew b/input-plugin/logstash-input-sqs-custom/gradlew new file mode 100755 index 000000000..fbd7c5158 --- /dev/null +++ b/input-plugin/logstash-input-sqs-custom/gradlew @@ -0,0 +1,185 @@ +#!/usr/bin/env sh + +# +# Copyright 2015 the original author or authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin or MSYS, switch paths to Windows format before running java +if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=`expr $i + 1` + done + case $i in + 0) set -- ;; + 1) set -- "$args0" ;; + 2) set -- "$args0" "$args1" ;; + 3) set -- "$args0" "$args1" "$args2" ;; + 4) set -- "$args0" "$args1" "$args2" "$args3" ;; + 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=`save "$@"` + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +exec "$JAVACMD" "$@" diff --git a/input-plugin/logstash-input-sqs-custom/gradlew.bat b/input-plugin/logstash-input-sqs-custom/gradlew.bat new file mode 100644 index 000000000..5093609d5 --- /dev/null +++ b/input-plugin/logstash-input-sqs-custom/gradlew.bat @@ -0,0 +1,104 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto init + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto init + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:init +@rem Get command-line arguments, handling Windows variants + +if not "%OS%" == "Windows_NT" goto win9xME_args + +:win9xME_args +@rem Slurp the command line arguments. +set CMD_LINE_ARGS= +set _SKIP=2 + +:win9xME_args_slurp +if "x%~1" == "x" goto execute + +set CMD_LINE_ARGS=%* + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/input-plugin/logstash-input-sqs-custom/src/main/java/org/logstashplugins/CustomSQS.java b/input-plugin/logstash-input-sqs-custom/src/main/java/org/logstashplugins/CustomSQS.java new file mode 100644 index 000000000..4cfd4f855 --- /dev/null +++ b/input-plugin/logstash-input-sqs-custom/src/main/java/org/logstashplugins/CustomSQS.java @@ -0,0 +1,509 @@ +package org.logstashplugins; + +import co.elastic.logstash.api.*; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.sqs.SqsClient; +import software.amazon.awssdk.services.sqs.model.*; +import software.amazon.awssdk.services.sts.StsClient; +import software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider; +import software.amazon.awssdk.services.sts.model.AssumeRoleRequest; + +import java.util.*; +import java.util.concurrent.CountDownLatch; +import java.util.function.Consumer; + +/** + * Custom SQS Input Plugin + * + * This plugin reads messages from AWS SQS with graceful error handling. + * + * @author IBM Guardium Universal Connectors Team + * @version 1.0.0 + */ +@LogstashPlugin(name = "custom_sqs") +public class CustomSQS implements Input { + + private static final Logger logger = LogManager.getLogger(CustomSQS.class); + + // Plugin configuration parameters + protected static final PluginConfigSpec QUEUE_NAME = PluginConfigSpec.requiredStringSetting("queue"); + protected static final PluginConfigSpec REGION = PluginConfigSpec.stringSetting("region", "us-east-1"); + protected static final PluginConfigSpec ACCESS_KEY = PluginConfigSpec.stringSetting("access_key_id"); + protected static final PluginConfigSpec SECRET_KEY = PluginConfigSpec.stringSetting("secret_access_key"); + protected static final PluginConfigSpec ROLE_ARN = PluginConfigSpec.stringSetting("role_arn"); + protected static final PluginConfigSpec ENDPOINT = PluginConfigSpec.stringSetting("endpoint"); + protected static final PluginConfigSpec POLLING_FREQUENCY = PluginConfigSpec.numSetting("polling_frequency", 20); + protected static final PluginConfigSpec MAX_MESSAGES = PluginConfigSpec.numSetting("max_messages", 10); + protected static final PluginConfigSpec TYPE = PluginConfigSpec.stringSetting("type"); + protected static final PluginConfigSpec> ADD_FIELD = PluginConfigSpec.hashSetting("add_field", Collections.emptyMap(), false, false); + protected static final PluginConfigSpec CODEC = PluginConfigSpec.stringSetting("codec", "plain"); + protected static final PluginConfigSpec USE_AWS_BUNDLED_CA = PluginConfigSpec.booleanSetting("use_aws_bundled_ca", true); + protected static final PluginConfigSpec> ADDITIONAL_SETTINGS = PluginConfigSpec.hashSetting("additional_settings", Collections.emptyMap(), false, false); + + private final String id; + private final String queueName; + private final String region; + private final String accessKeyId; + private final String secretAccessKey; + private final String roleArn; + private final String endpoint; + private final long pollingFrequency; + private final int maxMessages; + private final String type; + private final Map addField; + private final boolean useAwsBundledCa; + private final Map additionalSettings; + + private SqsClient sqsClient; + private String queueUrl; + private volatile boolean stopped = false; + private final CountDownLatch done = new CountDownLatch(1); + private boolean connectionValid = false; + private Consumer> consumer; + + /** + * Constructor called by Logstash + */ + public CustomSQS(String id, Configuration config, Context context) { + this.id = id; + this.queueName = config.get(QUEUE_NAME); + this.region = config.get(REGION); + this.accessKeyId = config.get(ACCESS_KEY); + this.secretAccessKey = config.get(SECRET_KEY); + this.roleArn = config.get(ROLE_ARN); + this.endpoint = config.get(ENDPOINT); + this.pollingFrequency = config.get(POLLING_FREQUENCY); + this.maxMessages = Math.toIntExact(config.get(MAX_MESSAGES)); + this.type = config.get(TYPE); + this.addField = config.get(ADD_FIELD); + this.useAwsBundledCa = config.get(USE_AWS_BUNDLED_CA); + this.additionalSettings = config.get(ADDITIONAL_SETTINGS); + + logger.info("Custom SQS Guardium input plugin initialized", + "queue", queueName, + "region", region); + } + + @Override + public void start(Consumer> consumer) { + this.consumer = consumer; + + // Try to connect with retry for transient errors + if (connectWithRetry()) { + // Successfully connected, start polling messages + pollMessages(); + } else { + // Failed after retries (credential error or max retries exceeded) + sleepIndefinitely(); + } + } + + /** + * Attempt to connect to SQS with retry logic for transient errors + * Credential errors are not retried + * + * @return true if connection successful, false otherwise + */ + private boolean connectWithRetry() { + int maxRetries = 5; + long sleepMs = 1000; // Start with 1 second + long maxSleepMs = 60000; // Max 60 seconds + + for (int attempt = 1; attempt <= maxRetries && !stopped; attempt++) { + try { + logger.info("Attempting to connect to SQS queue: {} (attempt {}/{})", + queueName, attempt, maxRetries); + + initializeSqsClient(); + connectionValid = true; + logger.info("Successfully connected to SQS queue: {}", queueName); + return true; + + } catch (SqsException e) { + connectionValid = false; + String errorCode = e.awsErrorDetails() != null ? e.awsErrorDetails().errorCode() : "UNKNOWN"; + String errorMessage = e.awsErrorDetails() != null ? e.awsErrorDetails().errorMessage() : e.getMessage(); + + // Extract only the first sentence/line of the error message + if (errorMessage != null && errorMessage.contains("\n")) { + errorMessage = errorMessage.substring(0, errorMessage.indexOf("\n")).trim(); + } + + // Check if it's a credential error (permanent - don't retry) + if (isCredentialError(errorCode)) { + logger.error("Credential error detected - Queue: " + queueName + + ", Region: " + region + + ", Error Code: " + errorCode + + ", Message: " + errorMessage + + " - Not retrying, requires configuration fix"); + return false; + } + + // Network or transient error - retry with backoff + logger.warn("Connection attempt {} failed - Queue: {}, Error Code: {}, Message: {} - Retrying in {}ms", + attempt, queueName, errorCode, errorMessage, sleepMs); + + if (attempt < maxRetries) { + sleep(sleepMs); + sleepMs = Math.min(sleepMs * 2, maxSleepMs); // Exponential backoff + } + + } catch (Exception e) { + // Unknown error - retry with backoff + connectionValid = false; + String errorMessage = e.getMessage(); + + if (errorMessage != null && errorMessage.contains("\n")) { + errorMessage = errorMessage.substring(0, errorMessage.indexOf("\n")).trim(); + } + + logger.warn("Unexpected error on attempt {} - Queue: {}, Error: {}, Type: {} - Retrying in {}ms", + attempt, queueName, errorMessage, e.getClass().getName(), sleepMs); + + if (attempt < maxRetries) { + sleep(sleepMs); + sleepMs = Math.min(sleepMs * 2, maxSleepMs); // Exponential backoff + } + } + } + + // Failed after all retries + logger.error("Failed to connect to SQS after {} attempts - Queue: {}, Region: {}", + maxRetries, queueName, region); + return false; + } + + /** + * Check if the error code indicates a credential/authentication error + * These errors are permanent and should not be retried + * + * @param errorCode AWS error code + * @return true if credential error, false otherwise + */ + private boolean isCredentialError(String errorCode) { + return "SignatureDoesNotMatch".equals(errorCode) || + "InvalidClientTokenId".equals(errorCode) || + "InvalidAccessKeyId".equals(errorCode) || + "AccessDenied".equals(errorCode) || + "UnrecognizedClientException".equals(errorCode) || + "InvalidSignatureException".equals(errorCode); + } + + /** + * Initialize SQS client with credentials + * This is where credential validation happens + */ + private void initializeSqsClient() { + logger.info("Initializing SQS client for queue: {}", queueName); + + AwsCredentialsProvider credentialsProvider; + + // Setup credentials provider + if (roleArn != null && !roleArn.isEmpty() && !roleArn.equals("guc_input_param_role-arn")) { + // Use role ARN for cross-account access + logger.info("Using role ARN for authentication: {}", roleArn); + + StsClient stsClient = StsClient.builder() + .region(Region.of(region)) + .credentialsProvider(StaticCredentialsProvider.create( + AwsBasicCredentials.create(accessKeyId, secretAccessKey))) + .build(); + + credentialsProvider = StsAssumeRoleCredentialsProvider.builder() + .refreshRequest(AssumeRoleRequest.builder() + .roleArn(roleArn) + .roleSessionName("sqs-guardium-" + System.currentTimeMillis()) + .build()) + .stsClient(stsClient) + .build(); + + } else if (accessKeyId != null && !accessKeyId.isEmpty()) { + // Use static credentials + logger.info("Using static credentials for authentication"); + credentialsProvider = StaticCredentialsProvider.create( + AwsBasicCredentials.create(accessKeyId, secretAccessKey)); + } else { + // Use default credentials chain + logger.info("Using default credentials provider"); + credentialsProvider = DefaultCredentialsProvider.create(); + } + + // Create SQS client builder + software.amazon.awssdk.services.sqs.SqsClientBuilder clientBuilder = SqsClient.builder() + .region(Region.of(region)) + .credentialsProvider(credentialsProvider); + + // Apply custom endpoint if provided (for VPC endpoints or proxies) + if (endpoint != null && !endpoint.isEmpty()) { + try { + clientBuilder.endpointOverride(java.net.URI.create(endpoint)); + logger.info("Using custom SQS endpoint: {}", endpoint); + } catch (Exception e) { + logger.error("Invalid endpoint URL: {}", endpoint, e); + } + } + + // Apply additional settings if provided + if (additionalSettings != null && !additionalSettings.isEmpty()) { + applyAdditionalSettings(clientBuilder); + } + + sqsClient = clientBuilder.build(); + + // Get queue URL - this validates credentials + GetQueueUrlRequest getQueueUrlRequest = GetQueueUrlRequest.builder() + .queueName(queueName) + .build(); + + GetQueueUrlResponse response = sqsClient.getQueueUrl(getQueueUrlRequest); + queueUrl = response.queueUrl(); + + logger.info("Successfully retrieved queue URL: {}", queueUrl); + } + + /** + * Poll messages from SQS queue + */ + private void pollMessages() { + logger.info("Starting SQS message polling", "queue", queueName, "interval", pollingFrequency); + + while (!stopped) { + try { + // Check if client is still valid before using it + if (sqsClient == null || stopped) { + break; + } + + // Receive messages from queue + ReceiveMessageRequest receiveRequest = ReceiveMessageRequest.builder() + .queueUrl(queueUrl) + .maxNumberOfMessages(maxMessages) + .waitTimeSeconds((int) pollingFrequency) + .build(); + + ReceiveMessageResponse receiveResponse = sqsClient.receiveMessage(receiveRequest); + List messages = receiveResponse.messages(); + + // Process each message + for (Message message : messages) { + if (stopped) break; // Check stopped flag during message processing + processMessage(message); + } + + } catch (SqsException e) { + if (stopped) break; // Exit if stopping + + logger.error("Error receiving SQS messages - Queue: " + queueName + + ", Error: " + e.getMessage() + + ", Code: " + (e.awsErrorDetails() != null ? e.awsErrorDetails().errorCode() : "UNKNOWN")); + + // Sleep before retry + sleep(pollingFrequency * 1000); + + } catch (IllegalStateException e) { + // Connection pool shut down - plugin is stopping + if (stopped) { + logger.info("SQS client closed during shutdown - Queue: " + queueName); + break; + } + logger.error("Unexpected IllegalStateException in SQS polling - Queue: " + queueName + + ", Error: " + e.getMessage()); + sleep(pollingFrequency * 1000); + + } catch (Exception e) { + if (stopped) break; // Exit if stopping + + logger.error("Unexpected error in SQS polling - Queue: " + queueName + + ", Error: " + e.getMessage() + + ", Type: " + e.getClass().getName()); + + // Sleep before retry + sleep(pollingFrequency * 1000); + } + } + + done.countDown(); + } + + /** + * Process a single SQS message + */ + private void processMessage(Message message) { + try { + // Create event map + Map event = new HashMap<>(); + event.put("message", message.body()); + event.put("sqs_message_id", message.messageId()); + event.put("sqs_receipt_handle", message.receiptHandle()); + + // Add type if specified + if (type != null && !type.isEmpty()) { + event.put("type", type); + } + + // Send to Logstash pipeline + consumer.accept(event); + + // Delete message from queue after successful processing + DeleteMessageRequest deleteRequest = DeleteMessageRequest.builder() + .queueUrl(queueUrl) + .receiptHandle(message.receiptHandle()) + .build(); + + sqsClient.deleteMessage(deleteRequest); + + logger.debug("Successfully processed and deleted message: {}", message.messageId()); + + } catch (Exception e) { + logger.error("Error processing SQS message", + "messageId", message.messageId(), + "error", e.getMessage()); + } + } + + /** + * Sleep indefinitely when connection is invalid + * This keeps the plugin alive without processing messages + */ + private void sleepIndefinitely() { + logger.error("SQS input not running due to connection failure. Fix credentials and reload pipeline."); + + while (!stopped) { + sleep(pollingFrequency * 1000); + logger.error("SQS input sleeping due to connection failure", + "queue", queueName, + "interval", pollingFrequency); + } + + done.countDown(); + } + + /** + * Apply additional settings to the SQS client builder + * + * This method processes all settings in the additional_settings map and applies + * them to the client builder. Each setting is handled individually with proper + * error handling to ensure one failing setting doesn't prevent others from being applied. + * + * Supported settings: + * - ssl_ca_bundle: Path to custom SSL certificate file + * - (Add more settings as needed) + */ + private void applyAdditionalSettings(software.amazon.awssdk.services.sqs.SqsClientBuilder clientBuilder) { + logger.info("Applying {} additional setting(s) to SQS client", additionalSettings.size()); + + additionalSettings.forEach((key, value) -> { + if (value == null) { + logger.warn("Skipping additional setting '{}' with null value", key); + return; + } + + try { + logger.debug("Processing additional setting: {} = {}", key, value); + + // Handle each setting type + if ("ssl_ca_bundle".equals(key)) { + configureSslCertificate(clientBuilder, value.toString()); + } else { + // Warn about unrecognized settings to help catch typos + logger.warn("Unrecognized additional setting '{}' = {} (will be ignored). Valid settings: ssl_ca_bundle", key, value); + } + + } catch (Exception e) { + logger.error("Failed to apply additional setting '{}': {}", key, e.getMessage(), e); + // Continue processing other settings even if one fails + } + }); + } + + /** + * Configure custom SSL certificate for the SQS client + * + * @param clientBuilder The SQS client builder + * @param certPath Path to the SSL certificate file + */ + private void configureSslCertificate(software.amazon.awssdk.services.sqs.SqsClientBuilder clientBuilder, String certPath) { + logger.info("Configuring custom SSL certificate from: {}", certPath); + + java.io.File certFile = new java.io.File(certPath); + if (!certFile.exists()) { + logger.error("Custom SSL certificate file not found at path: {}. Please verify the path exists and is accessible.", certPath); + return; + } + + try { + // Set the custom CA bundle as a system property + // This will be used by the AWS SDK for SSL/TLS connections + System.setProperty("aws.caBundlePath", certPath); + logger.info("Successfully configured custom SSL certificate path: {}", certPath); + } catch (Exception e) { + logger.error("Failed to configure SSL certificate from '{}': {}", certPath, e.getMessage(), e); + throw new RuntimeException("SSL certificate configuration failed", e); + } + } + + /** + * Sleep for specified milliseconds + */ + private void sleep(long milliseconds) { + try { + Thread.sleep(milliseconds); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + @Override + public void stop() { + stopped = true; + logger.info("Stopping SQS input plugin", "queue", queueName); + + // Close SQS client safely + SqsClient clientToClose = sqsClient; + sqsClient = null; // Set to null first to prevent further use + + if (clientToClose != null) { + try { + clientToClose.close(); + logger.info("SQS client closed successfully", "queue", queueName); + } catch (Exception e) { + logger.warn("Error closing SQS client (this is normal during shutdown)", "error", e.getMessage()); + } + } + } + + @Override + public void awaitStop() throws InterruptedException { + done.await(); + } + + @Override + public Collection> configSchema() { + return Arrays.asList( + QUEUE_NAME, + REGION, + ACCESS_KEY, + SECRET_KEY, + ROLE_ARN, + ENDPOINT, + POLLING_FREQUENCY, + MAX_MESSAGES, + TYPE, + ADD_FIELD, + CODEC, + USE_AWS_BUNDLED_CA, + ADDITIONAL_SETTINGS + ); + } + + @Override + public String getId() { + return id; + } +} diff --git a/input-plugin/logstash-input-sqs-custom/src/test/java/org/logstashplugins/CustomSQSTest.java b/input-plugin/logstash-input-sqs-custom/src/test/java/org/logstashplugins/CustomSQSTest.java new file mode 100644 index 000000000..e0da9eb78 --- /dev/null +++ b/input-plugin/logstash-input-sqs-custom/src/test/java/org/logstashplugins/CustomSQSTest.java @@ -0,0 +1,269 @@ +package org.logstashplugins; + +import co.elastic.logstash.api.Configuration; +import org.junit.Assert; +import org.junit.Test; +import org.logstash.plugins.ConfigurationImpl; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.util.HashMap; +import java.util.Map; + +/** + * Unit tests for SQSInput plugin + */ +public class CustomSQSTest { + + @Test + public void testSQSInputCreation_WithValidConfig() { + Map configValues = new HashMap<>(); + configValues.put(CustomSQS.QUEUE_NAME.name(), "test-queue"); + configValues.put(CustomSQS.REGION.name(), "us-east-1"); + configValues.put(CustomSQS.ACCESS_KEY.name(), "test-access-key"); + configValues.put(CustomSQS.SECRET_KEY.name(), "test-secret-key"); + configValues.put(CustomSQS.POLLING_FREQUENCY.name(), 20L); + configValues.put(CustomSQS.MAX_MESSAGES.name(), 10L); + + Configuration config = new ConfigurationImpl(configValues); + CustomSQS input = new CustomSQS("test-id", config, null); + + Assert.assertNotNull("SQS Input should be created", input); + Assert.assertEquals("Plugin ID should match", "test-id", input.getId()); + } + + @Test + public void testSQSInputCreation_WithRoleArn() { + Map configValues = new HashMap<>(); + configValues.put(CustomSQS.QUEUE_NAME.name(), "test-queue"); + configValues.put(CustomSQS.REGION.name(), "us-west-2"); + configValues.put(CustomSQS.ACCESS_KEY.name(), "test-access-key"); + configValues.put(CustomSQS.SECRET_KEY.name(), "test-secret-key"); + configValues.put(CustomSQS.ROLE_ARN.name(), "arn:aws:iam::123456789012:role/test-role"); + configValues.put(CustomSQS.POLLING_FREQUENCY.name(), 30L); + configValues.put(CustomSQS.MAX_MESSAGES.name(), 5L); + configValues.put(CustomSQS.TYPE.name(), "sqs-input"); + + Configuration config = new ConfigurationImpl(configValues); + CustomSQS input = new CustomSQS("test-id-2", config, null); + + Assert.assertNotNull("SQS Input with role ARN should be created", input); + Assert.assertEquals("Plugin ID should match", "test-id-2", input.getId()); + } + + @Test + public void testSQSInputCreation_WithDefaultValues() { + Map configValues = new HashMap<>(); + configValues.put(CustomSQS.QUEUE_NAME.name(), "default-queue"); + configValues.put(CustomSQS.ACCESS_KEY.name(), "test-key"); + configValues.put(CustomSQS.SECRET_KEY.name(), "test-secret"); + + Configuration config = new ConfigurationImpl(configValues); + CustomSQS input = new CustomSQS("test-id-3", config, null); + + Assert.assertNotNull("SQS Input with defaults should be created", input); + Assert.assertEquals("Plugin ID should match", "test-id-3", input.getId()); + } + + @Test + public void testConfigSchema_NotNull() { + Map configValues = new HashMap<>(); + configValues.put(CustomSQS.QUEUE_NAME.name(), "test-queue"); + configValues.put(CustomSQS.ACCESS_KEY.name(), "test-key"); + configValues.put(CustomSQS.SECRET_KEY.name(), "test-secret"); + + Configuration config = new ConfigurationImpl(configValues); + CustomSQS input = new CustomSQS("test-id", config, null); + + Assert.assertNotNull("Config schema should not be null", input.configSchema()); + Assert.assertTrue("Config schema should have multiple fields", input.configSchema().size() > 0); + } + + @Test + public void testStop_DoesNotThrowException() { + Map configValues = new HashMap<>(); + configValues.put(CustomSQS.QUEUE_NAME.name(), "test-queue"); + configValues.put(CustomSQS.ACCESS_KEY.name(), "test-key"); + configValues.put(CustomSQS.SECRET_KEY.name(), "test-secret"); + + Configuration config = new ConfigurationImpl(configValues); + CustomSQS input = new CustomSQS("test-id", config, null); + + // Stop should not throw exception + input.stop(); + + Assert.assertTrue("Stop completed successfully", true); + } + + @Test + public void testSQSInputCreation_WithInvalidCredentials() { + // Test that plugin can be created even with invalid credentials + // The actual connection failure will happen when start() is called + Map configValues = new HashMap<>(); + configValues.put(CustomSQS.QUEUE_NAME.name(), "test-queue"); + configValues.put(CustomSQS.REGION.name(), "us-east-1"); + configValues.put(CustomSQS.ACCESS_KEY.name(), "INVALID_ACCESS_KEY"); + configValues.put(CustomSQS.SECRET_KEY.name(), "INVALID_SECRET_KEY"); + configValues.put(CustomSQS.POLLING_FREQUENCY.name(), 20L); + configValues.put(CustomSQS.MAX_MESSAGES.name(), 10L); + + Configuration config = new ConfigurationImpl(configValues); + CustomSQS input = new CustomSQS("test-invalid-creds", config, null); + + // Plugin should be created successfully + // Credential validation happens during start(), not during construction + Assert.assertNotNull("SQS Input should be created even with invalid credentials", input); + Assert.assertEquals("Plugin ID should match", "test-invalid-creds", input.getId()); + + // Note: We don't call start() here because it would attempt to connect to AWS + // The plugin's error handling will catch credential errors during start() + // and enter sleep mode without crashing the pipeline + } + + @Test + public void testSQSInputCreation_WithCustomEndpoint() { + Map configValues = new HashMap<>(); + configValues.put(CustomSQS.QUEUE_NAME.name(), "test-queue"); + configValues.put(CustomSQS.REGION.name(), "us-east-1"); + configValues.put(CustomSQS.ACCESS_KEY.name(), "test-key"); + configValues.put(CustomSQS.SECRET_KEY.name(), "test-secret"); + configValues.put(CustomSQS.ENDPOINT.name(), "https://vpce-xxxxx.sqs.us-east-1.vpce.amazonaws.com"); + + Configuration config = new ConfigurationImpl(configValues); + CustomSQS input = new CustomSQS("test-endpoint", config, null); + + Assert.assertNotNull("SQS Input with custom endpoint should be created", input); + Assert.assertEquals("Plugin ID should match", "test-endpoint", input.getId()); + } + + @Test + public void testSQSInputCreation_WithAwsBundledCA() { + Map configValues = new HashMap<>(); + configValues.put(CustomSQS.QUEUE_NAME.name(), "test-queue"); + configValues.put(CustomSQS.REGION.name(), "us-east-1"); + configValues.put(CustomSQS.ACCESS_KEY.name(), "test-key"); + configValues.put(CustomSQS.SECRET_KEY.name(), "test-secret"); + configValues.put(CustomSQS.USE_AWS_BUNDLED_CA.name(), true); + + Configuration config = new ConfigurationImpl(configValues); + CustomSQS input = new CustomSQS("test-bundled-ca", config, null); + + Assert.assertNotNull("SQS Input with AWS bundled CA should be created", input); + Assert.assertEquals("Plugin ID should match", "test-bundled-ca", input.getId()); + } + + @Test + public void testSQSInputCreation_WithCustomSslCertificate() throws IOException { + // Create a temporary test certificate file + File tempCert = createTestCertificateFile(); + + try { + Map configValues = new HashMap<>(); + configValues.put(CustomSQS.QUEUE_NAME.name(), "test-queue"); + configValues.put(CustomSQS.REGION.name(), "us-east-1"); + configValues.put(CustomSQS.ACCESS_KEY.name(), "test-key"); + configValues.put(CustomSQS.SECRET_KEY.name(), "test-secret"); + + // Configure additional_settings with ssl_ca_bundle + Map additionalSettings = new HashMap<>(); + additionalSettings.put("ssl_ca_bundle", tempCert.getAbsolutePath()); + configValues.put(CustomSQS.ADDITIONAL_SETTINGS.name(), additionalSettings); + + Configuration config = new ConfigurationImpl(configValues); + CustomSQS input = new CustomSQS("test-ssl-cert", config, null); + + Assert.assertNotNull("SQS Input with custom SSL certificate should be created", input); + Assert.assertEquals("Plugin ID should match", "test-ssl-cert", input.getId()); + + // Note: The actual SSL configuration happens during start() when the SQS client is built + // This test verifies that the configuration is accepted without errors + } finally { + // Clean up + if (tempCert != null && tempCert.exists()) { + tempCert.delete(); + } + } + } + + @Test + public void testSQSInputCreation_WithNonExistentSslCertificate() { + // Test that plugin can be created even with non-existent certificate path + // The actual validation happens during start() + Map configValues = new HashMap<>(); + configValues.put(CustomSQS.QUEUE_NAME.name(), "test-queue"); + configValues.put(CustomSQS.REGION.name(), "us-east-1"); + configValues.put(CustomSQS.ACCESS_KEY.name(), "test-key"); + configValues.put(CustomSQS.SECRET_KEY.name(), "test-secret"); + + Map additionalSettings = new HashMap<>(); + additionalSettings.put("ssl_ca_bundle", "/non/existent/path/ca-bundle.pem"); + configValues.put(CustomSQS.ADDITIONAL_SETTINGS.name(), additionalSettings); + + Configuration config = new ConfigurationImpl(configValues); + CustomSQS input = new CustomSQS("test-missing-cert", config, null); + + Assert.assertNotNull("SQS Input should be created even with non-existent cert path", input); + Assert.assertEquals("Plugin ID should match", "test-missing-cert", input.getId()); + } + + @Test + public void testSQSInputCreation_WithAllSslOptions() throws IOException { + File tempCert = createTestCertificateFile(); + + try { + Map configValues = new HashMap<>(); + configValues.put(CustomSQS.QUEUE_NAME.name(), "test-queue"); + configValues.put(CustomSQS.REGION.name(), "us-east-1"); + configValues.put(CustomSQS.ACCESS_KEY.name(), "test-key"); + configValues.put(CustomSQS.SECRET_KEY.name(), "test-secret"); + configValues.put(CustomSQS.ENDPOINT.name(), "https://vpce-xxxxx.sqs.us-east-1.vpce.amazonaws.com"); + configValues.put(CustomSQS.USE_AWS_BUNDLED_CA.name(), false); + + Map additionalSettings = new HashMap<>(); + additionalSettings.put("ssl_ca_bundle", tempCert.getAbsolutePath()); + configValues.put(CustomSQS.ADDITIONAL_SETTINGS.name(), additionalSettings); + + Configuration config = new ConfigurationImpl(configValues); + CustomSQS input = new CustomSQS("test-all-ssl", config, null); + + Assert.assertNotNull("SQS Input with all SSL options should be created", input); + Assert.assertEquals("Plugin ID should match", "test-all-ssl", input.getId()); + } finally { + if (tempCert != null && tempCert.exists()) { + tempCert.delete(); + } + } + } + + /** + * Helper method to create a DUMMY test certificate file + * + * @return File containing a dummy PEM certificate for testing + * @throws IOException if file creation fails + */ + private File createTestCertificateFile() throws IOException { + File tempFile = File.createTempFile("test-ca-", ".pem"); + + // DUMMY/FAKE certificate - Only for testing configuration parsing + String dummyCert = "-----BEGIN CERTIFICATE-----\n" + + "TESTTESTTESTTESTTESTESTTESTTESTTESTTESTTESTESTTESTTESTTESTTESTTESTEST\n" + + "TESTTESTTESTTESTTESTESTTESTTESTTESTTESTTESTESTTESTTESTTESTTESTTESTEST\n" + + "TESTTESTTESTTESTTESTESTTESTTESTTESTTESTTESTESTTESTTESTTESTTESTTESTEST\n" + + "TESTTESTTESTTESTTESTESTTESTTESTTESTTESTTESTESTTESTTESTTESTTESTTESTEST\n" + + "TESTTESTTESTTESTTESTESTTESTTESTTESTTESTTESTESTTESTTESTTESTTESTTESTEST\n" + + "TESTTESTTESTTESTTESTESTTESTTESTTESTTESTTESTESTTESTTESTTESTTESTTESTEST\n" + + "TESTTESTTESTTESTTESTESTTESTTESTTESTTESTTESTESTTESTTESTTESTTESTTESTEST\n" + + "TESTTESTTESTTESTTESTESTTESTTESTTESTTESTTESTESTTESTTESTTESTTESTTESTEST\n" + + "TESTTESTTESTTESTTESTESTTESTTESTTESTTESTTESTESTTESTTESTTESTTESTTESTEST\n" + + "TESTTESTTESTTESTTESTESTTESTTESTTESTTESTTESTESTTESTTESTTESTTESTTESTEST\n" + + "-----END CERTIFICATE-----\n"; + + try (FileWriter writer = new FileWriter(tempFile)) { + writer.write(dummyCert); + } + + return tempFile; + } +} diff --git a/input-plugin/logstash-input-sqs/README.md b/input-plugin/logstash-input-sqs/README.md index bbf0e6b46..345e75175 100644 --- a/input-plugin/logstash-input-sqs/README.md +++ b/input-plugin/logstash-input-sqs/README.md @@ -38,6 +38,7 @@ SQS is a simple, scalable queue system that is part of the Amazon Web Services s 4. The "consumer" identity must have the following permissions on the queue: + ``` sqs:ChangeMessageVisibility sqs:ChangeMessageVisibilityBatch @@ -53,32 +54,32 @@ SQS is a simple, scalable queue system that is part of the Amazon Web Services s sqs:ListQueues sqs:ReceiveMessage + ``` 5. Create a user and apply the below IAM Policy to the user. -``` - { - "Statement": [ - { - "Action": [ - "sqs:ChangeMessageVisibility", - "sqs:ChangeMessageVisibilityBatch", - "sqs:DeleteMessage", - "sqs:DeleteMessageBatch", - "sqs:GetQueueAttributes", - "sqs:GetQueueUrl", - "sqs:ListQueues", - "sqs:ReceiveMessage" - ], - "Effect": "Allow", - "Resource": [ - "arn:aws:sqs:us-east-1:123456789012:Logstash" - ] - } - ] - } -``` - + ``` + { + "Statement": [ + { + "Action": [ + "sqs:ChangeMessageVisibility", + "sqs:ChangeMessageVisibilityBatch", + "sqs:DeleteMessage", + "sqs:DeleteMessageBatch", + "sqs:GetQueueAttributes", + "sqs:GetQueueUrl", + "sqs:ListQueues", + "sqs:ReceiveMessage" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:sqs:us-east-1:123456789012:Logstash" + ] + } + ] + } + ``` ### b. Parameters: @@ -91,9 +92,6 @@ SQS is a simple, scalable queue system that is part of the Amazon Web Services s | region | String | No | | | role_arn | string | No | | - - - #### `access_key_id` The `access_key_id` setting allows to set the access key ID for the user that has access to SQS. This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order: @@ -161,4 +159,4 @@ store sync_timezone ``` systemctl restart guard-snif ``` -3. On the Guardium machine, go to **Setup > Tools and Views > Configure Universal Connector** page and restart the Universal Connector by disabling and then enabling it. +3. On the Guardium machine, go to **Setup** > **Tools and Views** > **Configure Universal Connector** page and restart the Universal Connector by disabling and then enabling it. diff --git a/input-plugin/logstash-input-sqs/SQSInputPackage/SQS/input.conf b/input-plugin/logstash-input-sqs/SQSInputPackage/SQS/input.conf index b9e2bef2d..d0eee6560 100644 --- a/input-plugin/logstash-input-sqs/SQSInputPackage/SQS/input.conf +++ b/input-plugin/logstash-input-sqs/SQSInputPackage/SQS/input.conf @@ -1,11 +1,19 @@ -input{ +input { sqs { - queue => "guc_input_param_queue" - access_key_id => "${GUC_CRED_ACCESS_KEY_ID}" - region => "guc_input_param_region" - role_arn => "guc_input_param_role-arn" - secret_access_key => "${GUC_CRED_SECRET_ACCESS_KEY}" - add_field => { "account_id" => "guc_input_param_account_id" } - type => "guc_input_param_type" + queue => "guc_input_param_queue" + access_key_id => "${GUC_CRED_ACCESS_KEY_ID}" + region => "guc_input_param_region" + role_arn => "guc_input_param_role-arn" + secret_access_key => "${GUC_CRED_SECRET_ACCESS_KEY}" + # Optional: Specify a custom endpoint (e.g., proxy) + # endpoint => "https://proxy.company.com" + # Set to true to use AWS's bundled CA certificates for SSL/TLS connections + use_aws_bundled_ca => false + # Optional: Provide additional settings (e.g., custom SSL certificate bundle) + # additional_settings => { + # ssl_ca_bundle => "/usr/share/logstash/third_party/" + # } + add_field => { "account_id" => "guc_input_param_account_id" } + type => "guc_input_param_type" } } \ No newline at end of file diff --git a/versions.yml b/versions.yml index 4b99d99a1..65ae8041d 100644 --- a/versions.yml +++ b/versions.yml @@ -1,18 +1,19 @@ --- dependencies: commonsValidator: 1.7 - log4jCore: 2.22.0 + log4jCore: 2.25.3 log4jApi: 2.17.2 - commonsLang: 3.7 + commonsLang: 3.18.0 gson: 2.8.9 junit: 4.12 jrubyComplete: 9.2.7.0 junitJupiter: 5.7.1 mockitoAll: 2.0.2-beta json: 20231013 - parboiledJava: 1.1.8 + parboiledJava: 1.4.1 javaxJson: 1.1.4 guava: 32.1.3-jre commonsText: 1.10.0 tinkergraphGremlin: 3.6.8 - rdf4jQueryparserSparql: 5.0.3 \ No newline at end of file + rdf4jQueryparserSparql: 5.0.3 + commonsBeanutils: 1.11.0 \ No newline at end of file