From 36f88481d02333d7cdffce418f39b51d31de48eb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 Jan 2025 10:05:33 -0500 Subject: [PATCH 001/658] chore(ci): Bump bufbuild/buf-setup-action from 1.49.0 to 1.50.0 (#22258) Bumps [bufbuild/buf-setup-action](https://github.com/bufbuild/buf-setup-action) from 1.49.0 to 1.50.0. - [Release notes](https://github.com/bufbuild/buf-setup-action/releases) - [Commits](https://github.com/bufbuild/buf-setup-action/compare/v1.49.0...v1.50.0) --- updated-dependencies: - dependency-name: bufbuild/buf-setup-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/protobuf.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/protobuf.yml b/.github/workflows/protobuf.yml index 1899b78441..20fa9f864f 100644 --- a/.github/workflows/protobuf.yml +++ b/.github/workflows/protobuf.yml @@ -21,7 +21,7 @@ jobs: # Run `git checkout` - uses: actions/checkout@v4 # Install the `buf` CLI - - uses: bufbuild/buf-setup-action@v1.49.0 + - uses: bufbuild/buf-setup-action@v1.50.0 # Perform breaking change detection against the `master` branch - uses: bufbuild/buf-breaking-action@v1.1.4 with: From 7ee7bf7362d8abcc375079821a498f6075d85bef Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 Jan 2025 10:07:59 -0500 Subject: [PATCH 002/658] chore(ci): Bump docker/build-push-action from 6.11.0 to 6.12.0 (#22259) Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 6.11.0 to 6.12.0. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/v6.11.0...v6.12.0) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/environment.yml | 2 +- .github/workflows/regression.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/environment.yml b/.github/workflows/environment.yml index e155eb811f..31691d0f2f 100644 --- a/.github/workflows/environment.yml +++ b/.github/workflows/environment.yml @@ -62,7 +62,7 @@ jobs: org.opencontainers.image.title=Vector development environment org.opencontainers.image.url=https://github.com/vectordotdev/vector - name: Build and push - uses: docker/build-push-action@v6.11.0 + uses: docker/build-push-action@v6.12.0 with: context: . file: ./scripts/environment/Dockerfile diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 1e13adb872..0837242a55 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -204,7 +204,7 @@ jobs: uses: docker/setup-buildx-action@v3.8.0 - name: Build 'vector' target image - uses: docker/build-push-action@v6.11.0 + uses: docker/build-push-action@v6.12.0 with: context: baseline-vector/ cache-from: type=gha @@ -243,7 +243,7 @@ jobs: uses: docker/setup-buildx-action@v3.8.0 - name: Build 'vector' target image - uses: docker/build-push-action@v6.11.0 + uses: docker/build-push-action@v6.12.0 with: context: comparison-vector/ cache-from: type=gha From a60b8bb9b84e7f70a2fe58c3b01d79f8542ef91b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 Jan 2025 15:58:01 +0000 Subject: [PATCH 003/658] chore(deps): Bump data-encoding from 2.6.0 to 2.7.0 (#22250) Bumps [data-encoding](https://github.com/ia0/data-encoding) from 2.6.0 to 2.7.0. - [Commits](https://github.com/ia0/data-encoding/commits) --- updated-dependencies: - dependency-name: data-encoding dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- lib/dnsmsg-parser/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4d4e3ad137..bfcd602399 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2857,9 +2857,9 @@ dependencies = [ [[package]] name = "data-encoding" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" +checksum = "0e60eed09d8c01d3cee5b7d30acb059b76614c918fa0f992e0dd6eeb10daad6f" [[package]] name = "data-url" diff --git a/lib/dnsmsg-parser/Cargo.toml b/lib/dnsmsg-parser/Cargo.toml index ceab56f024..ddb22ddfba 100644 --- a/lib/dnsmsg-parser/Cargo.toml +++ b/lib/dnsmsg-parser/Cargo.toml @@ -7,7 +7,7 @@ publish = false license = "MIT" [dependencies] -data-encoding = "2.6" +data-encoding = "2.7" hickory-proto.workspace = true snafu.workspace = true From 504a596649b2ff3d0f92bb1c8e2fa0ac06b0af8d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 Jan 2025 15:58:29 +0000 Subject: [PATCH 004/658] chore(deps): Bump directories from 5.0.1 to 6.0.0 (#22251) Bumps [directories](https://github.com/soc/directories-rs) from 5.0.1 to 6.0.0. - [Commits](https://github.com/soc/directories-rs/commits) --- updated-dependencies: - dependency-name: directories dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 52 ++++++++++++++++++++++++++++++++++++++++++++----- vdev/Cargo.toml | 2 +- 2 files changed, 48 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bfcd602399..f646e49ec6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2365,7 +2365,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "45b1f4c00870f07dc34adcac82bb6a72cc5aabca8536ba1797e01df51d2ce9a0" dependencies = [ - "directories", + "directories 5.0.1", "serde", "thiserror 1.0.68", "toml", @@ -3033,7 +3033,16 @@ version = "5.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a49173b84e034382284f27f1af4dcbbd231ffa358c0fe316541a7337f376a35" dependencies = [ - "dirs-sys", + "dirs-sys 0.4.1", +] + +[[package]] +name = "directories" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16f5094c54661b38d03bd7e50df373292118db60b585c08a411c6d840017fe7d" +dependencies = [ + "dirs-sys 0.5.0", ] [[package]] @@ -3054,10 +3063,22 @@ checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" dependencies = [ "libc", "option-ext", - "redox_users", + "redox_users 0.4.3", "windows-sys 0.48.0", ] +[[package]] +name = "dirs-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" +dependencies = [ + "libc", + "option-ext", + "redox_users 0.5.0", + "windows-sys 0.59.0", +] + [[package]] name = "dirs-sys-next" version = "0.1.2" @@ -3065,7 +3086,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" dependencies = [ "libc", - "redox_users", + "redox_users 0.4.3", "winapi", ] @@ -5513,6 +5534,16 @@ version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" +[[package]] +name = "libredox" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +dependencies = [ + "bitflags 2.6.0", + "libc", +] + [[package]] name = "libz-sys" version = "1.1.12" @@ -8044,6 +8075,17 @@ dependencies = [ "thiserror 1.0.68", ] +[[package]] +name = "redox_users" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd6f9d3d47bdd2ad6945c5015a226ec6155d0bcdfd8f7cd29f86b71f8de99d2b" +dependencies = [ + "getrandom 0.2.15", + "libredox", + "thiserror 2.0.3", +] + [[package]] name = "regex" version = "1.11.1" @@ -10874,7 +10916,7 @@ dependencies = [ "clap-verbosity-flag", "clap_complete", "confy", - "directories", + "directories 6.0.0", "dunce", "glob", "hex", diff --git a/vdev/Cargo.toml b/vdev/Cargo.toml index 82d8d03a2d..eda13a0ae3 100644 --- a/vdev/Cargo.toml +++ b/vdev/Cargo.toml @@ -14,7 +14,7 @@ clap.workspace = true clap-verbosity-flag = "3.0.2" clap_complete = "4.5.42" confy = "0.6.1" -directories = "5.0.1" +directories = "6.0.0" # remove this when stabilized https://doc.rust-lang.org/stable/std/path/fn.absolute.html dunce = "1.0.5" glob.workspace = true From 02d40b9f32f7694a6b60073f605adc81d9b9e192 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 Jan 2025 15:59:06 +0000 Subject: [PATCH 005/658] chore(deps): Bump cargo-lock from 10.0.1 to 10.1.0 (#22252) Bumps [cargo-lock](https://github.com/rustsec/rustsec) from 10.0.1 to 10.1.0. - [Release notes](https://github.com/rustsec/rustsec/releases) - [Commits](https://github.com/rustsec/rustsec/commits) --- updated-dependencies: - dependency-name: cargo-lock dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- lib/vector-vrl/web-playground/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f646e49ec6..2c28a258b0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1931,9 +1931,9 @@ checksum = "a3e368af43e418a04d52505cf3dbc23dda4e3407ae2fa99fd0e4f308ce546acc" [[package]] name = "cargo-lock" -version = "10.0.1" +version = "10.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6469776d007022d505bbcc2be726f5f096174ae76d710ebc609eb3029a45b551" +checksum = "c06acb4f71407ba205a07cb453211e0e6a67b21904e47f6ba1f9589e38f2e454" dependencies = [ "semver 1.0.24", "serde", diff --git a/lib/vector-vrl/web-playground/Cargo.toml b/lib/vector-vrl/web-playground/Cargo.toml index 6d5cf3a9dd..386c7fa55c 100644 --- a/lib/vector-vrl/web-playground/Cargo.toml +++ b/lib/vector-vrl/web-playground/Cargo.toml @@ -18,4 +18,4 @@ vector-vrl-functions = { path = "../functions" } enrichment = { path = "../../enrichment" } [build-dependencies] -cargo-lock = "10.0.1" +cargo-lock = "10.1.0" From c0830fd72f7118540d807a1bd62802e9b8c7c6b5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 Jan 2025 16:01:34 +0000 Subject: [PATCH 006/658] chore(deps): Bump uuid from 1.11.1 to 1.12.0 (#22253) Bumps [uuid](https://github.com/uuid-rs/uuid) from 1.11.1 to 1.12.0. - [Release notes](https://github.com/uuid-rs/uuid/releases) - [Commits](https://github.com/uuid-rs/uuid/compare/1.11.1...1.12.0) --- updated-dependencies: - dependency-name: uuid dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2c28a258b0..1a087b49c4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10884,9 +10884,9 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "uuid" -version = "1.11.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b913a3b5fe84142e269d63cc62b64319ccaf89b748fc31fe025177f767a756c4" +checksum = "744018581f9a3454a9e15beb8a33b017183f1e7c0cd170232a2d1453b23a51c4" dependencies = [ "getrandom 0.2.15", "rand 0.8.5", diff --git a/Cargo.toml b/Cargo.toml index a23e1566be..6103732668 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -162,7 +162,7 @@ tokio = { version = "1.43.0", default-features = false, features = ["full"] } toml = { version = "0.8.19", default-features = false, features = ["display", "parse"] } tonic = { version = "0.11", default-features = false, features = ["transport", "codegen", "prost", "tls", "tls-roots", "gzip"] } tonic-build = { version = "0.11", default-features = false, features = ["transport", "prost"] } -uuid = { version = "1.11.1", features = ["v4", "v7", "serde"] } +uuid = { version = "1.12.0", features = ["v4", "v7", "serde"] } vector-lib = { path = "lib/vector-lib", default-features = false, features = ["vrl"] } vector-config = { path = "lib/vector-config" } vector-config-common = { path = "lib/vector-config-common" } From 91beca71452c3636c7969f4281cc51cb89338b35 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 Jan 2025 16:01:59 +0000 Subject: [PATCH 007/658] chore(deps): Bump ipnet from 2.10.1 to 2.11.0 (#22254) Bumps [ipnet](https://github.com/krisprice/ipnet) from 2.10.1 to 2.11.0. - [Release notes](https://github.com/krisprice/ipnet/releases) - [Changelog](https://github.com/krisprice/ipnet/blob/master/RELEASES.md) - [Commits](https://github.com/krisprice/ipnet/commits/2.11.0) --- updated-dependencies: - dependency-name: ipnet dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1a087b49c4..aa0bb76e98 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5082,9 +5082,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.10.1" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" +checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" dependencies = [ "serde", ] From 2378a21892adfd259ea8746c3c3ed4da717dce79 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 Jan 2025 16:35:25 +0000 Subject: [PATCH 008/658] chore(deps): Bump the aws group with 2 updates (#22245) Bumps the aws group with 2 updates: [aws-sdk-sns](https://github.com/awslabs/aws-sdk-rust) and [aws-sdk-secretsmanager](https://github.com/awslabs/aws-sdk-rust). Updates `aws-sdk-sns` from 1.3.0 to 1.6.0 - [Release notes](https://github.com/awslabs/aws-sdk-rust/releases) - [Commits](https://github.com/awslabs/aws-sdk-rust/commits) Updates `aws-sdk-secretsmanager` from 1.3.0 to 1.6.0 - [Release notes](https://github.com/awslabs/aws-sdk-rust/releases) - [Commits](https://github.com/awslabs/aws-sdk-rust/commits) --- updated-dependencies: - dependency-name: aws-sdk-sns dependency-type: direct:production update-type: version-update:semver-minor dependency-group: aws - dependency-name: aws-sdk-secretsmanager dependency-type: direct:production update-type: version-update:semver-minor dependency-group: aws ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 8 ++++---- Cargo.toml | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index aa0bb76e98..061775d1cc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1001,9 +1001,9 @@ dependencies = [ [[package]] name = "aws-sdk-secretsmanager" -version = "1.3.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faea24d86bcabc65048014abd3ee5763a5e8877f678d9cd688a12e086ebe2dbd" +checksum = "14d4c6f85ab6cf9b833b2d69af57c412990a1266be5ed02bcb6c4564522cd679" dependencies = [ "aws-credential-types", "aws-http", @@ -1024,9 +1024,9 @@ dependencies = [ [[package]] name = "aws-sdk-sns" -version = "1.3.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48bff824fe28888cc4ce6acb52ba8e5cd9f4f38fc5bd1bb40d916e3000e17b57" +checksum = "683e4cecb5fd7500b4fce010206e0dba496b669c5fe9bbe9962c900c02a35203" dependencies = [ "aws-credential-types", "aws-http", diff --git a/Cargo.toml b/Cargo.toml index 6103732668..7bdcfd9769 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -216,13 +216,13 @@ metrics-tracing-context.workspace = true # AWS - Official SDK aws-sdk-s3 = { version = "1.4.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } aws-sdk-sqs = { version = "1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } -aws-sdk-sns = { version = "1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } +aws-sdk-sns = { version = "1.6.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } aws-sdk-cloudwatch = { version = "1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } aws-sdk-cloudwatchlogs = { version = "1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } aws-sdk-elasticsearch = { version = "1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } aws-sdk-firehose = { version = "1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } aws-sdk-kinesis = { version = "1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } -aws-sdk-secretsmanager = { version = "1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } +aws-sdk-secretsmanager = { version = "1.6.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } # The sts crate is needed despite not being referred to anywhere in the code because we need to set the # `behavior-version-latest` feature. Without this we get a runtime panic when `auth.assume_role` authentication # is configured. From 698ef061349c6586aec94c597b038a92af657cf9 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 21 Jan 2025 15:00:10 -0500 Subject: [PATCH 009/658] chore(ci): add amqp sink scope --- .github/workflows/semantic.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/semantic.yml b/.github/workflows/semantic.yml index 8c15013b40..9ad7da120f 100644 --- a/.github/workflows/semantic.yml +++ b/.github/workflows/semantic.yml @@ -184,6 +184,7 @@ jobs: sample transform tag_cardinality_limit transform throttle transform + amqp sink apex sink aws_cloudwatch_logs sink aws_cloudwatch_metrics sink From dd63177195eb1a9bc1ff2a434f9bcb7e908d1503 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 Jan 2025 16:38:35 +0000 Subject: [PATCH 010/658] chore(deps): Bump convert_case from 0.6.0 to 0.7.1 (#22249) * chore(deps): Bump convert_case from 0.6.0 to 0.7.1 Bumps [convert_case](https://github.com/rutrum/convert-case) from 0.6.0 to 0.7.1. - [Commits](https://github.com/rutrum/convert-case/commits) --- updated-dependencies: - dependency-name: convert_case dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * use new variants * update licenses --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Pavlos Rontidis --- Cargo.lock | 11 ++++++++++- LICENSE-3rdparty.csv | 1 + lib/vector-config-common/Cargo.toml | 2 +- lib/vector-config-common/src/human_friendly.rs | 2 +- 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 061775d1cc..9b97a8d201 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2450,6 +2450,15 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "convert_case" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb402b8d4c85569410425650ce3eddc7d698ed96d39a73f941b08fb63082f1e7" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "cookie-factory" version = "0.3.2" @@ -11249,7 +11258,7 @@ dependencies = [ name = "vector-config-common" version = "0.1.0" dependencies = [ - "convert_case 0.6.0", + "convert_case 0.7.1", "darling 0.20.8", "proc-macro2 1.0.93", "quote 1.0.38", diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index fdf4e295ec..61dae05d49 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -148,6 +148,7 @@ const-oid,https://github.com/RustCrypto/formats/tree/master/const-oid,Apache-2.0 const_fn,https://github.com/taiki-e/const_fn,Apache-2.0 OR MIT,The const_fn Authors convert_case,https://github.com/rutrum/convert-case,MIT,David Purdum convert_case,https://github.com/rutrum/convert-case,MIT,Rutrum +convert_case,https://github.com/rutrum/convert-case,MIT,rutrum cookie-factory,https://github.com/rust-bakery/cookie-factory,MIT,"Geoffroy Couprie , Pierre Chifflier " core-foundation,https://github.com/servo/core-foundation-rs,MIT OR Apache-2.0,The Servo Project Developers core2,https://github.com/bbqsrc/core2,Apache-2.0 OR MIT,Brendan Molloy diff --git a/lib/vector-config-common/Cargo.toml b/lib/vector-config-common/Cargo.toml index 4813d3885d..cce22a047f 100644 --- a/lib/vector-config-common/Cargo.toml +++ b/lib/vector-config-common/Cargo.toml @@ -5,7 +5,7 @@ edition = "2021" license = "MPL-2.0" [dependencies] -convert_case = { version = "0.6", default-features = false } +convert_case = { version = "0.7", default-features = false } darling = { version = "0.20", default-features = false, features = ["suggestions"] } proc-macro2 = { version = "1.0", default-features = false } serde.workspace = true diff --git a/lib/vector-config-common/src/human_friendly.rs b/lib/vector-config-common/src/human_friendly.rs index a341168cca..70894d9a4b 100644 --- a/lib/vector-config-common/src/human_friendly.rs +++ b/lib/vector-config-common/src/human_friendly.rs @@ -70,7 +70,7 @@ pub fn generate_human_friendly_string(input: &str) -> String { // respectively. let converter = Converter::new() .to_case(Case::Title) - .remove_boundaries(&[Boundary::LowerDigit, Boundary::UpperDigit]); + .remove_boundaries(&[Boundary::LOWER_DIGIT, Boundary::UPPER_DIGIT]); let normalized = converter.convert(input); let replaced_segments = normalized From 26421c7f328c121eed4c9ea0a809b20660d36fcb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 Jan 2025 16:39:51 +0000 Subject: [PATCH 011/658] chore(deps): Bump notify from 7.0.0 to 8.0.0 (#22173) Bumps [notify](https://github.com/notify-rs/notify) from 7.0.0 to 8.0.0. - [Release notes](https://github.com/notify-rs/notify/releases) - [Changelog](https://github.com/notify-rs/notify/blob/main/CHANGELOG.md) - [Commits](https://github.com/notify-rs/notify/compare/notify-7.0.0...notify-8.0.0) --- updated-dependencies: - dependency-name: notify dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 47 ++++++++++++++++++++++------------------------- Cargo.toml | 2 +- 2 files changed, 23 insertions(+), 26 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9b97a8d201..9fd07e03e0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1645,9 +1645,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.6.0" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36" [[package]] name = "bitmask-enum" @@ -2625,7 +2625,7 @@ version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "crossterm_winapi", "futures-core", "mio", @@ -5001,11 +5001,11 @@ dependencies = [ [[package]] name = "inotify" -version = "0.10.2" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdd168d97690d0b8c412d6b6c10360277f4d7ee495c5d0d5d5fe0854923255cc" +checksum = "f37dccff2791ab604f9babef0ba14fbe0be30bd368dc541e2b08d07c8aa908f3" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.8.0", "inotify-sys", "libc", ] @@ -6216,7 +6216,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "cfg-if", "cfg_aliases 0.2.1", "libc", @@ -6280,11 +6280,11 @@ checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21" [[package]] name = "notify" -version = "7.0.0" +version = "8.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c533b4c39709f9ba5005d8002048266593c1cfaf3c5f0739d5b8ab0c6c504009" +checksum = "2fee8403b3d66ac7b26aee6e40a897d85dc5ce26f44da36b8b73e987cc52e943" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "filetime", "fsevent-sys", "inotify", @@ -6294,17 +6294,14 @@ dependencies = [ "mio", "notify-types", "walkdir", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] name = "notify-types" -version = "1.0.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7393c226621f817964ffb3dc5704f9509e107a8b024b489cc2c1b217378785df" -dependencies = [ - "instant", -] +checksum = "5e0826a989adedc2a244799e823aece04662b66609d96af8dff7ac6df9a8925d" [[package]] name = "ntapi" @@ -6669,7 +6666,7 @@ version = "0.10.68" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6174bc48f102d208783c2c84bf931bb75927a617866870de8a4ea85597f871f5" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "cfg-if", "foreign-types", "libc", @@ -7465,7 +7462,7 @@ checksum = "b4c2511913b88df1637da85cc8d96ec8e43a3f8bb8ccb71ee1ac240d6f3df58d" dependencies = [ "bit-set 0.5.3", "bit-vec 0.6.3", - "bitflags 2.6.0", + "bitflags 2.8.0", "lazy_static", "num-traits", "rand 0.8.5", @@ -7920,7 +7917,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eabd94c2f37801c20583fc49dd5cd6b0ba68c716787c2dd6ed18571e1e63117b" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "cassowary", "compact_str", "crossterm", @@ -7941,7 +7938,7 @@ version = "11.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d86a7c4638d42c44551f4791a20e687dbb4c3de1f33c43dd71e355cd429def1" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", ] [[package]] @@ -8519,7 +8516,7 @@ version = "0.38.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "99e4ea3e1cdc4b559b8e5650f9c8e5998e3e5c1343b4eaf034565f32318d63c0" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "errno", "libc", "linux-raw-sys 0.4.14", @@ -8674,7 +8671,7 @@ version = "15.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2ee1e066dc922e513bda599c6ccb5f3bb2b0ea5870a579448f2622993f0a9a2f" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "cfg-if", "clipboard-win", "libc", @@ -10294,7 +10291,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61c5bb1d698276a2443e5ecfabc1008bf15a36c12e6a7176e7bf089ea9131140" dependencies = [ "async-compression", - "bitflags 2.6.0", + "bitflags 2.8.0", "bytes 1.9.0", "futures-core", "futures-util", @@ -10316,7 +10313,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5" dependencies = [ "base64 0.21.7", - "bitflags 2.6.0", + "bitflags 2.8.0", "bytes 1.9.0", "http 1.1.0", "http-body 1.0.0", @@ -11998,7 +11995,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d24d6bcc7f734a4091ecf8d7a64c5f7d7066f45585c1861eba06449909609c8a" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "widestring 1.0.2", "windows-sys 0.52.0", ] diff --git a/Cargo.toml b/Cargo.toml index 7bdcfd9769..9c90d47a8f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -342,7 +342,7 @@ mongodb = { version = "2.8.2", default-features = false, features = ["tokio-runt async-nats = { version = "0.38.0", default-features = false, features = ["ring"], optional = true } nkeys = { version = "0.4.4", default-features = false, optional = true } nom = { version = "7.1.3", default-features = false, optional = true } -notify = { version = "7.0.0", default-features = false, features = ["macos_fsevent"] } +notify = { version = "8.0.0", default-features = false, features = ["macos_fsevent"] } openssl = { version = "0.10.68", default-features = false, features = ["vendored"] } openssl-probe = { version = "0.1.5", default-features = false } ordered-float = { version = "4.6.0", default-features = false } From 7eebd9471d3cc11b54717253f44ee8487a36bd1e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 Jan 2025 17:20:32 +0000 Subject: [PATCH 012/658] chore(deps): Bump the patches group with 15 updates (#22244) * chore(deps): Bump the patches group with 15 updates Bumps the patches group with 15 updates: | Package | From | To | | --- | --- | --- | | [aws-sigv4](https://github.com/smithy-lang/smithy-rs) | `1.2.6` | `1.2.7` | | [aws-smithy-http](https://github.com/smithy-lang/smithy-rs) | `0.60.11` | `0.60.12` | | [aws-smithy-types](https://github.com/smithy-lang/smithy-rs) | `1.2.11` | `1.2.12` | | [aws-smithy-runtime](https://github.com/smithy-lang/smithy-rs) | `1.7.4` | `1.7.6` | | [aws-smithy-async](https://github.com/smithy-lang/smithy-rs) | `1.2.3` | `1.2.4` | | [serde_json](https://github.com/serde-rs/json) | `1.0.135` | `1.0.137` | | [prost-reflect](https://github.com/andrewhickman/prost-reflect) | `0.14.3` | `0.14.4` | | [chrono-tz](https://github.com/chronotope/chrono-tz) | `0.10.0` | `0.10.1` | | [indexmap](https://github.com/indexmap-rs/indexmap) | `2.7.0` | `2.7.1` | | [listenfd](https://github.com/mitsuhiko/listenfd) | `1.0.1` | `1.0.2` | | [semver](https://github.com/dtolnay/semver) | `1.0.24` | `1.0.25` | | [strip-ansi-escapes](https://github.com/luser/strip-ansi-escapes) | `0.2.0` | `0.2.1` | | [similar-asserts](https://github.com/mitsuhiko/similar-asserts) | `1.6.0` | `1.6.1` | | [log](https://github.com/rust-lang/log) | `0.4.22` | `0.4.25` | | [enumflags2](https://github.com/meithecatte/enumflags2) | `0.7.10` | `0.7.11` | Updates `aws-sigv4` from 1.2.6 to 1.2.7 - [Release notes](https://github.com/smithy-lang/smithy-rs/releases) - [Changelog](https://github.com/smithy-lang/smithy-rs/blob/main/CHANGELOG.md) - [Commits](https://github.com/smithy-lang/smithy-rs/commits) Updates `aws-smithy-http` from 0.60.11 to 0.60.12 - [Release notes](https://github.com/smithy-lang/smithy-rs/releases) - [Changelog](https://github.com/smithy-lang/smithy-rs/blob/main/CHANGELOG.md) - [Commits](https://github.com/smithy-lang/smithy-rs/commits) Updates `aws-smithy-types` from 1.2.11 to 1.2.12 - [Release notes](https://github.com/smithy-lang/smithy-rs/releases) - [Changelog](https://github.com/smithy-lang/smithy-rs/blob/main/CHANGELOG.md) - [Commits](https://github.com/smithy-lang/smithy-rs/commits) Updates `aws-smithy-runtime` from 1.7.4 to 1.7.6 - [Release notes](https://github.com/smithy-lang/smithy-rs/releases) - [Changelog](https://github.com/smithy-lang/smithy-rs/blob/main/CHANGELOG.md) - [Commits](https://github.com/smithy-lang/smithy-rs/commits) Updates `aws-smithy-async` from 1.2.3 to 1.2.4 - [Release notes](https://github.com/smithy-lang/smithy-rs/releases) - [Changelog](https://github.com/smithy-lang/smithy-rs/blob/main/CHANGELOG.md) - [Commits](https://github.com/smithy-lang/smithy-rs/commits) Updates `serde_json` from 1.0.135 to 1.0.137 - [Release notes](https://github.com/serde-rs/json/releases) - [Commits](https://github.com/serde-rs/json/compare/v1.0.135...v1.0.137) Updates `prost-reflect` from 0.14.3 to 0.14.4 - [Changelog](https://github.com/andrewhickman/prost-reflect/blob/main/CHANGELOG.md) - [Commits](https://github.com/andrewhickman/prost-reflect/compare/0.14.3...0.14.4) Updates `chrono-tz` from 0.10.0 to 0.10.1 - [Release notes](https://github.com/chronotope/chrono-tz/releases) - [Commits](https://github.com/chronotope/chrono-tz/commits) Updates `indexmap` from 2.7.0 to 2.7.1 - [Changelog](https://github.com/indexmap-rs/indexmap/blob/master/RELEASES.md) - [Commits](https://github.com/indexmap-rs/indexmap/compare/2.7.0...2.7.1) Updates `listenfd` from 1.0.1 to 1.0.2 - [Changelog](https://github.com/mitsuhiko/listenfd/blob/master/CHANGELOG.md) - [Commits](https://github.com/mitsuhiko/listenfd/compare/1.0.1...1.0.2) Updates `semver` from 1.0.24 to 1.0.25 - [Release notes](https://github.com/dtolnay/semver/releases) - [Commits](https://github.com/dtolnay/semver/compare/1.0.24...1.0.25) Updates `strip-ansi-escapes` from 0.2.0 to 0.2.1 - [Commits](https://github.com/luser/strip-ansi-escapes/commits) Updates `similar-asserts` from 1.6.0 to 1.6.1 - [Changelog](https://github.com/mitsuhiko/similar-asserts/blob/main/CHANGELOG.md) - [Commits](https://github.com/mitsuhiko/similar-asserts/compare/1.6.0...1.6.1) Updates `log` from 0.4.22 to 0.4.25 - [Release notes](https://github.com/rust-lang/log/releases) - [Changelog](https://github.com/rust-lang/log/blob/master/CHANGELOG.md) - [Commits](https://github.com/rust-lang/log/compare/0.4.22...0.4.25) Updates `enumflags2` from 0.7.10 to 0.7.11 - [Release notes](https://github.com/meithecatte/enumflags2/releases) - [Commits](https://github.com/meithecatte/enumflags2/compare/v0.7.10...v0.7.11) --- updated-dependencies: - dependency-name: aws-sigv4 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: aws-smithy-http dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: aws-smithy-types dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: aws-smithy-runtime dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: aws-smithy-async dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: serde_json dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: prost-reflect dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: chrono-tz dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: indexmap dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: listenfd dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: semver dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: strip-ansi-escapes dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: similar-asserts dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: log dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: enumflags2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches ... Signed-off-by: dependabot[bot] * dd-rust-license-tool write --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Pavlos Rontidis --- Cargo.lock | 135 +++++++++++++++++-------------------- Cargo.toml | 22 +++--- LICENSE-3rdparty.csv | 3 +- lib/codecs/Cargo.toml | 2 +- lib/file-source/Cargo.toml | 4 +- lib/vector-core/Cargo.toml | 4 +- vdev/Cargo.toml | 2 +- 7 files changed, 80 insertions(+), 92 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9fd07e03e0..37009f736e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -492,7 +492,7 @@ dependencies = [ "fnv", "futures-util", "http 1.1.0", - "indexmap 2.7.0", + "indexmap 2.7.1", "mime", "multer", "num-traits", @@ -542,7 +542,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69117c43c01d81a69890a9f5dd6235f2f027ca8d1ec62d6d3c5e01ca0edb4f2b" dependencies = [ "bytes 1.9.0", - "indexmap 2.7.0", + "indexmap 2.7.1", "serde", "serde_json", ] @@ -1136,9 +1136,9 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.2.6" +version = "1.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d3820e0c08d0737872ff3c7c1f21ebbb6693d832312d6152bf18ef50a5471c2" +checksum = "690118821e46967b3c4501d67d7d52dd75106a9c54cf36cefa1985cedbe94e05" dependencies = [ "aws-credential-types", "aws-smithy-eventstream", @@ -1160,9 +1160,9 @@ dependencies = [ [[package]] name = "aws-smithy-async" -version = "1.2.3" +version = "1.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "427cb637d15d63d6f9aae26358e1c9a9c09d5aa490d64b09354c8217cfef0f28" +checksum = "fa59d1327d8b5053c54bf2eaae63bf629ba9e904434d0835a28ed3c0ed0a614e" dependencies = [ "futures-util", "pin-project-lite", @@ -1192,9 +1192,9 @@ dependencies = [ [[package]] name = "aws-smithy-eventstream" -version = "0.60.5" +version = "0.60.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cef7d0a272725f87e51ba2bf89f8c21e4df61b9e49ae1ac367a6d69916ef7c90" +checksum = "8b18559a41e0c909b77625adf2b8c50de480a8041e5e4a3f5f7d177db70abc5a" dependencies = [ "aws-smithy-types", "bytes 1.9.0", @@ -1203,9 +1203,9 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.60.11" +version = "0.60.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c8bc3e8fdc6b8d07d976e301c02fe553f72a39b7a9fea820e023268467d7ab6" +checksum = "7809c27ad8da6a6a68c454e651d4962479e81472aa19ae99e59f9aba1f9713cc" dependencies = [ "aws-smithy-eventstream", "aws-smithy-runtime-api", @@ -1243,9 +1243,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.7.4" +version = "1.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f20685047ca9d6f17b994a07f629c813f08b5bce65523e47124879e60103d45" +checksum = "a05dd41a70fc74051758ee75b5c4db2c0ca070ed9229c3df50e9475cda1cb985" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -1287,9 +1287,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.2.11" +version = "1.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38ddc9bd6c28aeb303477170ddd183760a956a03e083b3902a990238a7e3792d" +checksum = "a28f6feb647fb5e0d5b50f0472c19a7db9462b74e2fec01bb0b44eedcc834e97" dependencies = [ "base64-simd", "bytes 1.9.0", @@ -1935,7 +1935,7 @@ version = "10.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c06acb4f71407ba205a07cb453211e0e6a67b21904e47f6ba1f9589e38f2e454" dependencies = [ - "semver 1.0.24", + "semver 1.0.25", "serde", "toml", "url", @@ -2065,9 +2065,9 @@ dependencies = [ [[package]] name = "chrono-tz" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd6dd8046d00723a59a2f8c5f295c515b9bb9a331ee4f8f3d4dd49e428acd3b6" +checksum = "9c6ac4f2c0bf0f44e9161aec9675e1050aa4a530663c4a9e37e108fa948bca9f" dependencies = [ "chrono", "chrono-tz-build", @@ -3362,18 +3362,18 @@ dependencies = [ [[package]] name = "enumflags2" -version = "0.7.10" +version = "0.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d232db7f5956f3f14313dc2f87985c58bd2c695ce124c8cdd984e08e15ac133d" +checksum = "ba2f4b465f5318854c6f8dd686ede6c0a9dc67d4b1ac241cf0eb51521a309147" dependencies = [ "enumflags2_derive", ] [[package]] name = "enumflags2_derive" -version = "0.7.10" +version = "0.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de0d48a183585823424a4ce1aa132d174a6a81bd540895822eb4c8373a8e49e8" +checksum = "fc4caf64a58d7a6d65ab00639b046ff54399a39f5f2554728895ace4b297cd79" dependencies = [ "proc-macro2 1.0.93", "quote 1.0.38", @@ -3594,7 +3594,7 @@ dependencies = [ "flate2", "futures 0.3.31", "glob", - "indexmap 2.7.0", + "indexmap 2.7.1", "libc", "quickcheck", "scan_fmt", @@ -4089,7 +4089,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.9", - "indexmap 2.7.0", + "indexmap 2.7.1", "slab", "tokio", "tokio-util", @@ -4108,7 +4108,7 @@ dependencies = [ "futures-core", "futures-sink", "http 1.1.0", - "indexmap 2.7.0", + "indexmap 2.7.1", "slab", "tokio", "tokio-util", @@ -4951,9 +4951,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.7.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62f822373a4fe84d4bb149bf54e584a7f4abec90e072ed49cda0edea5b95471f" +checksum = "8c9c992b02b5b4c94ea26e32fe5bccb7aa7d9f390ab5c1221ff895bc7ea8b652" dependencies = [ "equivalent", "hashbrown 0.15.2", @@ -5594,9 +5594,9 @@ checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" [[package]] name = "listenfd" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0500463acd96259d219abb05dc57e5a076ef04b2db9a2112846929b5f174c96" +checksum = "b87bc54a4629b4294d0b3ef041b64c40c611097a677d9dc07b2c67739fe39dba" dependencies = [ "libc", "uuid", @@ -5627,9 +5627,9 @@ checksum = "3a69c0481fc2424cb55795de7da41add33372ea75a94f9b6588ab6a2826dfebc" [[package]] name = "log" -version = "0.4.22" +version = "0.4.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f" [[package]] name = "loki-logproto" @@ -5839,7 +5839,7 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1ada651cd6bdffe01e5f35067df53491f1fe853d2b154008ca2bd30b3d3fcf6" dependencies = [ - "indexmap 2.7.0", + "indexmap 2.7.1", "itoa", "lockfree-object-pool", "metrics", @@ -5860,7 +5860,7 @@ dependencies = [ "crossbeam-epoch", "crossbeam-utils", "hashbrown 0.15.2", - "indexmap 2.7.0", + "indexmap 2.7.1", "metrics", "ordered-float 4.6.0", "quanta", @@ -6104,7 +6104,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17ebbe97acce52d06aebed4cd4a87c0941f4b2519b59b82b4feb5bd0ce003dfd" dependencies = [ - "indexmap 2.7.0", + "indexmap 2.7.1", "itertools 0.13.0", "ndarray", "noisy_float", @@ -6968,7 +6968,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" dependencies = [ "fixedbitset", - "indexmap 2.7.0", + "indexmap 2.7.1", ] [[package]] @@ -7445,7 +7445,7 @@ dependencies = [ name = "prometheus-parser" version = "0.1.0" dependencies = [ - "indexmap 2.7.0", + "indexmap 2.7.1", "nom", "prost 0.12.6", "prost-build 0.12.6", @@ -7599,9 +7599,9 @@ dependencies = [ [[package]] name = "prost-reflect" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20ae544fca2892fd4b7e9ff26cba1090cedf1d4d95c2aded1af15d2f93f270b8" +checksum = "bc9647f03b808b79abca8408b1609be9887ba90453c940d00332a60eeb6f5748" dependencies = [ "base64 0.22.1", "once_cell", @@ -8483,7 +8483,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" dependencies = [ - "semver 1.0.24", + "semver 1.0.25", ] [[package]] @@ -8825,9 +8825,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.24" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cb6eb87a131f756572d7fb904f6e7b68633f09cca868c5df1c4b8d1a694bbba" +checksum = "f79dfe2d285b0488816f30e700a7438c5a73d816b5b7d3ac72fbc48b0d185e03" dependencies = [ "serde", ] @@ -8910,11 +8910,11 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.135" +version = "1.0.137" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b0d7ba2887406110130a978386c4e1befb98c674b4fba677954e4db976630d9" +checksum = "930cfb6e6abf99298aaad7d29abbef7a9999a9a8806a40088f55f0dcec03146b" dependencies = [ - "indexmap 2.7.0", + "indexmap 2.7.1", "itoa", "memchr", "ryu", @@ -9012,7 +9012,7 @@ dependencies = [ "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.7.0", + "indexmap 2.7.1", "serde", "serde_derive", "serde_json", @@ -9062,7 +9062,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.7.0", + "indexmap 2.7.1", "itoa", "ryu", "serde", @@ -9201,9 +9201,9 @@ dependencies = [ [[package]] name = "similar-asserts" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfe85670573cd6f0fa97940f26e7e6601213c3b0555246c24234131f88c5709e" +checksum = "9f08357795f0d604ea7d7130f22c74b03838c959bdb14adde3142aab4d18a293" dependencies = [ "console", "similar", @@ -9438,9 +9438,9 @@ dependencies = [ [[package]] name = "strip-ansi-escapes" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55ff8ef943b384c414f54aefa961dd2bd853add74ec75e7ac74cf91dba62bcfa" +checksum = "2a8f8038e7e7969abb3f1b7c2a811225e9296da208539e0f79c5251d6cac0025" dependencies = [ "vte", ] @@ -10146,7 +10146,7 @@ version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap 2.7.0", + "indexmap 2.7.1", "toml_datetime", "winnow 0.5.18", ] @@ -10157,7 +10157,7 @@ version = "0.20.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70f427fce4d84c72b5b732388bf4a9f4531b53f74e2887e3ecb2481f68f66d81" dependencies = [ - "indexmap 2.7.0", + "indexmap 2.7.1", "toml_datetime", "winnow 0.5.18", ] @@ -10168,7 +10168,7 @@ version = "0.22.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "583c44c02ad26b0c3f3066fe629275e50627026c51ac2e595cca4c230ce1ce1d" dependencies = [ - "indexmap 2.7.0", + "indexmap 2.7.1", "serde", "serde_spanned", "toml_datetime", @@ -10926,7 +10926,7 @@ dependencies = [ "dunce", "glob", "hex", - "indexmap 2.7.0", + "indexmap 2.7.1", "indicatif", "itertools 0.14.0", "log", @@ -11027,7 +11027,7 @@ dependencies = [ "hyper 0.14.28", "hyper-openssl 0.9.2", "hyper-proxy", - "indexmap 2.7.0", + "indexmap 2.7.1", "indoc", "inventory", "ipnet", @@ -11086,7 +11086,7 @@ dependencies = [ "rstest", "rumqttc", "seahash", - "semver 1.0.24", + "semver 1.0.25", "serde", "serde-toml-merge", "serde_bytes", @@ -11212,7 +11212,7 @@ dependencies = [ "crossbeam-utils", "derivative", "futures 0.3.31", - "indexmap 2.7.0", + "indexmap 2.7.1", "metrics", "paste", "pin-project", @@ -11235,7 +11235,7 @@ dependencies = [ "chrono-tz", "encoding_rs", "http 0.2.9", - "indexmap 2.7.0", + "indexmap 2.7.1", "inventory", "no-proxy", "num-traits", @@ -11302,7 +11302,7 @@ dependencies = [ "headers", "http 0.2.9", "hyper-proxy", - "indexmap 2.7.0", + "indexmap 2.7.1", "ipnet", "metrics", "metrics-tracing-context", @@ -11539,7 +11539,7 @@ dependencies = [ "humantime", "iana-time-zone", "idna 1.0.3", - "indexmap 2.7.0", + "indexmap 2.7.1", "indoc", "influxdb-line-protocol", "itertools 0.14.0", @@ -11605,22 +11605,11 @@ checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" [[package]] name = "vte" -version = "0.11.1" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5022b5fbf9407086c180e9557be968742d839e68346af7792b8592489732197" +checksum = "231fdcd7ef3037e8330d8e17e61011a2c244126acc0a982f4040ac3f9f0bc077" dependencies = [ - "utf8parse", - "vte_generate_state_changes", -] - -[[package]] -name = "vte_generate_state_changes" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d257817081c7dffcdbab24b9e62d2def62e2ff7d00b1c20062551e6cccc145ff" -dependencies = [ - "proc-macro2 1.0.93", - "quote 1.0.38", + "memchr", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 9c90d47a8f..9dfa65d5b8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -137,13 +137,13 @@ members = [ anyhow = "1.0.95" cfg-if = { version = "1.0.0", default-features = false } chrono = { version = "0.4.39", default-features = false, features = ["clock", "serde"] } -chrono-tz = { version = "0.10.0", default-features = false, features = ["serde"] } +chrono-tz = { version = "0.10.1", default-features = false, features = ["serde"] } clap = { version = "4.5.26", default-features = false, features = ["derive", "error-context", "env", "help", "std", "string", "usage", "wrap_help"] } flate2 = { version = "1.0.35", default-features = false, features = ["default"] } futures = { version = "0.3.31", default-features = false, features = ["compat", "io-compat", "std"], package = "futures" } glob = { version = "0.3.2", default-features = false } hickory-proto = { version = "0.24.2", default-features = false, features = ["dnssec"] } -indexmap = { version = "2.7.0", default-features = false, features = ["serde", "std"] } +indexmap = { version = "2.7.1", default-features = false, features = ["serde", "std"] } metrics = "0.24.1" metrics-tracing-context = { version = "0.17.0", default-features = false } metrics-util = { version = "0.18.0", default-features = false, features = ["registry"] } @@ -155,7 +155,7 @@ prost-build = { version = "0.12", default-features = false } prost-reflect = { version = "0.14", features = ["serde"], default-features = false } prost-types = { version = "0.12", default-features = false } rand = { version = "0.8.5", default-features = false, features = ["small_rng"] } -serde_json = { version = "1.0.135", default-features = false, features = ["raw_value", "std"] } +serde_json = { version = "1.0.137", default-features = false, features = ["raw_value", "std"] } serde = { version = "1.0.217", default-features = false, features = ["alloc", "derive", "rc"] } snafu = { version = "0.7.5", default-features = false, features = ["futures", "std"] } tokio = { version = "1.43.0", default-features = false, features = ["full"] } @@ -228,14 +228,14 @@ aws-sdk-secretsmanager = { version = "1.6.0", default-features = false, features # is configured. aws-sdk-sts = { version = "1.3.1", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } aws-types = { version = "1.3.3", default-features = false, optional = true } -aws-sigv4 = { version = "1.2.6", default-features = false, features = ["sign-http"], optional = true } +aws-sigv4 = { version = "1.2.7", default-features = false, features = ["sign-http"], optional = true } aws-config = { version = "~1.0.1", default-features = false, features = ["behavior-version-latest", "credentials-process", "sso", "rt-tokio"], optional = true } aws-credential-types = { version = "1.2.1", default-features = false, features = ["hardcoded-credentials"], optional = true } aws-smithy-http = { version = "0.60", default-features = false, features = ["event-stream", "rt-tokio"], optional = true } aws-smithy-types = { version = "1.2.11", default-features = false, features = ["rt-tokio"], optional = true } aws-smithy-runtime-api = { version = "1.7.3", default-features = false, optional = true } -aws-smithy-runtime = { version = "1.7.4", default-features = false, features = ["client", "connector-hyper-0-14-x", "rt-tokio"], optional = true } -aws-smithy-async = { version = "1.2.3", default-features = false, features = ["rt-tokio"], optional = true } +aws-smithy-runtime = { version = "1.7.6", default-features = false, features = ["client", "connector-hyper-0-14-x", "rt-tokio"], optional = true } +aws-smithy-async = { version = "1.2.4", default-features = false, features = ["rt-tokio"], optional = true } # Azure azure_core = { version = "0.17", default-features = false, features = ["enable_reqwest"], optional = true } @@ -334,7 +334,7 @@ ipnet = { version = "2", default-features = false, optional = true, features = [ itertools = { version = "0.14.0", default-features = false, optional = false, features = ["use_alloc"] } k8s-openapi = { version = "0.22.0", default-features = false, features = ["v1_26"], optional = true } kube = { version = "0.93.0", default-features = false, features = ["client", "openssl-tls", "runtime"], optional = true } -listenfd = { version = "1.0.1", default-features = false, optional = true } +listenfd = { version = "1.0.2", default-features = false, optional = true } lru = { version = "0.12.5", default-features = false, optional = true } maxminddb = { version = "0.24.0", default-features = false, optional = true } md-5 = { version = "0.10", default-features = false, optional = true } @@ -358,12 +358,12 @@ regex = { version = "1.11.1", default-features = false, features = ["std", "perf roaring = { version = "0.10.10", default-features = false, features = ["std"], optional = true } rumqttc = { version = "0.24.0", default-features = false, features = ["use-rustls"], optional = true } seahash = { version = "4.1.0", default-features = false } -semver = { version = "1.0.24", default-features = false, features = ["serde", "std"], optional = true } +semver = { version = "1.0.25", default-features = false, features = ["serde", "std"], optional = true } smallvec = { version = "1", default-features = false, features = ["union", "serde"] } snap = { version = "1.1.1", default-features = false } socket2 = { version = "0.5.8", default-features = false } stream-cancel = { version = "0.8.2", default-features = false } -strip-ansi-escapes = { version = "0.2.0", default-features = false } +strip-ansi-escapes = { version = "0.2.1", default-features = false } syslog = { version = "6.1.1", default-features = false, optional = true } tikv-jemallocator = { version = "0.6.0", default-features = false, features = ["unprefixed_malloc_on_supported_platforms"], optional = true } tokio-postgres = { version = "0.7.12", default-features = false, features = ["runtime", "with-chrono-0_4"], optional = true } @@ -407,7 +407,7 @@ openssl-src = { version = "300", default-features = false, features = ["force-en [dev-dependencies] approx = "0.5.1" assert_cmd = { version = "2.0.16", default-features = false } -aws-smithy-runtime = { version = "1.7.4", default-features = false, features = ["tls-rustls"] } +aws-smithy-runtime = { version = "1.7.6", default-features = false, features = ["tls-rustls"] } azure_core = { version = "0.17", default-features = false, features = ["enable_reqwest", "azurite_workaround"] } azure_identity = { version = "0.17", default-features = false, features = ["enable_reqwest"] } azure_storage_blobs = { version = "0.17", default-features = false, features = ["azurite_workaround"] } @@ -416,7 +416,7 @@ base64 = "0.22.1" criterion = { version = "0.5.1", features = ["html_reports", "async_tokio"] } itertools = { version = "0.14.0", default-features = false, features = ["use_alloc"] } libc = "0.2.167" -similar-asserts = "1.6.0" +similar-asserts = "1.6.1" proptest.workspace = true quickcheck = "1.0.3" reqwest = { version = "0.11", features = ["json"] } diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 61dae05d49..9e634db470 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -352,7 +352,7 @@ libz-sys,https://github.com/rust-lang/libz-sys,MIT OR Apache-2.0,"Alex Crichton linked-hash-map,https://github.com/contain-rs/linked-hash-map,MIT OR Apache-2.0,"Stepan Koltsov , Andrew Paseltiner " linked_hash_set,https://github.com/alexheretic/linked-hash-set,Apache-2.0,Alex Butler linux-raw-sys,https://github.com/sunfishcode/linux-raw-sys,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,Dan Gohman -listenfd,https://github.com/mitsuhiko/rust-listenfd,Apache-2.0,Armin Ronacher +listenfd,https://github.com/mitsuhiko/listenfd,Apache-2.0,Armin Ronacher litemap,https://github.com/unicode-org/icu4x,Unicode-3.0,The ICU4X Project Developers lockfree-object-pool,https://github.com/EVaillant/lockfree-object-pool,BSL-1.0,Etienne Vaillant log,https://github.com/rust-lang/log,MIT OR Apache-2.0,The Rust Project Developers @@ -689,7 +689,6 @@ void,https://github.com/reem/rust-void,MIT,Jonathan Reem vsimd,https://github.com/Nugine/simd,MIT,The vsimd Authors vte,https://github.com/alacritty/vte,Apache-2.0 OR MIT,"Joe Wilm , Christian Duerr " -vte_generate_state_changes,https://github.com/jwilm/vte,Apache-2.0 OR MIT,Christian Duerr wait-timeout,https://github.com/alexcrichton/wait-timeout,MIT OR Apache-2.0,Alex Crichton waker-fn,https://github.com/smol-rs/waker-fn,Apache-2.0 OR MIT,Stjepan Glavina walkdir,https://github.com/BurntSushi/walkdir,Unlicense OR MIT,Andrew Gallant diff --git a/lib/codecs/Cargo.toml b/lib/codecs/Cargo.toml index 992c162e89..722400ac59 100644 --- a/lib/codecs/Cargo.toml +++ b/lib/codecs/Cargo.toml @@ -44,7 +44,7 @@ vector-core = { path = "../vector-core", default-features = false, features = [" futures.workspace = true indoc = { version = "2", default-features = false } tokio = { version = "1", features = ["test-util"] } -similar-asserts = "1.6.0" +similar-asserts = "1.6.1" vector-core = { path = "../vector-core", default-features = false, features = ["vrl", "test"] } rstest = "0.24.0" tracing-test = "0.2.5" diff --git a/lib/file-source/Cargo.toml b/lib/file-source/Cargo.toml index ffc102f25f..3c92d265bb 100644 --- a/lib/file-source/Cargo.toml +++ b/lib/file-source/Cargo.toml @@ -38,7 +38,7 @@ default-features = false features = [] [dependencies.indexmap] -version = "2.7.0" +version = "2.7.1" default-features = false features = ["serde"] @@ -76,7 +76,7 @@ features = ["full"] criterion = "0.5" quickcheck = "1" tempfile = "3.15.0" -similar-asserts = "1.6.0" +similar-asserts = "1.6.1" [[bench]] name = "buffer" diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml index 0b0613b4da..3c5c4d811c 100644 --- a/lib/vector-core/Cargo.toml +++ b/lib/vector-core/Cargo.toml @@ -17,7 +17,7 @@ chrono-tz.workspace = true crossbeam-utils = { version = "0.8.21", default-features = false } derivative = { version = "2.2.0", default-features = false } dyn-clone = { version = "1.0.17", default-features = false } -enumflags2 = { version = "0.7.10", default-features = false } +enumflags2 = { version = "0.7.11", default-features = false } float_eq = { version = "1.0", default-features = false } futures.workspace = true futures-util = { version = "0.3.29", default-features = false, features = ["std"] } @@ -82,7 +82,7 @@ env-test-util = "1.0.1" quickcheck = "1" quickcheck_macros = "1" proptest = "1.5" -similar-asserts = "1.6.0" +similar-asserts = "1.6.1" tokio-test = "0.4.4" toml.workspace = true ndarray = "0.16.1" diff --git a/vdev/Cargo.toml b/vdev/Cargo.toml index eda13a0ae3..16c773d293 100644 --- a/vdev/Cargo.toml +++ b/vdev/Cargo.toml @@ -22,7 +22,7 @@ hex = "0.4.3" indexmap.workspace = true indicatif = { version = "0.17.9", features = ["improved_unicode"] } itertools = "0.14.0" -log = "0.4.22" +log = "0.4.25" # watch https://github.com/epage/anstyle for official interop with Clap owo-colors = { version = "4.1.0", features = ["supports-colors"] } paste = "1.0.15" From b03b3b34ce55924a3fbeb88e943da581123a072c Mon Sep 17 00:00:00 2001 From: Geoffrey Oxberry Date: Tue, 21 Jan 2025 10:49:31 -0800 Subject: [PATCH 013/658] chore(ci): smp version: 0.19.3 -> 0.20.1 (#22266) To improve logging and help with debugging, this commit updates Vector's CI to use the latest version of `smp`, v0.20.1. Signed-off-by: Geoffrey M. Oxberry --- .github/workflows/regression.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 0837242a55..1f425d11ff 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -105,7 +105,7 @@ jobs: - name: Set SMP version id: experimental-meta run: | - export SMP_CRATE_VERSION="0.19.3" + export SMP_CRATE_VERSION="0.20.1" echo "smp crate version: ${SMP_CRATE_VERSION}" echo "SMP_CRATE_VERSION=${SMP_CRATE_VERSION}" >> $GITHUB_OUTPUT From 604a51bf7b46e3aa27ebdd278dcbeb5022629e40 Mon Sep 17 00:00:00 2001 From: "Brian L. Troutwine" Date: Tue, 21 Jan 2025 16:17:17 -0800 Subject: [PATCH 014/658] chore(ci): Update lading to 0.25.4 (#22271) Update lading to 0.25.4 This release of lading contains a fix for splunk-hec tests that do not transmit an ackId. Signed-off-by: Brian L. Troutwine --- regression/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression/config.yaml b/regression/config.yaml index cc37662a0a..b9b54de25b 100644 --- a/regression/config.yaml +++ b/regression/config.yaml @@ -1,5 +1,5 @@ lading: - version: 0.25.3 + version: 0.25.4 target: From b4aaaa88fe96ef23d3e8c4e40a8c2045aa9f56c5 Mon Sep 17 00:00:00 2001 From: pomacanthidae Date: Wed, 22 Jan 2025 09:17:50 +0900 Subject: [PATCH 015/658] feat(pulsar sink): support tls options (#22148) * add tls_options to pulsar sink * add integration test * add tls options to source * add changelog for tls_options * rename changelog * move unused import to test * add cue for tls options * fix typos * rename fields * fix format * update cue * rename to verify_certificate * fix format * rename tls_options to tls * fix changelog * fix format --- ...sink_source_support_tls_options.feature.md | 3 + scripts/integration/pulsar/compose.yaml | 14 ++- scripts/integration/pulsar/test.yaml | 2 +- src/sinks/pulsar/config.rs | 46 +++++-- src/sinks/pulsar/integration_tests.rs | 29 ++++- src/sinks/pulsar/sink.rs | 6 +- src/sources/pulsar.rs | 113 ++++++++++++++++-- tests/data/Makefile | 18 +++ .../certs/pulsar-chain.cert.pem | 98 +++++++++++++++ .../intermediate_server/certs/pulsar.cert.pem | 32 +++++ .../ca/intermediate_server/csr/pulsar.csr.pem | 17 +++ tests/data/ca/intermediate_server/index.txt | 1 + .../data/ca/intermediate_server/index.txt.old | 1 + .../ca/intermediate_server/newcerts/1008.pem | 32 +++++ .../private/pulsar.key.pem | 27 +++++ tests/data/ca/intermediate_server/serial | 2 +- tests/data/ca/intermediate_server/serial.old | 2 +- .../components/sinks/base/pulsar.cue | 29 +++++ .../components/sources/base/pulsar.cue | 29 +++++ 19 files changed, 469 insertions(+), 32 deletions(-) create mode 100644 changelog.d/pulsar_sink_source_support_tls_options.feature.md create mode 100644 tests/data/ca/intermediate_server/certs/pulsar-chain.cert.pem create mode 100644 tests/data/ca/intermediate_server/certs/pulsar.cert.pem create mode 100644 tests/data/ca/intermediate_server/csr/pulsar.csr.pem create mode 100644 tests/data/ca/intermediate_server/newcerts/1008.pem create mode 100644 tests/data/ca/intermediate_server/private/pulsar.key.pem diff --git a/changelog.d/pulsar_sink_source_support_tls_options.feature.md b/changelog.d/pulsar_sink_source_support_tls_options.feature.md new file mode 100644 index 0000000000..1a326ece8d --- /dev/null +++ b/changelog.d/pulsar_sink_source_support_tls_options.feature.md @@ -0,0 +1,3 @@ +The `pulsar` source and sink now support configuration of TLS options via the `tls` configuration field. + +authors: pomacanthidae diff --git a/scripts/integration/pulsar/compose.yaml b/scripts/integration/pulsar/compose.yaml index b73d35909b..0e963cd2e0 100644 --- a/scripts/integration/pulsar/compose.yaml +++ b/scripts/integration/pulsar/compose.yaml @@ -3,6 +3,16 @@ version: '3' services: pulsar: image: docker.io/apachepulsar/pulsar:${CONFIG_VERSION} - command: bin/pulsar standalone + command: sh -c "bin/apply-config-from-env.py conf/standalone.conf && bin/pulsar standalone" ports: - - 6650:6650 + - 6650:6650 + - 6651:6651 + environment: + - PULSAR_PREFIX_brokerServicePortTls=6651 + - PULSAR_PREFIX_tlsKeyFilePath=/etc/pulsar/certs/pulsar.key.pem + - PULSAR_PREFIX_tlsCertificateFilePath=/etc/pulsar/certs/pulsar.cert.pem + - PULSAR_PREFIX_tlsTrustCertsFilePath=/etc/pulsar/certs/ca-chain.cert.pem + volumes: + - ../../../tests/data/ca/intermediate_server/private/pulsar.key.pem:/etc/pulsar/certs/pulsar.key.pem:ro + - ../../../tests/data//ca/intermediate_server/certs/pulsar.cert.pem:/etc/pulsar/certs/pulsar.cert.pem:ro + - ../../../tests/data/ca/intermediate_server/certs/ca-chain.cert.pem:/etc/pulsar/certs/ca-chain.cert.pem:ro diff --git a/scripts/integration/pulsar/test.yaml b/scripts/integration/pulsar/test.yaml index 824f0e0f29..b629a582d0 100644 --- a/scripts/integration/pulsar/test.yaml +++ b/scripts/integration/pulsar/test.yaml @@ -4,7 +4,7 @@ features: test_filter: '::pulsar::integration_tests::' env: - PULSAR_ADDRESS: pulsar://pulsar:6650 + PULSAR_HOST: pulsar matrix: version: [latest] diff --git a/src/sinks/pulsar/config.rs b/src/sinks/pulsar/config.rs index 93b6e81fd4..d565b73ae4 100644 --- a/src/sinks/pulsar/config.rs +++ b/src/sinks/pulsar/config.rs @@ -5,7 +5,7 @@ use crate::{ pulsar::sink::{healthcheck, PulsarSink}, }, }; -use futures_util::FutureExt; +use futures_util::{FutureExt, TryFutureExt}; use pulsar::{ authentication::oauth2::{OAuth2Authentication, OAuth2Params}, compression, @@ -14,7 +14,7 @@ use pulsar::{ TokioExecutor, }; use pulsar::{error::AuthenticationError, OperationRetryOptions}; -use snafu::ResultExt; +use std::path::Path; use std::time::Duration; use vector_lib::codecs::{encoding::SerializerConfig, TextSerializerConfig}; use vector_lib::config::DataType; @@ -82,6 +82,10 @@ pub struct PulsarSinkConfig { #[configurable(derived)] #[serde(default)] pub connection_retry_options: Option, + + #[configurable(derived)] + #[serde(default)] + pub(crate) tls: Option, } /// Event batching behavior. @@ -206,6 +210,25 @@ pub struct CustomConnectionRetryOptions { pub keep_alive_secs: Option, } +#[configurable_component] +#[configurable(description = "TLS options configuration for the Pulsar client.")] +#[derive(Clone, Debug)] +pub struct PulsarTlsOptions { + /// File path containing a list of PEM encoded certificates. + #[configurable(metadata(docs::examples = "/etc/certs/chain.pem"))] + pub ca_file: String, + + /// Enables certificate verification. + /// + /// Do NOT set this to `false` unless you understand the risks of not verifying the validity of certificates. + pub verify_certificate: Option, + + /// Whether hostname verification is enabled when verify_certificate is false. + /// + /// Set to true if not specified. + pub verify_hostname: Option, +} + impl Default for PulsarSinkConfig { fn default() -> Self { Self { @@ -221,12 +244,13 @@ impl Default for PulsarSinkConfig { auth: None, acknowledgements: Default::default(), connection_retry_options: None, + tls: None, } } } impl PulsarSinkConfig { - pub(crate) async fn create_pulsar_client(&self) -> Result, PulsarError> { + pub(crate) async fn create_pulsar_client(&self) -> crate::Result> { let mut builder = Pulsar::builder(&self.endpoint, TokioExecutor); if let Some(auth) = &self.auth { builder = match ( @@ -246,10 +270,10 @@ impl PulsarSinkConfig { scope: oauth2.scope.clone(), }), ), - _ => return Err(PulsarError::Authentication(AuthenticationError::Custom( + _ => return Err(Box::new(PulsarError::Authentication(AuthenticationError::Custom( "Invalid auth config: can only specify name and token or oauth2 configuration" .to_string(), - ))), + ))))?, }; } @@ -292,7 +316,14 @@ impl PulsarSinkConfig { let operation_retry_opts = OperationRetryOptions::default(); builder = builder.with_operation_retry_options(operation_retry_opts); - builder.build().await + if let Some(options) = &self.tls { + builder = builder.with_certificate_chain_file(Path::new(&options.ca_file))?; + builder = + builder.with_allow_insecure_connection(!options.verify_certificate.unwrap_or(true)); + builder = builder + .with_tls_hostname_verification_enabled(options.verify_hostname.unwrap_or(true)); + } + builder.build().map_err(|e| e.into()).await } pub(crate) fn build_producer_options(&self) -> ProducerOptions { @@ -354,10 +385,9 @@ impl SinkConfig for PulsarSinkConfig { let client = self .create_pulsar_client() .await - .context(super::sink::CreatePulsarSinkSnafu)?; + .map_err(|e| super::sink::BuildError::CreatePulsarSink { source: e })?; let sink = PulsarSink::new(client, self.clone())?; - let hc = healthcheck(self.clone()).boxed(); Ok((VectorSink::from_event_streamsink(sink), hc)) diff --git a/src/sinks/pulsar/integration_tests.rs b/src/sinks/pulsar/integration_tests.rs index 30a3d51a23..3b96ec0569 100644 --- a/src/sinks/pulsar/integration_tests.rs +++ b/src/sinks/pulsar/integration_tests.rs @@ -1,4 +1,4 @@ -use crate::sinks::pulsar::{config::PulsarSinkConfig, sink::PulsarSink}; +use crate::sinks::pulsar::{config::PulsarSinkConfig, config::PulsarTlsOptions, sink::PulsarSink}; use futures::StreamExt; use pulsar::SubType; @@ -9,10 +9,15 @@ use crate::test_util::{ components::{assert_sink_compliance, SINK_TAGS}, random_lines_with_stream, random_string, trace_init, }; +use crate::tls::TEST_PEM_INTERMEDIATE_CA_PATH; use bytes::Bytes; -fn pulsar_address() -> String { - std::env::var("PULSAR_ADDRESS").unwrap_or_else(|_| "pulsar://127.0.0.1:6650".into()) +fn pulsar_host() -> String { + std::env::var("PULSAR_HOST").unwrap_or_else(|_| "127.0.0.1".into()) +} + +fn pulsar_address(scheme: &str, port: u16) -> String { + format!("{}://{}:{}", scheme, pulsar_host(), port) } async fn pulsar_happy_reuse(mut cnf: PulsarSinkConfig) { @@ -80,7 +85,23 @@ async fn pulsar_happy_reuse(mut cnf: PulsarSinkConfig) { #[tokio::test] async fn pulsar_happy() { let cnf = PulsarSinkConfig { - endpoint: pulsar_address(), + endpoint: pulsar_address("pulsar", 6650), + // overriden by test + ..Default::default() + }; + + pulsar_happy_reuse(cnf).await +} + +#[tokio::test] +async fn pulsar_happy_tls() { + let cnf = PulsarSinkConfig { + endpoint: pulsar_address("pulsar+ssl", 6651), + tls: Some(PulsarTlsOptions { + ca_file: TEST_PEM_INTERMEDIATE_CA_PATH.into(), + verify_certificate: None, + verify_hostname: None, + }), // overriden by test ..Default::default() }; diff --git a/src/sinks/pulsar/sink.rs b/src/sinks/pulsar/sink.rs index 479d4455b9..85847d8b14 100644 --- a/src/sinks/pulsar/sink.rs +++ b/src/sinks/pulsar/sink.rs @@ -1,6 +1,6 @@ use async_trait::async_trait; use bytes::Bytes; -use pulsar::{Error as PulsarError, Pulsar, TokioExecutor}; +use pulsar::{Pulsar, TokioExecutor}; use serde::Serialize; use snafu::Snafu; use std::collections::HashMap; @@ -16,7 +16,9 @@ use crate::sinks::prelude::*; #[snafu(visibility(pub(crate)))] pub(crate) enum BuildError { #[snafu(display("creating pulsar producer failed: {}", source))] - CreatePulsarSink { source: PulsarError }, + CreatePulsarSink { + source: Box, + }, } pub(crate) struct PulsarSink { diff --git a/src/sources/pulsar.rs b/src/sources/pulsar.rs index ad0922ec93..349279667c 100644 --- a/src/sources/pulsar.rs +++ b/src/sources/pulsar.rs @@ -10,6 +10,7 @@ use pulsar::{ message::proto::MessageIdData, Authentication, Consumer, Pulsar, SubType, TokioExecutor, }; +use std::path::Path; use tokio_util::codec::FramedRead; use vector_lib::{ @@ -100,6 +101,10 @@ pub struct PulsarSourceConfig { #[configurable(metadata(docs::hidden))] #[serde(default)] log_namespace: Option, + + #[configurable(derived)] + #[serde(default)] + tls: Option, } /// Authentication configuration. @@ -172,6 +177,25 @@ struct DeadLetterQueuePolicy { pub dead_letter_topic: String, } +#[configurable_component] +#[configurable(description = "TLS options configuration for the Pulsar client.")] +#[derive(Clone, Debug)] +pub struct TlsOptions { + /// File path containing a list of PEM encoded certificates + #[configurable(metadata(docs::examples = "/etc/certs/chain.pem"))] + pub ca_file: String, + + /// Enables certificate verification. + /// + /// Do NOT set this to `false` unless you understand the risks of not verifying the validity of certificates. + pub verify_certificate: Option, + + /// Whether hostname verification is enabled when verify_certificate is false + /// + /// Set to true if not specified. + pub verify_hostname: Option, +} + #[derive(Debug)] struct FinalizerEntry { topic: String, @@ -263,10 +287,17 @@ impl PulsarSourceConfig { ), }; } + if let Some(options) = &self.tls { + builder = builder.with_certificate_chain_file(Path::new(&options.ca_file))?; + builder = + builder.with_allow_insecure_connection(!options.verify_certificate.unwrap_or(true)); + builder = builder + .with_tls_hostname_verification_enabled(options.verify_hostname.unwrap_or(true)); + } let pulsar = builder.build().await?; - let mut consumer_builder = pulsar + let mut consumer_builder: pulsar::ConsumerBuilder = pulsar .consumer() .with_topics(&self.topics) .with_subscription_type(SubType::Shared) @@ -523,37 +554,85 @@ mod integration_tests { use crate::config::log_schema; use crate::test_util::components::{assert_source_compliance, SOURCE_TAGS}; use crate::test_util::{collect_n, random_string, trace_init}; + use crate::tls::TEST_PEM_INTERMEDIATE_CA_PATH; - fn pulsar_address() -> String { - std::env::var("PULSAR_ADDRESS").unwrap_or_else(|_| "pulsar://127.0.0.1:6650".into()) + fn pulsar_host() -> String { + std::env::var("PULSAR_HOST").unwrap_or_else(|_| "127.0.0.1".into()) } + fn pulsar_address(scheme: &str, port: u16) -> String { + format!("{}://{}:{}", scheme, pulsar_host(), port) + } #[tokio::test] async fn consumes_event_with_acknowledgements() { - pulsar_send_receive(true, LogNamespace::Legacy).await; + pulsar_send_receive( + &pulsar_address("pulsar", 6650), + true, + LogNamespace::Legacy, + None, + ) + .await; } #[tokio::test] async fn consumes_event_with_acknowledgements_vector_namespace() { - pulsar_send_receive(true, LogNamespace::Vector).await; + pulsar_send_receive( + &pulsar_address("pulsar", 6650), + true, + LogNamespace::Vector, + None, + ) + .await; } #[tokio::test] async fn consumes_event_without_acknowledgements() { - pulsar_send_receive(false, LogNamespace::Legacy).await; + pulsar_send_receive( + &pulsar_address("pulsar", 6650), + false, + LogNamespace::Legacy, + None, + ) + .await; } #[tokio::test] async fn consumes_event_without_acknowledgements_vector_namespace() { - pulsar_send_receive(false, LogNamespace::Vector).await; + pulsar_send_receive( + &pulsar_address("pulsar", 6650), + false, + LogNamespace::Vector, + None, + ) + .await; } - async fn pulsar_send_receive(acknowledgements: bool, log_namespace: LogNamespace) { + #[tokio::test] + async fn consumes_event_with_tls() { + pulsar_send_receive( + &pulsar_address("pulsar+ssl", 6651), + false, + LogNamespace::Vector, + Some(TlsOptions { + ca_file: TEST_PEM_INTERMEDIATE_CA_PATH.into(), + verify_certificate: None, + verify_hostname: None, + }), + ) + .await; + } + + async fn pulsar_send_receive( + endpoint: &str, + acknowledgements: bool, + log_namespace: LogNamespace, + tls: Option, + ) { trace_init(); let topic = format!("test-{}", random_string(10)); let cnf = PulsarSourceConfig { - endpoint: pulsar_address(), + endpoint: endpoint.into(), topics: vec![topic.clone()], consumer_name: None, subscription_name: None, @@ -565,12 +644,20 @@ mod integration_tests { decoding: DeserializerConfig::Bytes, acknowledgements: acknowledgements.into(), log_namespace: None, + tls: tls.clone(), }; + let mut builder = Pulsar::::builder(&cnf.endpoint, TokioExecutor); + if let Some(options) = &tls { + builder = builder + .with_certificate_chain_file(Path::new(&options.ca_file)) + .unwrap(); + builder = + builder.with_allow_insecure_connection(!options.verify_certificate.unwrap_or(true)); + builder = builder + .with_tls_hostname_verification_enabled(options.verify_hostname.unwrap_or(true)); + } - let pulsar = Pulsar::::builder(&cnf.endpoint, TokioExecutor) - .build() - .await - .unwrap(); + let pulsar = builder.build().await.unwrap(); let consumer = cnf.create_consumer().await.unwrap(); let decoder = DecodingConfig::new( diff --git a/tests/data/Makefile b/tests/data/Makefile index e22c324885..f3550bf8bf 100644 --- a/tests/data/Makefile +++ b/tests/data/Makefile @@ -175,6 +175,24 @@ ca/intermediate_server/private/kafka.p12: ca/intermediate_server/private/kafka.k -in ca/intermediate_server/certs/kafka.cert.pem \ -inkey ca/intermediate_server/private/kafka.key.pem +ca/intermediate_server/private/pulsar.key.pem: + openssl genrsa -out ca/intermediate_server/private/pulsar.key.pem 2048 + +ca/intermediate_server/csr/pulsar.csr.pem: ca/intermediate_server/private/pulsar.key.pem + openssl req -config ca/intermediate_server/openssl.cnf \ + -key ca/intermediate_server/private/pulsar.key.pem \ + -subj '/CN=pulsar/OU=Vector/O=Datadog/ST=New York/L=New York/C=US' \ + -new -sha256 -out ca/intermediate_server/csr/pulsar.csr.pem + +ca/intermediate_server/certs/pulsar.cert.pem: ca/intermediate_server/csr/pulsar.csr.pem + openssl ca -batch -config ca/intermediate_server/openssl.cnf \ + -extensions server_cert -days 3650 -notext -md sha256 \ + -in ca/intermediate_server/csr/pulsar.csr.pem \ + -out ca/intermediate_server/certs/pulsar.cert.pem + +ca/intermediate_server/certs/pulsar-chain.cert.pem: ca/intermediate_server/certs/ca-chain.cert.pem ca/intermediate_server/certs/pulsar.cert.pem + cat ca/intermediate_server/certs/pulsar.cert.pem ca/intermediate_server/certs/ca-chain.cert.pem > ca/intermediate_server/certs/pulsar-chain.cert.pem + ca/intermediate_client/private/localhost.p12: ca/intermediate_client/private/localhost.key.pem ca/intermediate_client/certs/localhost.cert.pem ca/intermediate_client/certs/ca-chain.cert.pem openssl pkcs12 -chain -export -password pass:NOPASS -CAfile ca/intermediate_client/certs/ca-chain.cert.pem\ -out ca/intermediate_client/private/localhost.p12 \ diff --git a/tests/data/ca/intermediate_server/certs/pulsar-chain.cert.pem b/tests/data/ca/intermediate_server/certs/pulsar-chain.cert.pem new file mode 100644 index 0000000000..4c2d3e8ced --- /dev/null +++ b/tests/data/ca/intermediate_server/certs/pulsar-chain.cert.pem @@ -0,0 +1,98 @@ +-----BEGIN CERTIFICATE----- +MIIFgzCCA2ugAwIBAgICEAgwDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCVVMx +ETAPBgNVBAgMCE5ldyBZb3JrMRAwDgYDVQQKDAdEYXRhZG9nMQ8wDQYDVQQLDAZW +ZWN0b3IxJjAkBgNVBAMMHVZlY3RvciBJbnRlcm1lZGlhdGUgU2VydmVyIENBMB4X +DTI0MTIzMDA1MzE1OVoXDTM0MTIyODA1MzE1OVowZzELMAkGA1UEBhMCVVMxETAP +BgNVBAgMCE5ldyBZb3JrMREwDwYDVQQHDAhOZXcgWW9yazEQMA4GA1UECgwHRGF0 +YWRvZzEPMA0GA1UECwwGVmVjdG9yMQ8wDQYDVQQDDAZwdWxzYXIwggEiMA0GCSqG +SIb3DQEBAQUAA4IBDwAwggEKAoIBAQChjtFd/pz494HLGdR5yMAgBYNKTlItM1Eb +1XAMWuTNAYhOITkBoAE2v7Dg9Z9gQLr6/HK99z2dfAKBFvaoz4aWcquk8TGOGjfY +TJamJuGhzfPYv4g5zJq15B+RIwd7/WkH1PzAk94xclBsnDob5nQ70SdZuKwF4R60 +8n6fegPSSMxG0PIgmwJnX1Gwz5RBuomZZn9onQvZTdJqyHpPSQYBHQXgtNGwPS2d +3FAPBxlGu+OajwDfkcLPUFz3tPf4t2iSitCqtQu/aGH450syx/hmwtZq63aReEkG +DOLXBq5MrGqDAzKIltbWoBYWgx6pkk56XcWv83AlxqvauHBmCN+9AgMBAAGjggEz +MIIBLzAJBgNVHRMEAjAAMBEGCWCGSAGG+EIBAQQEAwIGQDAzBglghkgBhvhCAQ0E +JhYkT3BlblNTTCBHZW5lcmF0ZWQgU2VydmVyIENlcnRpZmljYXRlMB0GA1UdDgQW +BBSP3J69fVPZ+HDbxHtOcgbm16ZK3jCBlQYDVR0jBIGNMIGKgBQ8PTovzNWCA32Z +xFjx4ds2760NQaFupGwwajESMBAGA1UEAwwJVmVjdG9yIENBMQ8wDQYDVQQLDAZW +ZWN0b3IxEDAOBgNVBAoMB0RhdGFkb2cxETAPBgNVBAgMCE5ldyBZb3JrMREwDwYD +VQQHDAhOZXcgWW9yazELMAkGA1UEBhMCVVOCAhAAMA4GA1UdDwEB/wQEAwIFoDAT +BgNVHSUEDDAKBggrBgEFBQcDATANBgkqhkiG9w0BAQsFAAOCAgEAaED846BaRdU7 +GGNJcAk5Xzq6vjO5gG+0xMzfiG/oQEIizNDJ1PXAN343QUDHR7q+ISRY7w+nr3Ar +pHSSdn3iPzznfVWWhtPnMhDxcvZPlwCR+TXwbkhNMQyK/K+pnRBUjZzWqIT2h0fG +vr6b6TsVc445ED/fb2W3HKEwOTxeiLFRAsvS/Z5pbKYGP5qTNQgkGD6WBx9vQvrT +1zQvZNo9vur41dYwPdcRqkvc/lB9I69zGSG78xzAc48ZRKbrhZ/zigD63FBFzcaL +FeCsAIg2fzp3dw6SrUhIzsnDxML79kd1cTyZv9IeRWx0Af6DDRVLZahC1fh+26cH +a9yt4002e9bbardqmyoX71Jc6ybmDhSik32PYYkE53c/qEwCuGy8/jm1UW5QixNw +ntsBw4iuf8Q3MTf46l9DmBzfxMOK0CX/aDoPCnj9BJ3UT+eviyvwvYzXn0hJyfLZ +taC9HLDJV2Xx3ggLO8wBGXlCqvwDH+EzNaSFIRGgY1gacjcYMWBPgddUgJ/mgsH4 +28J3Q8QS5jjXUaDtTuDRkWhF5j801YOpgCXA4jVFvUmw+aqbkRArWz/iXFxTe2UP +Q1F7GIUhalRtO2Fti7Tb+KfUB5ybs2S7NQ9HUi2SQBCnTZKEFGW6RahHMC9mn3E9 +Na6qmTaFImoYHs7zrfGEFcEowaQlMcI= +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIFtzCCA5+gAwIBAgICEAAwDQYJKoZIhvcNAQELBQAwajESMBAGA1UEAwwJVmVj +dG9yIENBMQ8wDQYDVQQLDAZWZWN0b3IxEDAOBgNVBAoMB0RhdGFkb2cxETAPBgNV +BAgMCE5ldyBZb3JrMREwDwYDVQQHDAhOZXcgWW9yazELMAkGA1UEBhMCVVMwHhcN +MjIwNjA3MjIyNzUzWhcNMzIwNjA0MjIyNzUzWjBrMQswCQYDVQQGEwJVUzERMA8G +A1UECAwITmV3IFlvcmsxEDAOBgNVBAoMB0RhdGFkb2cxDzANBgNVBAsMBlZlY3Rv +cjEmMCQGA1UEAwwdVmVjdG9yIEludGVybWVkaWF0ZSBTZXJ2ZXIgQ0EwggIiMA0G +CSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCy/mB0/ZwfgKrSZPQIFaGPtRA9xL2N +o2SsHndZ8k2XOCV225Hb2fzNH+o2WGNSjwmGjLP/uXb47KH0cHCAyFGzSjp++8/O +zoZaFiO0P5El02hQxmoabO3Cqu/N62EFsLfpSM828JM6YOn9p+WXUDn1+YPNoOOE +H142p4/RjFnXNHkzR3geXU4Pfi3KXDrMi8vK42lDqXPLPs6rhreBAfQ2dsYyqhz6 +tg6FzZuXxxzEYyYtNgGh+zTji99WCBMLbCmRcDurRjdTDO7m4O3PrwbGUy0xdLeb +HJiNGvUDCPH4bfwLiNqwVIZY38RBCAqbCnrqRhDaZIfAUev4mq3Kqh6KUeO/U7Vx +/5J5rL5ApREKOfWPATHMprBuEU2rs3N+MPBA04HoiFlu311urCxVEA1qsZCTkoCg +GHuDIVSU4E4hT4co95/J0to4zWgPlfPg1+cXyU8lAIMe7JdCGkG9cDe7Umw/GSbt +ZdoCMQZ6WyyiW2Hw+7sFD3V3VzYa5YA/rjKZRduPmGWKrs+mAa5J5pM2M22rrjbd +EpfTHWLS9s6cPN3/jxpCxn6Hv/KhIYRAcIterugag1+clvS1ajVjxBRavOxPBsf+ +hYdh7S5NTZnT98gjkc3yOuGQm7BPtXau+IYZRlWcB0dJ4/E2P69hmWQezSo9VVWh +5/K1RkbPvqTGZQIDAQABo2YwZDAdBgNVHQ4EFgQUPD06L8zVggN9mcRY8eHbNu+t +DUEwHwYDVR0jBBgwFoAURTWK6ARqnZkz8rktUc5PrtasIh8wEgYDVR0TAQH/BAgw +BgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwDQYJKoZIhvcNAQELBQADggIBAGqaGBuL +2J6Na8RHx/GmSeuZFiVcWhmd/I9bVpeMSYHSZujA2nay6OGaUYs0Lq/G5OKgsuT9 +AIHnsn7VUE1zqoDfXac/K8sXlOig8if7rTb+06jgymaP1YSELg3R+pBsdkZnXVil +izh/9FvzoyV+QQlIhojqCIybVFgxa1XFHq4QCPhDfwkg+tp9RctfwNmWgsJ63H19 +RmxN+H2xIrySvObwXnB4j6D4wvgu468QXQMEuSsnLcIQFg6Zteqe8fixbqTiOTBf +Dk1k+EpB9VMEkIPvMdfa48vseXdBEe6Ma9zGuJC76q4q1ZapVLTvOUP5Y24khlgd +cj5tfP7o7yc6HqymfXAcD1lzP2JQhqaRxA4I18Nrd+aHi+G1EM2c3cicuD3n6Iw9 +9oqdCwmMfS25fv5cyA5B6hRusIZ9wRopTi7at+JHl0GIt/FelaTYI7kRmAqgakQe +oEKLpXcH8lRJW802DmXm7ka4eQzwxa7Ngyf8O+JOFtGO0+EshuLJovxiPl6IyLyG +NJ/dHq3ad+46YVManbHdyjHxgT5PSvJFkq0Yluvf44NIyP5QRTCAvfH76bu7hXgS +QoQj5t5ILn6meQRTR79r2iwpQTanPLTEdoZvmrE4TeUBev9BA5KpiPPA3i3ZF/oV +0EYorXCNri7M/jylGW7AuWvNUyaVR6xgxAn6 +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIFujCCA6KgAwIBAgIJAKhPL9BkNaFGMA0GCSqGSIb3DQEBCwUAMGoxEjAQBgNV +BAMMCVZlY3RvciBDQTEPMA0GA1UECwwGVmVjdG9yMRAwDgYDVQQKDAdEYXRhZG9n +MREwDwYDVQQIDAhOZXcgWW9yazERMA8GA1UEBwwITmV3IFlvcmsxCzAJBgNVBAYT +AlVTMB4XDTIyMDYwNzIyMjc1MloXDTQyMDYwMjIyMjc1MlowajESMBAGA1UEAwwJ +VmVjdG9yIENBMQ8wDQYDVQQLDAZWZWN0b3IxEDAOBgNVBAoMB0RhdGFkb2cxETAP +BgNVBAgMCE5ldyBZb3JrMREwDwYDVQQHDAhOZXcgWW9yazELMAkGA1UEBhMCVVMw +ggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQC9c1T+NXTNmqiiV36NSEJt +7mo0cyv8Byk2ZGdC85vHBm45QDY5USoh0vgonzPpWgSMggPn1WbR0f1y+LBwXdlM ++ZyZh2RVVeUrSjJ88lLHVn4DfywpdDkwQaFj1VmOsj2I9rMMrgc5x5n1Hj7lwZ+t +uPVSAGmgKp4iFfzLph9r/rjP1TUAnVUComfTUVS+Gd7zoGPOc14cMJXG6g2P2aAU +P6dg5uQlTxRmagnlx7bwm3lRwv6LMtnAdnjwBDBxr933nucAnk21GgE92GejiO3Z +OwlzIdzBI23lPcWi5pq+vCTgAArNq24W1Ha+7Jn5QewNTGKFyyYAJetZAwCUR8QS +Ip++2GE2pNhaGqcV5u1Tbwl02eD6p2qRqjfgLxmb+aC6xfl0n9kiFGPZppjCqDEW +sw+gX66nf+qxZVRWpJon2kWcFvhTnLqoa3T3+9+KIeamz2lW6wxMnki/Co2EA1Wa +mmedaUUcRPCgMx9aCktRkMyH6bEY8/vfJ07juxUsszOc46T00Scmn6Vkuo9Uc3Kf +2Q2N6Wo4jtyAiMO4gAwq5kzzpBAhNgRfLHOb83r2gAUj2Y4Vln/UUR/KR8ZbJi4i +r1BjX16Lz3yblJXXb1lp4uZynlbHNaAevXyGlRqHddM2ykKtAX/vgJcZRGSvms11 +uce/cqzrzx60AhpLRma5CwIDAQABo2MwYTAdBgNVHQ4EFgQURTWK6ARqnZkz8rkt +Uc5PrtasIh8wHwYDVR0jBBgwFoAURTWK6ARqnZkz8rktUc5PrtasIh8wDwYDVR0T +AQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAYYwDQYJKoZIhvcNAQELBQADggIBAEf5 +TR3hq/DtSAmsYotu1lAWz/OlTpG+7AdqSOHB878X4ETN3xaQ+KWvSwvf0K70ZDTV +tFOTh/r43cpzPifPKd1P+2ctnQEzrBtAacvyETLq1ABRK9VJOtfJ6Xk5KZXPhKdY +t353PQgBgW8YzQ2adq2B7FtgIlX7f1DIndjcMZBbolETR6xt9QwB/UnPI7Mwt01T ++bCBhr1fWAbZ4YAMlQ0xRam4qUOTjxgfmePrmSrv4HO7cXHMsRMLiXk+BLcx959/ +K/B6xzpzn6366Eqnqlo/uDiMpo5ud2I/Snz5PduB6oLztPMEf/8RmkG5tpHXYdWr +tM64WqNGO+ikluIrrtYvtyZS4DfsLAMfMYZcxX/Uw56gHo0i2c8I6+6JvGWdvOJ0 +FjrsKeIQoRlV77z025kI4V9jKi3XNMEsAIH+W7KNSut0X80yX7SugvQGoe0GDkXu +0fy8hMC3uTN2LEycYFRRfoIeKPLi6OZFK0PdS2E15d8PEU3n3W4eBCPgMtmiOKLY +d8QNBC8XLAuBoK9R8luCJpOJWUcFXjLpjcDab4V2hKTuAs+GQyDh/Xx4wF1yHX0r +zIkyN0EkOD/SvD8X4uFaM4mdsAh+ucn4ryUV7i5PgvDM9z4InHAMAee1ebBl0U+h ++NzMWF5c5OwxD5o6/Wh1HopmzJiVNT2v9u0kHT/f +-----END CERTIFICATE----- diff --git a/tests/data/ca/intermediate_server/certs/pulsar.cert.pem b/tests/data/ca/intermediate_server/certs/pulsar.cert.pem new file mode 100644 index 0000000000..2a83a2e042 --- /dev/null +++ b/tests/data/ca/intermediate_server/certs/pulsar.cert.pem @@ -0,0 +1,32 @@ +-----BEGIN CERTIFICATE----- +MIIFgzCCA2ugAwIBAgICEAgwDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCVVMx +ETAPBgNVBAgMCE5ldyBZb3JrMRAwDgYDVQQKDAdEYXRhZG9nMQ8wDQYDVQQLDAZW +ZWN0b3IxJjAkBgNVBAMMHVZlY3RvciBJbnRlcm1lZGlhdGUgU2VydmVyIENBMB4X +DTI0MTIzMDA1MzE1OVoXDTM0MTIyODA1MzE1OVowZzELMAkGA1UEBhMCVVMxETAP +BgNVBAgMCE5ldyBZb3JrMREwDwYDVQQHDAhOZXcgWW9yazEQMA4GA1UECgwHRGF0 +YWRvZzEPMA0GA1UECwwGVmVjdG9yMQ8wDQYDVQQDDAZwdWxzYXIwggEiMA0GCSqG +SIb3DQEBAQUAA4IBDwAwggEKAoIBAQChjtFd/pz494HLGdR5yMAgBYNKTlItM1Eb +1XAMWuTNAYhOITkBoAE2v7Dg9Z9gQLr6/HK99z2dfAKBFvaoz4aWcquk8TGOGjfY +TJamJuGhzfPYv4g5zJq15B+RIwd7/WkH1PzAk94xclBsnDob5nQ70SdZuKwF4R60 +8n6fegPSSMxG0PIgmwJnX1Gwz5RBuomZZn9onQvZTdJqyHpPSQYBHQXgtNGwPS2d +3FAPBxlGu+OajwDfkcLPUFz3tPf4t2iSitCqtQu/aGH450syx/hmwtZq63aReEkG +DOLXBq5MrGqDAzKIltbWoBYWgx6pkk56XcWv83AlxqvauHBmCN+9AgMBAAGjggEz +MIIBLzAJBgNVHRMEAjAAMBEGCWCGSAGG+EIBAQQEAwIGQDAzBglghkgBhvhCAQ0E +JhYkT3BlblNTTCBHZW5lcmF0ZWQgU2VydmVyIENlcnRpZmljYXRlMB0GA1UdDgQW +BBSP3J69fVPZ+HDbxHtOcgbm16ZK3jCBlQYDVR0jBIGNMIGKgBQ8PTovzNWCA32Z +xFjx4ds2760NQaFupGwwajESMBAGA1UEAwwJVmVjdG9yIENBMQ8wDQYDVQQLDAZW +ZWN0b3IxEDAOBgNVBAoMB0RhdGFkb2cxETAPBgNVBAgMCE5ldyBZb3JrMREwDwYD +VQQHDAhOZXcgWW9yazELMAkGA1UEBhMCVVOCAhAAMA4GA1UdDwEB/wQEAwIFoDAT +BgNVHSUEDDAKBggrBgEFBQcDATANBgkqhkiG9w0BAQsFAAOCAgEAaED846BaRdU7 +GGNJcAk5Xzq6vjO5gG+0xMzfiG/oQEIizNDJ1PXAN343QUDHR7q+ISRY7w+nr3Ar +pHSSdn3iPzznfVWWhtPnMhDxcvZPlwCR+TXwbkhNMQyK/K+pnRBUjZzWqIT2h0fG +vr6b6TsVc445ED/fb2W3HKEwOTxeiLFRAsvS/Z5pbKYGP5qTNQgkGD6WBx9vQvrT +1zQvZNo9vur41dYwPdcRqkvc/lB9I69zGSG78xzAc48ZRKbrhZ/zigD63FBFzcaL +FeCsAIg2fzp3dw6SrUhIzsnDxML79kd1cTyZv9IeRWx0Af6DDRVLZahC1fh+26cH +a9yt4002e9bbardqmyoX71Jc6ybmDhSik32PYYkE53c/qEwCuGy8/jm1UW5QixNw +ntsBw4iuf8Q3MTf46l9DmBzfxMOK0CX/aDoPCnj9BJ3UT+eviyvwvYzXn0hJyfLZ +taC9HLDJV2Xx3ggLO8wBGXlCqvwDH+EzNaSFIRGgY1gacjcYMWBPgddUgJ/mgsH4 +28J3Q8QS5jjXUaDtTuDRkWhF5j801YOpgCXA4jVFvUmw+aqbkRArWz/iXFxTe2UP +Q1F7GIUhalRtO2Fti7Tb+KfUB5ybs2S7NQ9HUi2SQBCnTZKEFGW6RahHMC9mn3E9 +Na6qmTaFImoYHs7zrfGEFcEowaQlMcI= +-----END CERTIFICATE----- diff --git a/tests/data/ca/intermediate_server/csr/pulsar.csr.pem b/tests/data/ca/intermediate_server/csr/pulsar.csr.pem new file mode 100644 index 0000000000..42c5e55ba3 --- /dev/null +++ b/tests/data/ca/intermediate_server/csr/pulsar.csr.pem @@ -0,0 +1,17 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIICrDCCAZQCAQAwZzEPMA0GA1UEAwwGcHVsc2FyMQ8wDQYDVQQLDAZWZWN0b3Ix +EDAOBgNVBAoMB0RhdGFkb2cxETAPBgNVBAgMCE5ldyBZb3JrMREwDwYDVQQHDAhO +ZXcgWW9yazELMAkGA1UEBhMCVVMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK +AoIBAQChjtFd/pz494HLGdR5yMAgBYNKTlItM1Eb1XAMWuTNAYhOITkBoAE2v7Dg +9Z9gQLr6/HK99z2dfAKBFvaoz4aWcquk8TGOGjfYTJamJuGhzfPYv4g5zJq15B+R +Iwd7/WkH1PzAk94xclBsnDob5nQ70SdZuKwF4R608n6fegPSSMxG0PIgmwJnX1Gw +z5RBuomZZn9onQvZTdJqyHpPSQYBHQXgtNGwPS2d3FAPBxlGu+OajwDfkcLPUFz3 +tPf4t2iSitCqtQu/aGH450syx/hmwtZq63aReEkGDOLXBq5MrGqDAzKIltbWoBYW +gx6pkk56XcWv83AlxqvauHBmCN+9AgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAQEA +Vz+KO9XWHosTrmB3ZkYdKZt2ktItXLkWqFMggWDotdKdTNj1OaLfxX4Dmq4aO+Xw +hOlYKh40PEi0+3BqWhjysyssZTs/fQWmXtY+qyBF3fG+8wPwnhUvAczcSKVJqIIW +nKoixmHm1yrOGCGEraMldS3cfn1H8KBzWZKqY5oX0TIo323C/W54BcepJUjsFM+W +Fe5mO/rjeg/XCzbs7mFDNlv958BJ6but3Q6jUEjf7FhhVRyaNwIuzfDB0qbmEdQU +6djIqwEGs5MpAymLAvO+mIhIiYU3CxhSGLp0o85FYrKchk4joMD6U1Y18gEjrCye ++EOtC2qh34fI/6vZYYQAjw== +-----END CERTIFICATE REQUEST----- diff --git a/tests/data/ca/intermediate_server/index.txt b/tests/data/ca/intermediate_server/index.txt index ab4efe2c2f..0402ad0946 100644 --- a/tests/data/ca/intermediate_server/index.txt +++ b/tests/data/ca/intermediate_server/index.txt @@ -5,3 +5,4 @@ V 320613195026Z 1004 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/C V 320613195253Z 1005 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/CN=kafka V 320731200837Z 1006 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/CN=dufs-https V 330412000039Z 1007 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/CN=rabbitmq +V 341228053159Z 1008 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/CN=pulsar diff --git a/tests/data/ca/intermediate_server/index.txt.old b/tests/data/ca/intermediate_server/index.txt.old index c76d7b6e20..ab4efe2c2f 100644 --- a/tests/data/ca/intermediate_server/index.txt.old +++ b/tests/data/ca/intermediate_server/index.txt.old @@ -4,3 +4,4 @@ V 320613194901Z 1003 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/C V 320613195026Z 1004 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/CN=postgres V 320613195253Z 1005 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/CN=kafka V 320731200837Z 1006 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/CN=dufs-https +V 330412000039Z 1007 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/CN=rabbitmq diff --git a/tests/data/ca/intermediate_server/newcerts/1008.pem b/tests/data/ca/intermediate_server/newcerts/1008.pem new file mode 100644 index 0000000000..2a83a2e042 --- /dev/null +++ b/tests/data/ca/intermediate_server/newcerts/1008.pem @@ -0,0 +1,32 @@ +-----BEGIN CERTIFICATE----- +MIIFgzCCA2ugAwIBAgICEAgwDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCVVMx +ETAPBgNVBAgMCE5ldyBZb3JrMRAwDgYDVQQKDAdEYXRhZG9nMQ8wDQYDVQQLDAZW +ZWN0b3IxJjAkBgNVBAMMHVZlY3RvciBJbnRlcm1lZGlhdGUgU2VydmVyIENBMB4X +DTI0MTIzMDA1MzE1OVoXDTM0MTIyODA1MzE1OVowZzELMAkGA1UEBhMCVVMxETAP +BgNVBAgMCE5ldyBZb3JrMREwDwYDVQQHDAhOZXcgWW9yazEQMA4GA1UECgwHRGF0 +YWRvZzEPMA0GA1UECwwGVmVjdG9yMQ8wDQYDVQQDDAZwdWxzYXIwggEiMA0GCSqG +SIb3DQEBAQUAA4IBDwAwggEKAoIBAQChjtFd/pz494HLGdR5yMAgBYNKTlItM1Eb +1XAMWuTNAYhOITkBoAE2v7Dg9Z9gQLr6/HK99z2dfAKBFvaoz4aWcquk8TGOGjfY +TJamJuGhzfPYv4g5zJq15B+RIwd7/WkH1PzAk94xclBsnDob5nQ70SdZuKwF4R60 +8n6fegPSSMxG0PIgmwJnX1Gwz5RBuomZZn9onQvZTdJqyHpPSQYBHQXgtNGwPS2d +3FAPBxlGu+OajwDfkcLPUFz3tPf4t2iSitCqtQu/aGH450syx/hmwtZq63aReEkG +DOLXBq5MrGqDAzKIltbWoBYWgx6pkk56XcWv83AlxqvauHBmCN+9AgMBAAGjggEz +MIIBLzAJBgNVHRMEAjAAMBEGCWCGSAGG+EIBAQQEAwIGQDAzBglghkgBhvhCAQ0E +JhYkT3BlblNTTCBHZW5lcmF0ZWQgU2VydmVyIENlcnRpZmljYXRlMB0GA1UdDgQW +BBSP3J69fVPZ+HDbxHtOcgbm16ZK3jCBlQYDVR0jBIGNMIGKgBQ8PTovzNWCA32Z +xFjx4ds2760NQaFupGwwajESMBAGA1UEAwwJVmVjdG9yIENBMQ8wDQYDVQQLDAZW +ZWN0b3IxEDAOBgNVBAoMB0RhdGFkb2cxETAPBgNVBAgMCE5ldyBZb3JrMREwDwYD +VQQHDAhOZXcgWW9yazELMAkGA1UEBhMCVVOCAhAAMA4GA1UdDwEB/wQEAwIFoDAT +BgNVHSUEDDAKBggrBgEFBQcDATANBgkqhkiG9w0BAQsFAAOCAgEAaED846BaRdU7 +GGNJcAk5Xzq6vjO5gG+0xMzfiG/oQEIizNDJ1PXAN343QUDHR7q+ISRY7w+nr3Ar +pHSSdn3iPzznfVWWhtPnMhDxcvZPlwCR+TXwbkhNMQyK/K+pnRBUjZzWqIT2h0fG +vr6b6TsVc445ED/fb2W3HKEwOTxeiLFRAsvS/Z5pbKYGP5qTNQgkGD6WBx9vQvrT +1zQvZNo9vur41dYwPdcRqkvc/lB9I69zGSG78xzAc48ZRKbrhZ/zigD63FBFzcaL +FeCsAIg2fzp3dw6SrUhIzsnDxML79kd1cTyZv9IeRWx0Af6DDRVLZahC1fh+26cH +a9yt4002e9bbardqmyoX71Jc6ybmDhSik32PYYkE53c/qEwCuGy8/jm1UW5QixNw +ntsBw4iuf8Q3MTf46l9DmBzfxMOK0CX/aDoPCnj9BJ3UT+eviyvwvYzXn0hJyfLZ +taC9HLDJV2Xx3ggLO8wBGXlCqvwDH+EzNaSFIRGgY1gacjcYMWBPgddUgJ/mgsH4 +28J3Q8QS5jjXUaDtTuDRkWhF5j801YOpgCXA4jVFvUmw+aqbkRArWz/iXFxTe2UP +Q1F7GIUhalRtO2Fti7Tb+KfUB5ybs2S7NQ9HUi2SQBCnTZKEFGW6RahHMC9mn3E9 +Na6qmTaFImoYHs7zrfGEFcEowaQlMcI= +-----END CERTIFICATE----- diff --git a/tests/data/ca/intermediate_server/private/pulsar.key.pem b/tests/data/ca/intermediate_server/private/pulsar.key.pem new file mode 100644 index 0000000000..455e171c03 --- /dev/null +++ b/tests/data/ca/intermediate_server/private/pulsar.key.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAoY7RXf6c+PeByxnUecjAIAWDSk5SLTNRG9VwDFrkzQGITiE5 +AaABNr+w4PWfYEC6+vxyvfc9nXwCgRb2qM+GlnKrpPExjho32EyWpibhoc3z2L+I +OcyateQfkSMHe/1pB9T8wJPeMXJQbJw6G+Z0O9EnWbisBeEetPJ+n3oD0kjMRtDy +IJsCZ19RsM+UQbqJmWZ/aJ0L2U3Sash6T0kGAR0F4LTRsD0tndxQDwcZRrvjmo8A +35HCz1Bc97T3+LdokorQqrULv2hh+OdLMsf4ZsLWaut2kXhJBgzi1wauTKxqgwMy +iJbW1qAWFoMeqZJOel3Fr/NwJcar2rhwZgjfvQIDAQABAoIBAQCRDSJLayPSoJ5H +FPzhDu34khqLp4A+lvl2rQ55+U2+Bmc0Br4hCGCuoDMV91HN4LBAUlJO3uHSOJoi +/tQdQp4LDA+x+t/MPxvutuz3ZqdkV/5cl7KeemZZcuUosTOyDhaz8nWwDdMLwCAX +M7k0fZfUcPgMpTLF4JZqTeKEGs+9hzOsXMYLEsJX1oo3/ITb88DA2Uh0zFBZeGaG +wgb8tuXLpbwJY8KQTP9Y24UPEWW5hYNvSzBTKqGmfhQ4o/zDFU688ZYraQ+w3qxG +/dQjKiwlZPwyLq6LPhb160lZ/5inWaOH36vKbqLuToZ9m4VHSAfVTWFjhzQkYULB +cwmDPbcdAoGBAM7UcSNDZz0YtwluAml2r/yIjZ/cyI/4WnSJm7RVoIwZVvubDslk +sseWs0k7TmxQH/4npgV7nuZ2GtuOOtJ97UQ/ejfvBFIDA4EkEdVZNeFokE9Ab9Lt +iE9Nym9Ep5u5Y58ibVHTLpaBxap6i6tkEehWy4Fk4tfBLyJ8PL7JLyoDAoGBAMf3 +Ja9ovBFhkgT6s+COEVbkfLg07hL5stUqps61TfzqdcQ/M6KVfvC4cGr0vcRRehC8 +IlbK+3nW1j1DhUlsxXYYTYGLPWaf6IwkV7GiC6tDOr+OA795/ms/QbF47toXH2Ft +NvW3veBBSTz0viSsJpvLdth5cNfT/FeHVBo++oM/AoGAQ/QQZ+GRXcVs/bAjIrtX +/sRuo4NKceLK9VbwzUMEyILazTeYmBp5kpG0ve66SWPZ3wrvxQVko5tSH6iPMvEk +rzOb2Byzcq6CzD8pjMsOpMxR/XfSRZ69FjEcvYn093jlfjc1a5jSyahBZU117g2m +xIsfuZeH4BMchVwEKTq4QDkCgYApKRRWwOZFZaIa6EfyZwvmqO2LNUn0GfXXrwna +7rL45oILPT8xrjgM8MojGfGd4W+Q1kjzOKD++VvsDGP5MyyKHIKHsdOXtj616h0q +8UIZpKMJHwBif4gBJ2osT8pKlgvdkA+KEKJC8O1UYMRq5AymcQErmgPCSV5d3ftP +07rZHQKBgET20wFb0gN3kM7RLqz8xgPuu0ZVIoPr9+lZTqRMe0U2eI/WKFijiQ2L ++xiYsJkhsMY1nQc4uXiuUl2aGwbQRIIspP8H9qKjK7kzfadERCUu1I1qvHWwTwkp +1GNzsoKNMDf9le9DuEOMfP+TePOhPGf5GtEKeHZmiIMr/YhnDPUc +-----END RSA PRIVATE KEY----- diff --git a/tests/data/ca/intermediate_server/serial b/tests/data/ca/intermediate_server/serial index 617ba1c154..6cb3869343 100644 --- a/tests/data/ca/intermediate_server/serial +++ b/tests/data/ca/intermediate_server/serial @@ -1 +1 @@ -1008 +1009 diff --git a/tests/data/ca/intermediate_server/serial.old b/tests/data/ca/intermediate_server/serial.old index fb35a14c02..617ba1c154 100644 --- a/tests/data/ca/intermediate_server/serial.old +++ b/tests/data/ca/intermediate_server/serial.old @@ -1 +1 @@ -1007 +1008 diff --git a/website/cue/reference/components/sinks/base/pulsar.cue b/website/cue/reference/components/sinks/base/pulsar.cue index ba3c1b4b77..aa19003c3b 100644 --- a/website/cue/reference/components/sinks/base/pulsar.cue +++ b/website/cue/reference/components/sinks/base/pulsar.cue @@ -539,6 +539,35 @@ base: components: sinks: pulsar: configuration: { required: false type: string: {} } + tls: { + description: "TLS options configuration for the Pulsar client." + required: false + type: object: options: { + ca_file: { + description: "File path containing a list of PEM encoded certificates." + required: true + type: string: examples: ["/etc/certs/chain.pem"] + } + verify_certificate: { + description: """ + Enables certificate verification. + + Do NOT set this to `false` unless you understand the risks of not verifying the validity of certificates. + """ + required: false + type: bool: {} + } + verify_hostname: { + description: """ + Whether hostname verification is enabled when verify_certificate is false. + + Set to true if not specified. + """ + required: false + type: bool: {} + } + } + } topic: { description: "The Pulsar topic name to write events to." required: true diff --git a/website/cue/reference/components/sources/base/pulsar.cue b/website/cue/reference/components/sources/base/pulsar.cue index 14452ffe67..881d4d7f29 100644 --- a/website/cue/reference/components/sources/base/pulsar.cue +++ b/website/cue/reference/components/sources/base/pulsar.cue @@ -541,6 +541,35 @@ base: components: sources: pulsar: configuration: { required: false type: string: examples: ["subscription_name"] } + tls: { + description: "TLS options configuration for the Pulsar client." + required: false + type: object: options: { + ca_file: { + description: "File path containing a list of PEM encoded certificates" + required: true + type: string: examples: ["/etc/certs/chain.pem"] + } + verify_certificate: { + description: """ + Enables certificate verification. + + Do NOT set this to `false` unless you understand the risks of not verifying the validity of certificates. + """ + required: false + type: bool: {} + } + verify_hostname: { + description: """ + Whether hostname verification is enabled when verify_certificate is false + + Set to true if not specified. + """ + required: false + type: bool: {} + } + } + } topics: { description: "The Pulsar topic names to read events from." required: true From 0df7c486e7c7251da200167d55f6dbbc1910e6fb Mon Sep 17 00:00:00 2001 From: up2neck <163534172+up2neck@users.noreply.github.com> Date: Wed, 22 Jan 2025 20:32:19 +0200 Subject: [PATCH 016/658] chore(website): add notice on timestamp field renaming in the Elasticsearch sink, when use data_stream mode (#22196) Add notice on timestamp field renaming in the data_stream mode of Elasticsearch sink Signed-off-by: Up Neck <163534172+up2neck@users.noreply.github.com> --- src/sinks/elasticsearch/mod.rs | 3 +++ website/cue/reference/components/sinks/base/elasticsearch.cue | 3 +++ 2 files changed, 6 insertions(+) diff --git a/src/sinks/elasticsearch/mod.rs b/src/sinks/elasticsearch/mod.rs index d9a1a35440..696c8ecc20 100644 --- a/src/sinks/elasticsearch/mod.rs +++ b/src/sinks/elasticsearch/mod.rs @@ -68,6 +68,9 @@ pub enum ElasticsearchMode { /// Ingests documents in bulk, using the bulk API `create` action. /// /// Elasticsearch Data Streams only support the `create` action. + /// + /// If the mode is set to `data_stream` and a `timestamp` field is present in a message, + /// Vector renames this field to the expected `@timestamp` to comply with the Elastic Common Schema. DataStream, } diff --git a/website/cue/reference/components/sinks/base/elasticsearch.cue b/website/cue/reference/components/sinks/base/elasticsearch.cue index f18bc5d84c..16b73c3ecd 100644 --- a/website/cue/reference/components/sinks/base/elasticsearch.cue +++ b/website/cue/reference/components/sinks/base/elasticsearch.cue @@ -572,6 +572,9 @@ base: components: sinks: elasticsearch: configuration: { Ingests documents in bulk, using the bulk API `create` action. Elasticsearch Data Streams only support the `create` action. + + If the mode is set to `data_stream` and a `timestamp` field is present in a message, + Vector renames this field to the expected `@timestamp` to comply with the Elastic Common Schema. """ } } From 48fd7ea164009de1f6b302c74574779b0ef85a93 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Wed, 22 Jan 2025 16:35:05 -0500 Subject: [PATCH 017/658] fix(dev): make doc generation platform agnostic (#22223) * fix(dev): make doc generation platform agnostic * gate macro use * attempt to fix on windows * some good refactoring * another attempt for windows * more tweaks for windows * new workaround, revert unix.rs changes * doh, unix_datagram is not support on macos * allow cfg_if everywhere * Revert "allow cfg_if everywhere" This reverts commit 504e14fcafba1e01c7cb34ae30214b04abb9b4f2. * conditional include * cargo.lock bump --- Cargo.lock | 3 +- Cargo.toml | 1 + src/lib.rs | 3 ++ src/sinks/socket.rs | 78 +++++++++++++++++++++++++++++---------------- 4 files changed, 57 insertions(+), 28 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 37009f736e..5db3d7c754 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5549,7 +5549,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "libc", ] @@ -10987,6 +10987,7 @@ dependencies = [ "byteorder", "bytes 1.9.0", "bytesize", + "cfg-if", "chrono", "chrono-tz", "clap", diff --git a/Cargo.toml b/Cargo.toml index 9dfa65d5b8..c4ac0ffdf9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -177,6 +177,7 @@ vrl.workspace = true proptest = { workspace = true, optional = true } proptest-derive = { workspace = true, optional = true } snafu.workspace = true +cfg-if.workspace = true # Internal libs dnsmsg-parser = { path = "lib/dnsmsg-parser", optional = true } diff --git a/src/lib.rs b/src/lib.rs index 04048952af..59f91992a1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -21,6 +21,9 @@ //! The main library to support building Vector. +#[cfg(all(unix, feature = "sinks-socket"))] +#[macro_use] +extern crate cfg_if; #[macro_use] extern crate derivative; #[macro_use] diff --git a/src/sinks/socket.rs b/src/sinks/socket.rs index bf76916121..4768eede70 100644 --- a/src/sinks/socket.rs +++ b/src/sinks/socket.rs @@ -4,7 +4,7 @@ use vector_lib::codecs::{ }; use vector_lib::configurable::configurable_component; -#[cfg(unix)] +#[cfg(not(windows))] use crate::sinks::util::unix::UnixSinkConfig; use crate::{ codecs::{Encoder, EncodingConfig, EncodingConfigWithFraming, SinkType}, @@ -41,15 +41,12 @@ pub enum Mode { Udp(UdpMode), /// Send over a Unix domain socket (UDS), in stream mode. - #[cfg(unix)] #[serde(alias = "unix")] UnixStream(UnixMode), /// Send over a Unix domain socket (UDS), in datagram mode. /// Unavailable on macOS, due to send(2)'s apparent non-blocking behavior, /// resulting in ENOBUFS errors which we currently don't handle. - #[cfg(unix)] - #[cfg_attr(target_os = "macos", serde(skip))] UnixDatagram(UnixMode), } @@ -76,7 +73,6 @@ pub struct UdpMode { } /// Unix Domain Socket configuration. -#[cfg(unix)] #[configurable_component] #[derive(Clone, Debug)] pub struct UnixMode { @@ -87,6 +83,19 @@ pub struct UnixMode { encoding: EncodingConfigWithFraming, } +// Workaround for https://github.com/vectordotdev/vector/issues/22198. +#[cfg(windows)] +/// A Unix Domain Socket sink. +#[configurable_component] +#[derive(Clone, Debug)] +pub struct UnixSinkConfig { + /// The Unix socket path. + /// + /// This should be an absolute path. + #[configurable(metadata(docs::examples = "/path/to/socket"))] + pub path: std::path::PathBuf, +} + impl GenerateConfig for SocketSinkConfig { fn generate_config() -> toml::Value { toml::from_str( @@ -151,16 +160,28 @@ impl SinkConfig for SocketSinkConfig { super::util::service::net::UnixMode::Stream, ) } + #[allow(unused)] #[cfg(unix)] Mode::UnixDatagram(UnixMode { config, encoding }) => { - let transformer = encoding.transformer(); - let (framer, serializer) = encoding.build(SinkType::StreamBased)?; - let encoder = Encoder::::new(framer, serializer); - config.build( - transformer, - encoder, - super::util::service::net::UnixMode::Datagram, - ) + cfg_if! { + if #[cfg(not(target_os = "macos"))] { + let transformer = encoding.transformer(); + let (framer, serializer) = encoding.build(SinkType::StreamBased)?; + let encoder = Encoder::::new(framer, serializer); + config.build( + transformer, + encoder, + super::util::service::net::UnixMode::Datagram, + ) + } + else { + Err("UnixDatagram is not available on macOS platforms.".into()) + } + } + } + #[cfg(not(unix))] + Mode::UnixStream(_) | Mode::UnixDatagram(_) => { + Err("Unix modes are supported only on Unix platforms.".into()) } } } @@ -169,9 +190,7 @@ impl SinkConfig for SocketSinkConfig { let encoder_input_type = match &self.mode { Mode::Tcp(TcpMode { encoding, .. }) => encoding.config().1.input_type(), Mode::Udp(UdpMode { encoding, .. }) => encoding.config().input_type(), - #[cfg(unix)] Mode::UnixStream(UnixMode { encoding, .. }) => encoding.config().1.input_type(), - #[cfg(unix)] Mode::UnixDatagram(UnixMode { encoding, .. }) => encoding.config().1.input_type(), }; Input::new(encoder_input_type) @@ -188,8 +207,6 @@ mod test { future::ready, net::{SocketAddr, UdpSocket}, }; - #[cfg(unix)] - use std::{os::unix::net::UnixDatagram, path::PathBuf}; use futures::stream::StreamExt; use futures_util::stream; @@ -201,12 +218,19 @@ mod test { use tokio_stream::wrappers::TcpListenerStream; use tokio_util::codec::{FramedRead, LinesCodec}; use vector_lib::codecs::JsonSerializerConfig; - #[cfg(unix)] - use vector_lib::codecs::NativeJsonSerializerConfig; use super::*; - #[cfg(unix)] - use crate::test_util::random_metrics_with_stream; + + #[cfg(target_os = "windows")] + use cfg_if::cfg_if; + cfg_if! { if #[cfg(unix)] { + use vector_lib::codecs::NativeJsonSerializerConfig; + use crate::test_util::random_metrics_with_stream; + use std::path::PathBuf; + } } + #[cfg(all(unix, not(target_os = "macos")))] + use std::os::unix::net::UnixDatagram; + use crate::{ config::SinkContext, event::{Event, LogEvent}, @@ -223,20 +247,20 @@ mod test { enum DatagramSocket { Udp(UdpSocket), - #[cfg(unix)] + #[cfg(all(unix, not(target_os = "macos")))] Unix(UnixDatagram), } enum DatagramSocketAddr { Udp(SocketAddr), - #[cfg(unix)] + #[cfg(all(unix, not(target_os = "macos")))] Unix(PathBuf), } async fn test_datagram(datagram_addr: DatagramSocketAddr) { let receiver = match &datagram_addr { DatagramSocketAddr::Udp(addr) => DatagramSocket::Udp(UdpSocket::bind(addr).unwrap()), - #[cfg(unix)] + #[cfg(all(unix, not(target_os = "macos")))] DatagramSocketAddr::Unix(path) => { DatagramSocket::Unix(UnixDatagram::bind(path).unwrap()) } @@ -248,7 +272,7 @@ mod test { config: UdpSinkConfig::from_address(addr.to_string()), encoding: JsonSerializerConfig::default().into(), }), - #[cfg(unix)] + #[cfg(all(unix, not(target_os = "macos")))] DatagramSocketAddr::Unix(path) => Mode::UnixDatagram(UnixMode { config: UnixSinkConfig::new(path.to_path_buf()), encoding: (None::, JsonSerializerConfig::default()).into(), @@ -272,7 +296,7 @@ mod test { DatagramSocket::Udp(sock) => { sock.recv_from(&mut buf).expect("Did not receive message").0 } - #[cfg(unix)] + #[cfg(all(unix, not(target_os = "macos")))] DatagramSocket::Unix(sock) => sock.recv(&mut buf).expect("Did not receive message"), }; @@ -298,7 +322,7 @@ mod test { test_datagram(DatagramSocketAddr::Udp(next_addr_v6())).await; } - #[cfg(unix)] + #[cfg(all(unix, not(target_os = "macos")))] #[tokio::test] async fn unix_datagram() { trace_init(); From 21776bbd15113e34841dc6269358fc69b9166546 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Wed, 22 Jan 2025 16:43:26 -0500 Subject: [PATCH 018/658] chore(dev): bump bitflags (#22283) From 9161ca75417db37fad78848d31ca2a586321c2b2 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Wed, 22 Jan 2025 16:44:50 -0500 Subject: [PATCH 019/658] chore(dev): fix `vdev check scripts` warnings (#22277) * core(dev): fix vdev scripts warnings * use a more complex expression to evaluate retries * tweaks --- distribution/install.sh | 25 +++++++++++++------------ website/scripts/cue.sh | 4 ---- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/distribution/install.sh b/distribution/install.sh index c14713c482..b5404b40b8 100755 --- a/distribution/install.sh +++ b/distribution/install.sh @@ -594,21 +594,25 @@ downloader() { if [ "$1" = --check ]; then need_cmd "$_dld" elif [ "$_dld" = curl ]; then - check_curl_for_retry_support - _retry="$RETVAL" + if check_curl_for_retry_support; then + _retry=(--retry 3) + else + _retry=() + fi + get_ciphersuites_for_curl _ciphersuites="$RETVAL" if [ -n "$_ciphersuites" ]; then - _err=$(curl $_retry --proto '=https' --tlsv1.2 --ciphers "$_ciphersuites" --silent --show-error --fail --location "$1" --output "$2" 2>&1) + _err=$(curl "${_retry[@]}" --proto '=https' --tlsv1.2 --ciphers "$_ciphersuites" --silent --show-error --fail --location "$1" --output "$2" 2>&1) _status=$? else echo "Warning: Not enforcing strong cipher suites for TLS, this is potentially less secure" if ! check_help_for "$3" curl --proto --tlsv1.2; then echo "Warning: Not enforcing TLS v1.2, this is potentially less secure" - _err=$(curl $_retry --silent --show-error --fail --location "$1" --output "$2" 2>&1) + _err=$(curl "${_retry[@]}" --silent --show-error --fail --location "$1" --output "$2" 2>&1) _status=$? else - _err=$(curl $_retry --proto '=https' --tlsv1.2 --silent --show-error --fail --location "$1" --output "$2" 2>&1) + _err=$(curl "${_retry[@]}" --proto '=https' --tlsv1.2 --silent --show-error --fail --location "$1" --output "$2" 2>&1) _status=$? fi fi @@ -707,16 +711,13 @@ check_help_for() { true # not strictly needed } -# Check if curl supports the --retry flag, then pass it to the curl invocation. +# Check if curl supports the --retry flag check_curl_for_retry_support() { - local _retry_supported="" - # "unspecified" is for arch, allows for possibility old OS using macports, homebrew, etc. if check_help_for "notspecified" "curl" "--retry"; then - _retry_supported="--retry 3" + return 0 + else + return 1 fi - - RETVAL="$_retry_supported" - } # Return cipher suite string specified by user, otherwise return strong TLS 1.2-1.3 cipher suites diff --git a/website/scripts/cue.sh b/website/scripts/cue.sh index f257a54ea4..84fb172961 100755 --- a/website/scripts/cue.sh +++ b/website/scripts/cue.sh @@ -19,10 +19,6 @@ cmd_check() { cargo vdev check docs } -cmd_fmt() { - list-docs-files | xargs cue fmt "$@" -} - cmd_list() { list-docs-files } From 082a154e69c65166ed52f1a1b44deb9b067069e3 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Wed, 22 Jan 2025 17:12:03 -0500 Subject: [PATCH 020/658] chore(ci): display regression report in summary (#22284) * chore(ci): display regression report in summary * fix path * fix path --- .github/workflows/regression.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 1f425d11ff..f6b69c446d 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -547,6 +547,22 @@ jobs: env: FAILED: ${{ contains(needs.*.result, 'failure') }} steps: + - name: Download capture-artifacts + uses: actions/download-artifact@v4 + with: + name: capture-artifacts + path: downloads + + - name: Display Markdown Summary + run: | + unzip downloads/capture-artifacts -d results + REPORT_MD=results/report.md + if [ -f ${REPORT_MD} ]; then + cat ${REPORT_MD} >> $GITHUB_STEP_SUMMARY + else + echo "Did not find ${REPORT_MD} file." + fi + - name: exit run: | echo "failed=${{ env.FAILED }}" From 17d14d3b150569a87f7fec9288c16601ce0064de Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Wed, 22 Jan 2025 19:25:19 -0500 Subject: [PATCH 021/658] docs(external): improve proto codecs docs (#22280) * regen docs * regen again --- lib/codecs/src/decoding/format/protobuf.rs | 9 +++++++-- lib/codecs/src/encoding/format/protobuf.rs | 4 +++- .../reference/components/sinks/base/amqp.cue | 4 +++- .../sinks/base/aws_cloudwatch_logs.cue | 4 +++- .../sinks/base/aws_kinesis_firehose.cue | 4 +++- .../sinks/base/aws_kinesis_streams.cue | 4 +++- .../reference/components/sinks/base/aws_s3.cue | 4 +++- .../reference/components/sinks/base/aws_sns.cue | 4 +++- .../reference/components/sinks/base/aws_sqs.cue | 4 +++- .../components/sinks/base/azure_blob.cue | 4 +++- .../reference/components/sinks/base/console.cue | 4 +++- .../reference/components/sinks/base/file.cue | 4 +++- .../sinks/base/gcp_chronicle_unstructured.cue | 4 +++- .../components/sinks/base/gcp_cloud_storage.cue | 4 +++- .../components/sinks/base/gcp_pubsub.cue | 4 +++- .../reference/components/sinks/base/http.cue | 4 +++- .../components/sinks/base/humio_logs.cue | 4 +++- .../reference/components/sinks/base/kafka.cue | 4 +++- .../reference/components/sinks/base/loki.cue | 4 +++- .../reference/components/sinks/base/mqtt.cue | 4 +++- .../reference/components/sinks/base/nats.cue | 4 +++- .../components/sinks/base/opentelemetry.cue | 4 +++- .../components/sinks/base/papertrail.cue | 4 +++- .../reference/components/sinks/base/pulsar.cue | 4 +++- .../reference/components/sinks/base/redis.cue | 4 +++- .../reference/components/sinks/base/socket.cue | 4 +++- .../components/sinks/base/splunk_hec_logs.cue | 4 +++- .../reference/components/sinks/base/webhdfs.cue | 4 +++- .../components/sinks/base/websocket.cue | 4 +++- .../reference/components/sources/base/amqp.cue | 17 +++++++++++++---- .../sources/base/aws_kinesis_firehose.cue | 17 +++++++++++++---- .../components/sources/base/aws_s3.cue | 17 +++++++++++++---- .../components/sources/base/aws_sqs.cue | 17 +++++++++++++---- .../components/sources/base/datadog_agent.cue | 17 +++++++++++++---- .../components/sources/base/demo_logs.cue | 17 +++++++++++++---- .../reference/components/sources/base/exec.cue | 17 +++++++++++++---- .../components/sources/base/file_descriptor.cue | 17 +++++++++++++---- .../components/sources/base/gcp_pubsub.cue | 17 +++++++++++++---- .../components/sources/base/heroku_logs.cue | 17 +++++++++++++---- .../reference/components/sources/base/http.cue | 17 +++++++++++++---- .../components/sources/base/http_client.cue | 17 +++++++++++++---- .../components/sources/base/http_server.cue | 17 +++++++++++++---- .../reference/components/sources/base/kafka.cue | 17 +++++++++++++---- .../reference/components/sources/base/nats.cue | 17 +++++++++++++---- .../components/sources/base/pulsar.cue | 17 +++++++++++++---- .../reference/components/sources/base/redis.cue | 17 +++++++++++++---- .../components/sources/base/socket.cue | 17 +++++++++++++---- .../reference/components/sources/base/stdin.cue | 17 +++++++++++++---- 48 files changed, 338 insertions(+), 106 deletions(-) diff --git a/lib/codecs/src/decoding/format/protobuf.rs b/lib/codecs/src/decoding/format/protobuf.rs index 42f8665891..0a4552b076 100644 --- a/lib/codecs/src/decoding/format/protobuf.rs +++ b/lib/codecs/src/decoding/format/protobuf.rs @@ -66,10 +66,15 @@ impl ProtobufDeserializerConfig { #[derive(Debug, Clone, PartialEq, Eq, Derivative)] #[derivative(Default)] pub struct ProtobufDeserializerOptions { - /// Path to desc file + /// The path to the protobuf descriptor set file. + /// + /// This file is the output of `protoc -I -o ` + /// + /// You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). pub desc_file: PathBuf, - /// message type. e.g package.message + /// The name of the message type to use for serializing. + #[configurable(metadata(docs::examples = "package.Message"))] pub message_type: String, } diff --git a/lib/codecs/src/encoding/format/protobuf.rs b/lib/codecs/src/encoding/format/protobuf.rs index d685b68cf6..c014a52945 100644 --- a/lib/codecs/src/encoding/format/protobuf.rs +++ b/lib/codecs/src/encoding/format/protobuf.rs @@ -47,7 +47,9 @@ impl ProtobufSerializerConfig { pub struct ProtobufSerializerOptions { /// The path to the protobuf descriptor set file. /// - /// This file is the output of `protoc -o ...` + /// This file is the output of `protoc -I -o ` + /// + /// You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). #[configurable(metadata(docs::examples = "/etc/vector/protobuf_descriptor_set.desc"))] pub desc_file: PathBuf, diff --git a/website/cue/reference/components/sinks/base/amqp.cue b/website/cue/reference/components/sinks/base/amqp.cue index bc1e21950b..205ca852fa 100644 --- a/website/cue/reference/components/sinks/base/amqp.cue +++ b/website/cue/reference/components/sinks/base/amqp.cue @@ -359,7 +359,9 @@ base: components: sinks: amqp: configuration: { description: """ The path to the protobuf descriptor set file. - This file is the output of `protoc -o ...` + This file is the output of `protoc -I -o ` + + You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). """ required: true type: string: examples: ["/etc/vector/protobuf_descriptor_set.desc"] diff --git a/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue b/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue index b9209542d5..be85c3848e 100644 --- a/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue +++ b/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue @@ -547,7 +547,9 @@ base: components: sinks: aws_cloudwatch_logs: configuration: { description: """ The path to the protobuf descriptor set file. - This file is the output of `protoc -o ...` + This file is the output of `protoc -I -o ` + + You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). """ required: true type: string: examples: ["/etc/vector/protobuf_descriptor_set.desc"] diff --git a/website/cue/reference/components/sinks/base/aws_kinesis_firehose.cue b/website/cue/reference/components/sinks/base/aws_kinesis_firehose.cue index 0eb16fad8f..5109d693a5 100644 --- a/website/cue/reference/components/sinks/base/aws_kinesis_firehose.cue +++ b/website/cue/reference/components/sinks/base/aws_kinesis_firehose.cue @@ -526,7 +526,9 @@ base: components: sinks: aws_kinesis_firehose: configuration: { description: """ The path to the protobuf descriptor set file. - This file is the output of `protoc -o ...` + This file is the output of `protoc -I -o ` + + You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). """ required: true type: string: examples: ["/etc/vector/protobuf_descriptor_set.desc"] diff --git a/website/cue/reference/components/sinks/base/aws_kinesis_streams.cue b/website/cue/reference/components/sinks/base/aws_kinesis_streams.cue index 34696bbfa1..6161ffbc07 100644 --- a/website/cue/reference/components/sinks/base/aws_kinesis_streams.cue +++ b/website/cue/reference/components/sinks/base/aws_kinesis_streams.cue @@ -526,7 +526,9 @@ base: components: sinks: aws_kinesis_streams: configuration: { description: """ The path to the protobuf descriptor set file. - This file is the output of `protoc -o ...` + This file is the output of `protoc -I -o ` + + You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). """ required: true type: string: examples: ["/etc/vector/protobuf_descriptor_set.desc"] diff --git a/website/cue/reference/components/sinks/base/aws_s3.cue b/website/cue/reference/components/sinks/base/aws_s3.cue index b916893922..fa9357214a 100644 --- a/website/cue/reference/components/sinks/base/aws_s3.cue +++ b/website/cue/reference/components/sinks/base/aws_s3.cue @@ -635,7 +635,9 @@ base: components: sinks: aws_s3: configuration: { description: """ The path to the protobuf descriptor set file. - This file is the output of `protoc -o ...` + This file is the output of `protoc -I -o ` + + You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). """ required: true type: string: examples: ["/etc/vector/protobuf_descriptor_set.desc"] diff --git a/website/cue/reference/components/sinks/base/aws_sns.cue b/website/cue/reference/components/sinks/base/aws_sns.cue index 27866ff512..073d18d3b2 100644 --- a/website/cue/reference/components/sinks/base/aws_sns.cue +++ b/website/cue/reference/components/sinks/base/aws_sns.cue @@ -457,7 +457,9 @@ base: components: sinks: aws_sns: configuration: { description: """ The path to the protobuf descriptor set file. - This file is the output of `protoc -o ...` + This file is the output of `protoc -I -o ` + + You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). """ required: true type: string: examples: ["/etc/vector/protobuf_descriptor_set.desc"] diff --git a/website/cue/reference/components/sinks/base/aws_sqs.cue b/website/cue/reference/components/sinks/base/aws_sqs.cue index 4928c77fab..e563600835 100644 --- a/website/cue/reference/components/sinks/base/aws_sqs.cue +++ b/website/cue/reference/components/sinks/base/aws_sqs.cue @@ -457,7 +457,9 @@ base: components: sinks: aws_sqs: configuration: { description: """ The path to the protobuf descriptor set file. - This file is the output of `protoc -o ...` + This file is the output of `protoc -I -o ` + + You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). """ required: true type: string: examples: ["/etc/vector/protobuf_descriptor_set.desc"] diff --git a/website/cue/reference/components/sinks/base/azure_blob.cue b/website/cue/reference/components/sinks/base/azure_blob.cue index 7368d87e3b..7264702c37 100644 --- a/website/cue/reference/components/sinks/base/azure_blob.cue +++ b/website/cue/reference/components/sinks/base/azure_blob.cue @@ -491,7 +491,9 @@ base: components: sinks: azure_blob: configuration: { description: """ The path to the protobuf descriptor set file. - This file is the output of `protoc -o ...` + This file is the output of `protoc -I -o ` + + You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). """ required: true type: string: examples: ["/etc/vector/protobuf_descriptor_set.desc"] diff --git a/website/cue/reference/components/sinks/base/console.cue b/website/cue/reference/components/sinks/base/console.cue index f0726c151e..89e9fc8f1d 100644 --- a/website/cue/reference/components/sinks/base/console.cue +++ b/website/cue/reference/components/sinks/base/console.cue @@ -343,7 +343,9 @@ base: components: sinks: console: configuration: { description: """ The path to the protobuf descriptor set file. - This file is the output of `protoc -o ...` + This file is the output of `protoc -I -o ` + + You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). """ required: true type: string: examples: ["/etc/vector/protobuf_descriptor_set.desc"] diff --git a/website/cue/reference/components/sinks/base/file.cue b/website/cue/reference/components/sinks/base/file.cue index 67da3eb264..0296da9053 100644 --- a/website/cue/reference/components/sinks/base/file.cue +++ b/website/cue/reference/components/sinks/base/file.cue @@ -363,7 +363,9 @@ base: components: sinks: file: configuration: { description: """ The path to the protobuf descriptor set file. - This file is the output of `protoc -o ...` + This file is the output of `protoc -I -o ` + + You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). """ required: true type: string: examples: ["/etc/vector/protobuf_descriptor_set.desc"] diff --git a/website/cue/reference/components/sinks/base/gcp_chronicle_unstructured.cue b/website/cue/reference/components/sinks/base/gcp_chronicle_unstructured.cue index c6ab40b56d..8c7b2f9b81 100644 --- a/website/cue/reference/components/sinks/base/gcp_chronicle_unstructured.cue +++ b/website/cue/reference/components/sinks/base/gcp_chronicle_unstructured.cue @@ -431,7 +431,9 @@ base: components: sinks: gcp_chronicle_unstructured: configuration: { description: """ The path to the protobuf descriptor set file. - This file is the output of `protoc -o ...` + This file is the output of `protoc -I -o ` + + You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). """ required: true type: string: examples: ["/etc/vector/protobuf_descriptor_set.desc"] diff --git a/website/cue/reference/components/sinks/base/gcp_cloud_storage.cue b/website/cue/reference/components/sinks/base/gcp_cloud_storage.cue index fa318427db..e423242f4b 100644 --- a/website/cue/reference/components/sinks/base/gcp_cloud_storage.cue +++ b/website/cue/reference/components/sinks/base/gcp_cloud_storage.cue @@ -504,7 +504,9 @@ base: components: sinks: gcp_cloud_storage: configuration: { description: """ The path to the protobuf descriptor set file. - This file is the output of `protoc -o ...` + This file is the output of `protoc -I -o ` + + You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). """ required: true type: string: examples: ["/etc/vector/protobuf_descriptor_set.desc"] diff --git a/website/cue/reference/components/sinks/base/gcp_pubsub.cue b/website/cue/reference/components/sinks/base/gcp_pubsub.cue index 03fb5a409e..bbafe257b1 100644 --- a/website/cue/reference/components/sinks/base/gcp_pubsub.cue +++ b/website/cue/reference/components/sinks/base/gcp_pubsub.cue @@ -410,7 +410,9 @@ base: components: sinks: gcp_pubsub: configuration: { description: """ The path to the protobuf descriptor set file. - This file is the output of `protoc -o ...` + This file is the output of `protoc -I -o ` + + You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). """ required: true type: string: examples: ["/etc/vector/protobuf_descriptor_set.desc"] diff --git a/website/cue/reference/components/sinks/base/http.cue b/website/cue/reference/components/sinks/base/http.cue index d46c88e890..75c17f4d50 100644 --- a/website/cue/reference/components/sinks/base/http.cue +++ b/website/cue/reference/components/sinks/base/http.cue @@ -456,7 +456,9 @@ base: components: sinks: http: configuration: { description: """ The path to the protobuf descriptor set file. - This file is the output of `protoc -o ...` + This file is the output of `protoc -I -o ` + + You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). """ required: true type: string: examples: ["/etc/vector/protobuf_descriptor_set.desc"] diff --git a/website/cue/reference/components/sinks/base/humio_logs.cue b/website/cue/reference/components/sinks/base/humio_logs.cue index 1c4255d45f..2c02c9258a 100644 --- a/website/cue/reference/components/sinks/base/humio_logs.cue +++ b/website/cue/reference/components/sinks/base/humio_logs.cue @@ -409,7 +409,9 @@ base: components: sinks: humio_logs: configuration: { description: """ The path to the protobuf descriptor set file. - This file is the output of `protoc -o ...` + This file is the output of `protoc -I -o ` + + You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). """ required: true type: string: examples: ["/etc/vector/protobuf_descriptor_set.desc"] diff --git a/website/cue/reference/components/sinks/base/kafka.cue b/website/cue/reference/components/sinks/base/kafka.cue index 35c7311dcc..13a41fa849 100644 --- a/website/cue/reference/components/sinks/base/kafka.cue +++ b/website/cue/reference/components/sinks/base/kafka.cue @@ -398,7 +398,9 @@ base: components: sinks: kafka: configuration: { description: """ The path to the protobuf descriptor set file. - This file is the output of `protoc -o ...` + This file is the output of `protoc -I -o ` + + You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). """ required: true type: string: examples: ["/etc/vector/protobuf_descriptor_set.desc"] diff --git a/website/cue/reference/components/sinks/base/loki.cue b/website/cue/reference/components/sinks/base/loki.cue index 26fe97cf75..622fc70f05 100644 --- a/website/cue/reference/components/sinks/base/loki.cue +++ b/website/cue/reference/components/sinks/base/loki.cue @@ -458,7 +458,9 @@ base: components: sinks: loki: configuration: { description: """ The path to the protobuf descriptor set file. - This file is the output of `protoc -o ...` + This file is the output of `protoc -I -o ` + + You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). """ required: true type: string: examples: ["/etc/vector/protobuf_descriptor_set.desc"] diff --git a/website/cue/reference/components/sinks/base/mqtt.cue b/website/cue/reference/components/sinks/base/mqtt.cue index 5b34016ed5..544fb6b123 100644 --- a/website/cue/reference/components/sinks/base/mqtt.cue +++ b/website/cue/reference/components/sinks/base/mqtt.cue @@ -353,7 +353,9 @@ base: components: sinks: mqtt: configuration: { description: """ The path to the protobuf descriptor set file. - This file is the output of `protoc -o ...` + This file is the output of `protoc -I -o ` + + You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). """ required: true type: string: examples: ["/etc/vector/protobuf_descriptor_set.desc"] diff --git a/website/cue/reference/components/sinks/base/nats.cue b/website/cue/reference/components/sinks/base/nats.cue index 9670422c6a..7dd4e20f69 100644 --- a/website/cue/reference/components/sinks/base/nats.cue +++ b/website/cue/reference/components/sinks/base/nats.cue @@ -443,7 +443,9 @@ base: components: sinks: nats: configuration: { description: """ The path to the protobuf descriptor set file. - This file is the output of `protoc -o ...` + This file is the output of `protoc -I -o ` + + You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). """ required: true type: string: examples: ["/etc/vector/protobuf_descriptor_set.desc"] diff --git a/website/cue/reference/components/sinks/base/opentelemetry.cue b/website/cue/reference/components/sinks/base/opentelemetry.cue index 62d4cf0712..8a13eeb34e 100644 --- a/website/cue/reference/components/sinks/base/opentelemetry.cue +++ b/website/cue/reference/components/sinks/base/opentelemetry.cue @@ -459,7 +459,9 @@ base: components: sinks: opentelemetry: configuration: protocol: { description: """ The path to the protobuf descriptor set file. - This file is the output of `protoc -o ...` + This file is the output of `protoc -I -o ` + + You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). """ required: true type: string: examples: ["/etc/vector/protobuf_descriptor_set.desc"] diff --git a/website/cue/reference/components/sinks/base/papertrail.cue b/website/cue/reference/components/sinks/base/papertrail.cue index a9f6c222b6..58ffad96df 100644 --- a/website/cue/reference/components/sinks/base/papertrail.cue +++ b/website/cue/reference/components/sinks/base/papertrail.cue @@ -343,7 +343,9 @@ base: components: sinks: papertrail: configuration: { description: """ The path to the protobuf descriptor set file. - This file is the output of `protoc -o ...` + This file is the output of `protoc -I -o ` + + You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). """ required: true type: string: examples: ["/etc/vector/protobuf_descriptor_set.desc"] diff --git a/website/cue/reference/components/sinks/base/pulsar.cue b/website/cue/reference/components/sinks/base/pulsar.cue index aa19003c3b..2ccc405a7a 100644 --- a/website/cue/reference/components/sinks/base/pulsar.cue +++ b/website/cue/reference/components/sinks/base/pulsar.cue @@ -477,7 +477,9 @@ base: components: sinks: pulsar: configuration: { description: """ The path to the protobuf descriptor set file. - This file is the output of `protoc -o ...` + This file is the output of `protoc -I -o ` + + You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). """ required: true type: string: examples: ["/etc/vector/protobuf_descriptor_set.desc"] diff --git a/website/cue/reference/components/sinks/base/redis.cue b/website/cue/reference/components/sinks/base/redis.cue index f9b0ea210a..903e947dee 100644 --- a/website/cue/reference/components/sinks/base/redis.cue +++ b/website/cue/reference/components/sinks/base/redis.cue @@ -396,7 +396,9 @@ base: components: sinks: redis: configuration: { description: """ The path to the protobuf descriptor set file. - This file is the output of `protoc -o ...` + This file is the output of `protoc -I -o ` + + You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). """ required: true type: string: examples: ["/etc/vector/protobuf_descriptor_set.desc"] diff --git a/website/cue/reference/components/sinks/base/socket.cue b/website/cue/reference/components/sinks/base/socket.cue index 3561d292e6..25a222adab 100644 --- a/website/cue/reference/components/sinks/base/socket.cue +++ b/website/cue/reference/components/sinks/base/socket.cue @@ -355,7 +355,9 @@ base: components: sinks: socket: configuration: { description: """ The path to the protobuf descriptor set file. - This file is the output of `protoc -o ...` + This file is the output of `protoc -I -o ` + + You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). """ required: true type: string: examples: ["/etc/vector/protobuf_descriptor_set.desc"] diff --git a/website/cue/reference/components/sinks/base/splunk_hec_logs.cue b/website/cue/reference/components/sinks/base/splunk_hec_logs.cue index 75ace3211d..7313ce5e1c 100644 --- a/website/cue/reference/components/sinks/base/splunk_hec_logs.cue +++ b/website/cue/reference/components/sinks/base/splunk_hec_logs.cue @@ -459,7 +459,9 @@ base: components: sinks: splunk_hec_logs: configuration: { description: """ The path to the protobuf descriptor set file. - This file is the output of `protoc -o ...` + This file is the output of `protoc -I -o ` + + You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). """ required: true type: string: examples: ["/etc/vector/protobuf_descriptor_set.desc"] diff --git a/website/cue/reference/components/sinks/base/webhdfs.cue b/website/cue/reference/components/sinks/base/webhdfs.cue index 7a12c2060e..85aeb1497b 100644 --- a/website/cue/reference/components/sinks/base/webhdfs.cue +++ b/website/cue/reference/components/sinks/base/webhdfs.cue @@ -409,7 +409,9 @@ base: components: sinks: webhdfs: configuration: { description: """ The path to the protobuf descriptor set file. - This file is the output of `protoc -o ...` + This file is the output of `protoc -I -o ` + + You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). """ required: true type: string: examples: ["/etc/vector/protobuf_descriptor_set.desc"] diff --git a/website/cue/reference/components/sinks/base/websocket.cue b/website/cue/reference/components/sinks/base/websocket.cue index 1cbb811dc9..ed28d5e3ae 100644 --- a/website/cue/reference/components/sinks/base/websocket.cue +++ b/website/cue/reference/components/sinks/base/websocket.cue @@ -390,7 +390,9 @@ base: components: sinks: websocket: configuration: { description: """ The path to the protobuf descriptor set file. - This file is the output of `protoc -o ...` + This file is the output of `protoc -I -o ` + + You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). """ required: true type: string: examples: ["/etc/vector/protobuf_descriptor_set.desc"] diff --git a/website/cue/reference/components/sources/base/amqp.cue b/website/cue/reference/components/sources/base/amqp.cue index d0ebadf50d..1a6fd003c6 100644 --- a/website/cue/reference/components/sources/base/amqp.cue +++ b/website/cue/reference/components/sources/base/amqp.cue @@ -226,14 +226,23 @@ base: components: sources: amqp: configuration: { required: false type: object: options: { desc_file: { - description: "Path to desc file" - required: false + description: """ + The path to the protobuf descriptor set file. + + This file is the output of `protoc -I -o ` + + You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). + """ + required: false type: string: default: "" } message_type: { - description: "message type. e.g package.message" + description: "The name of the message type to use for serializing." required: false - type: string: default: "" + type: string: { + default: "" + examples: ["package.Message"] + } } } } diff --git a/website/cue/reference/components/sources/base/aws_kinesis_firehose.cue b/website/cue/reference/components/sources/base/aws_kinesis_firehose.cue index f00dc9c67b..e580e602fc 100644 --- a/website/cue/reference/components/sources/base/aws_kinesis_firehose.cue +++ b/website/cue/reference/components/sources/base/aws_kinesis_firehose.cue @@ -229,14 +229,23 @@ base: components: sources: aws_kinesis_firehose: configuration: { required: false type: object: options: { desc_file: { - description: "Path to desc file" - required: false + description: """ + The path to the protobuf descriptor set file. + + This file is the output of `protoc -I -o ` + + You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). + """ + required: false type: string: default: "" } message_type: { - description: "message type. e.g package.message" + description: "The name of the message type to use for serializing." required: false - type: string: default: "" + type: string: { + default: "" + examples: ["package.Message"] + } } } } diff --git a/website/cue/reference/components/sources/base/aws_s3.cue b/website/cue/reference/components/sources/base/aws_s3.cue index d0787ebc6d..0bf2a88959 100644 --- a/website/cue/reference/components/sources/base/aws_s3.cue +++ b/website/cue/reference/components/sources/base/aws_s3.cue @@ -336,14 +336,23 @@ base: components: sources: aws_s3: configuration: { required: false type: object: options: { desc_file: { - description: "Path to desc file" - required: false + description: """ + The path to the protobuf descriptor set file. + + This file is the output of `protoc -I -o ` + + You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). + """ + required: false type: string: default: "" } message_type: { - description: "message type. e.g package.message" + description: "The name of the message type to use for serializing." required: false - type: string: default: "" + type: string: { + default: "" + examples: ["package.Message"] + } } } } diff --git a/website/cue/reference/components/sources/base/aws_sqs.cue b/website/cue/reference/components/sources/base/aws_sqs.cue index f8d9e4663e..96ee464464 100644 --- a/website/cue/reference/components/sources/base/aws_sqs.cue +++ b/website/cue/reference/components/sources/base/aws_sqs.cue @@ -331,14 +331,23 @@ base: components: sources: aws_sqs: configuration: { required: false type: object: options: { desc_file: { - description: "Path to desc file" - required: false + description: """ + The path to the protobuf descriptor set file. + + This file is the output of `protoc -I -o ` + + You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). + """ + required: false type: string: default: "" } message_type: { - description: "message type. e.g package.message" + description: "The name of the message type to use for serializing." required: false - type: string: default: "" + type: string: { + default: "" + examples: ["package.Message"] + } } } } diff --git a/website/cue/reference/components/sources/base/datadog_agent.cue b/website/cue/reference/components/sources/base/datadog_agent.cue index 60bb9aa6e8..d2185646d3 100644 --- a/website/cue/reference/components/sources/base/datadog_agent.cue +++ b/website/cue/reference/components/sources/base/datadog_agent.cue @@ -211,14 +211,23 @@ base: components: sources: datadog_agent: configuration: { required: false type: object: options: { desc_file: { - description: "Path to desc file" - required: false + description: """ + The path to the protobuf descriptor set file. + + This file is the output of `protoc -I -o ` + + You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). + """ + required: false type: string: default: "" } message_type: { - description: "message type. e.g package.message" + description: "The name of the message type to use for serializing." required: false - type: string: default: "" + type: string: { + default: "" + examples: ["package.Message"] + } } } } diff --git a/website/cue/reference/components/sources/base/demo_logs.cue b/website/cue/reference/components/sources/base/demo_logs.cue index b7af9828fb..e1cf5b972a 100644 --- a/website/cue/reference/components/sources/base/demo_logs.cue +++ b/website/cue/reference/components/sources/base/demo_logs.cue @@ -190,14 +190,23 @@ base: components: sources: demo_logs: configuration: { required: false type: object: options: { desc_file: { - description: "Path to desc file" - required: false + description: """ + The path to the protobuf descriptor set file. + + This file is the output of `protoc -I -o ` + + You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). + """ + required: false type: string: default: "" } message_type: { - description: "message type. e.g package.message" + description: "The name of the message type to use for serializing." required: false - type: string: default: "" + type: string: { + default: "" + examples: ["package.Message"] + } } } } diff --git a/website/cue/reference/components/sources/base/exec.cue b/website/cue/reference/components/sources/base/exec.cue index ad62978009..c51784220d 100644 --- a/website/cue/reference/components/sources/base/exec.cue +++ b/website/cue/reference/components/sources/base/exec.cue @@ -191,14 +191,23 @@ base: components: sources: exec: configuration: { required: false type: object: options: { desc_file: { - description: "Path to desc file" - required: false + description: """ + The path to the protobuf descriptor set file. + + This file is the output of `protoc -I -o ` + + You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). + """ + required: false type: string: default: "" } message_type: { - description: "message type. e.g package.message" + description: "The name of the message type to use for serializing." required: false - type: string: default: "" + type: string: { + default: "" + examples: ["package.Message"] + } } } } diff --git a/website/cue/reference/components/sources/base/file_descriptor.cue b/website/cue/reference/components/sources/base/file_descriptor.cue index 23124afa6a..855b9d4570 100644 --- a/website/cue/reference/components/sources/base/file_descriptor.cue +++ b/website/cue/reference/components/sources/base/file_descriptor.cue @@ -181,14 +181,23 @@ base: components: sources: file_descriptor: configuration: { required: false type: object: options: { desc_file: { - description: "Path to desc file" - required: false + description: """ + The path to the protobuf descriptor set file. + + This file is the output of `protoc -I -o ` + + You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). + """ + required: false type: string: default: "" } message_type: { - description: "message type. e.g package.message" + description: "The name of the message type to use for serializing." required: false - type: string: default: "" + type: string: { + default: "" + examples: ["package.Message"] + } } } } diff --git a/website/cue/reference/components/sources/base/gcp_pubsub.cue b/website/cue/reference/components/sources/base/gcp_pubsub.cue index 55b16cdfbc..d2b4b288d8 100644 --- a/website/cue/reference/components/sources/base/gcp_pubsub.cue +++ b/website/cue/reference/components/sources/base/gcp_pubsub.cue @@ -257,14 +257,23 @@ base: components: sources: gcp_pubsub: configuration: { required: false type: object: options: { desc_file: { - description: "Path to desc file" - required: false + description: """ + The path to the protobuf descriptor set file. + + This file is the output of `protoc -I -o ` + + You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). + """ + required: false type: string: default: "" } message_type: { - description: "message type. e.g package.message" + description: "The name of the message type to use for serializing." required: false - type: string: default: "" + type: string: { + default: "" + examples: ["package.Message"] + } } } } diff --git a/website/cue/reference/components/sources/base/heroku_logs.cue b/website/cue/reference/components/sources/base/heroku_logs.cue index 418f61da96..c7452de59a 100644 --- a/website/cue/reference/components/sources/base/heroku_logs.cue +++ b/website/cue/reference/components/sources/base/heroku_logs.cue @@ -223,14 +223,23 @@ base: components: sources: heroku_logs: configuration: { required: false type: object: options: { desc_file: { - description: "Path to desc file" - required: false + description: """ + The path to the protobuf descriptor set file. + + This file is the output of `protoc -I -o ` + + You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). + """ + required: false type: string: default: "" } message_type: { - description: "message type. e.g package.message" + description: "The name of the message type to use for serializing." required: false - type: string: default: "" + type: string: { + default: "" + examples: ["package.Message"] + } } } } diff --git a/website/cue/reference/components/sources/base/http.cue b/website/cue/reference/components/sources/base/http.cue index d53ca9b9f9..d098558e54 100644 --- a/website/cue/reference/components/sources/base/http.cue +++ b/website/cue/reference/components/sources/base/http.cue @@ -224,14 +224,23 @@ base: components: sources: http: configuration: { required: false type: object: options: { desc_file: { - description: "Path to desc file" - required: false + description: """ + The path to the protobuf descriptor set file. + + This file is the output of `protoc -I -o ` + + You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). + """ + required: false type: string: default: "" } message_type: { - description: "message type. e.g package.message" + description: "The name of the message type to use for serializing." required: false - type: string: default: "" + type: string: { + default: "" + examples: ["package.Message"] + } } } } diff --git a/website/cue/reference/components/sources/base/http_client.cue b/website/cue/reference/components/sources/base/http_client.cue index 609949072f..666ecc5d13 100644 --- a/website/cue/reference/components/sources/base/http_client.cue +++ b/website/cue/reference/components/sources/base/http_client.cue @@ -223,14 +223,23 @@ base: components: sources: http_client: configuration: { required: false type: object: options: { desc_file: { - description: "Path to desc file" - required: false + description: """ + The path to the protobuf descriptor set file. + + This file is the output of `protoc -I -o ` + + You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). + """ + required: false type: string: default: "" } message_type: { - description: "message type. e.g package.message" + description: "The name of the message type to use for serializing." required: false - type: string: default: "" + type: string: { + default: "" + examples: ["package.Message"] + } } } } diff --git a/website/cue/reference/components/sources/base/http_server.cue b/website/cue/reference/components/sources/base/http_server.cue index 543a97a42c..f28f0ff43a 100644 --- a/website/cue/reference/components/sources/base/http_server.cue +++ b/website/cue/reference/components/sources/base/http_server.cue @@ -224,14 +224,23 @@ base: components: sources: http_server: configuration: { required: false type: object: options: { desc_file: { - description: "Path to desc file" - required: false + description: """ + The path to the protobuf descriptor set file. + + This file is the output of `protoc -I -o ` + + You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). + """ + required: false type: string: default: "" } message_type: { - description: "message type. e.g package.message" + description: "The name of the message type to use for serializing." required: false - type: string: default: "" + type: string: { + default: "" + examples: ["package.Message"] + } } } } diff --git a/website/cue/reference/components/sources/base/kafka.cue b/website/cue/reference/components/sources/base/kafka.cue index 6e96d91f5f..54eb104ef7 100644 --- a/website/cue/reference/components/sources/base/kafka.cue +++ b/website/cue/reference/components/sources/base/kafka.cue @@ -235,14 +235,23 @@ base: components: sources: kafka: configuration: { required: false type: object: options: { desc_file: { - description: "Path to desc file" - required: false + description: """ + The path to the protobuf descriptor set file. + + This file is the output of `protoc -I -o ` + + You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). + """ + required: false type: string: default: "" } message_type: { - description: "message type. e.g package.message" + description: "The name of the message type to use for serializing." required: false - type: string: default: "" + type: string: { + default: "" + examples: ["package.Message"] + } } } } diff --git a/website/cue/reference/components/sources/base/nats.cue b/website/cue/reference/components/sources/base/nats.cue index 40516215de..6ac4de9e9f 100644 --- a/website/cue/reference/components/sources/base/nats.cue +++ b/website/cue/reference/components/sources/base/nats.cue @@ -278,14 +278,23 @@ base: components: sources: nats: configuration: { required: false type: object: options: { desc_file: { - description: "Path to desc file" - required: false + description: """ + The path to the protobuf descriptor set file. + + This file is the output of `protoc -I -o ` + + You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). + """ + required: false type: string: default: "" } message_type: { - description: "message type. e.g package.message" + description: "The name of the message type to use for serializing." required: false - type: string: default: "" + type: string: { + default: "" + examples: ["package.Message"] + } } } } diff --git a/website/cue/reference/components/sources/base/pulsar.cue b/website/cue/reference/components/sources/base/pulsar.cue index 881d4d7f29..acd3153311 100644 --- a/website/cue/reference/components/sources/base/pulsar.cue +++ b/website/cue/reference/components/sources/base/pulsar.cue @@ -284,14 +284,23 @@ base: components: sources: pulsar: configuration: { required: false type: object: options: { desc_file: { - description: "Path to desc file" - required: false + description: """ + The path to the protobuf descriptor set file. + + This file is the output of `protoc -I -o ` + + You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). + """ + required: false type: string: default: "" } message_type: { - description: "message type. e.g package.message" + description: "The name of the message type to use for serializing." required: false - type: string: default: "" + type: string: { + default: "" + examples: ["package.Message"] + } } } } diff --git a/website/cue/reference/components/sources/base/redis.cue b/website/cue/reference/components/sources/base/redis.cue index d9f167d400..3c88ae6163 100644 --- a/website/cue/reference/components/sources/base/redis.cue +++ b/website/cue/reference/components/sources/base/redis.cue @@ -196,14 +196,23 @@ base: components: sources: redis: configuration: { required: false type: object: options: { desc_file: { - description: "Path to desc file" - required: false + description: """ + The path to the protobuf descriptor set file. + + This file is the output of `protoc -I -o ` + + You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). + """ + required: false type: string: default: "" } message_type: { - description: "message type. e.g package.message" + description: "The name of the message type to use for serializing." required: false - type: string: default: "" + type: string: { + default: "" + examples: ["package.Message"] + } } } } diff --git a/website/cue/reference/components/sources/base/socket.cue b/website/cue/reference/components/sources/base/socket.cue index dc338ced7c..d56b9458a2 100644 --- a/website/cue/reference/components/sources/base/socket.cue +++ b/website/cue/reference/components/sources/base/socket.cue @@ -198,14 +198,23 @@ base: components: sources: socket: configuration: { required: false type: object: options: { desc_file: { - description: "Path to desc file" - required: false + description: """ + The path to the protobuf descriptor set file. + + This file is the output of `protoc -I -o ` + + You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). + """ + required: false type: string: default: "" } message_type: { - description: "message type. e.g package.message" + description: "The name of the message type to use for serializing." required: false - type: string: default: "" + type: string: { + default: "" + examples: ["package.Message"] + } } } } diff --git a/website/cue/reference/components/sources/base/stdin.cue b/website/cue/reference/components/sources/base/stdin.cue index ded73f44e9..971559f73e 100644 --- a/website/cue/reference/components/sources/base/stdin.cue +++ b/website/cue/reference/components/sources/base/stdin.cue @@ -181,14 +181,23 @@ base: components: sources: stdin: configuration: { required: false type: object: options: { desc_file: { - description: "Path to desc file" - required: false + description: """ + The path to the protobuf descriptor set file. + + This file is the output of `protoc -I -o ` + + You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). + """ + required: false type: string: default: "" } message_type: { - description: "message type. e.g package.message" + description: "The name of the message type to use for serializing." required: false - type: string: default: "" + type: string: { + default: "" + examples: ["package.Message"] + } } } } From b890bf6b7c855cd1970a22f8a28ba9f2872eaaea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Sampedro?= <5142014+rsrdesarrollo@users.noreply.github.com> Date: Thu, 23 Jan 2025 15:40:47 +0100 Subject: [PATCH 022/658] fix(deployment): #22264 add `--no-environment` to systemd service file at `ExecReload` (#22279) * fix #22264 * add changelog --- changelog.d/22279_systemd_service_reload.fix.md | 3 +++ distribution/systemd/vector.service | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 changelog.d/22279_systemd_service_reload.fix.md diff --git a/changelog.d/22279_systemd_service_reload.fix.md b/changelog.d/22279_systemd_service_reload.fix.md new file mode 100644 index 0000000000..1d9c1febb7 --- /dev/null +++ b/changelog.d/22279_systemd_service_reload.fix.md @@ -0,0 +1,3 @@ +The systemd service now validates the config with parameter `--no-environment` on service reload. + +authors: rsrdesarrollo diff --git a/distribution/systemd/vector.service b/distribution/systemd/vector.service index d9c6e78431..b78ff3131e 100644 --- a/distribution/systemd/vector.service +++ b/distribution/systemd/vector.service @@ -9,7 +9,7 @@ User=vector Group=vector ExecStartPre=/usr/bin/vector validate ExecStart=/usr/bin/vector -ExecReload=/usr/bin/vector validate +ExecReload=/usr/bin/vector validate --no-environment ExecReload=/bin/kill -HUP $MAINPID Restart=always AmbientCapabilities=CAP_NET_BIND_SERVICE From 318930b819872ce12c23bbee8475aafda1deeedb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ensar=20Saraj=C4=8Di=C4=87?= Date: Thu, 23 Jan 2025 16:23:16 +0100 Subject: [PATCH 023/658] feat(enriching): add `memory` enrichment table (#21348) * feat(vrl): add caching feature for VRL This adds additional VRL functions for reading and storing data into caches that can be configured in global options. Caches can store any VRL value and are meant to store data for shorter periods. All data gets TTL (time-to-live) assigned, based on cache configuration and gets removed when that TTL expires. * Add reads and writes metrics for VRL cache * Rename `cache_set` to `cache_put` * Add `cache_delete` VRL function * Add key not found test case for `cache_get` * Add placeholder implementation for fetching TTL using `cache_get` * Add memory enrichment table * Add basic sink implementation for memory enrichment table This implementation is based on `evmap`, for "lock-free" reading and writing. There is still a lock when data is refreshed, but that can be controlled, to have less interruptions. * Add ttl, refresh and scan intervals implementation for memory enrichment table * Add internal events for memory enrichment table * Remove initial implementation of VRL cache * Remove table name from memory enrichment table events * Add `SinkConfig` impl for `MemoryConfig` This adds a `SinkConfig` implementation for `MemoryConfig`, making it a shared configuration for both enrichment table component and the sink component. To ensure that the exact same `Memory` instance is used, created `Memory` is cached inside the config instance and then shared whenever one of the `build` variants are invoked. `Memory` is already built with sharing between different instances in mind, since it had to be shared across different threads and it is cloneable while keeping the same backing structure for data. The actual sink building is a placeholder. * Hook up enrichment tables as sinks when possible * Fix flushing in memory enrichment table * Fix failing memory table tests * Remove enrichment_tables from Graph and use them as sinks * Wrap memory metadata in a single object for easier mutex usage * Add byte size limit to memory enrichment table * Remove unnecessary duplicated key from memory table entries * Remove debugging log from memory table * Ensure `ConfigDiff` takes tables into account * Implement running topology changes for enrichment_tables like sinks * Make memory tables visible in `vector top` as sinks * Remove unnecessary clone when handling events * Fix tests after removing clones in `handle_value` * Reduce key clones when writing to memory table * Store data in memory table as JSON strings instead of `Value` objects * Enable configuration for disabling key tag in internal metrics * Add changelog entry * Add docs for memory enrichment_table * Fix typo in memory table docs * Apply suggestions from code review in documentation Co-authored-by: May Lee * Apply docs suggestions in code too * Run scan and flush on intervals and not only on writes * Ensure `scan_interval` can't be zero and handle zero `flush_interval` * Rename `sinks_and_table_sinks` to `all_sinks` * Use `Option` instead of `0` for optional memory config values * Rename `MemoryTableInternalMetricsConfig` to `InternalMetricsConfig` * Remove enrichment table `unwrap` from topology running * Use `expect` instead of `unwrap` when capturing write lock Co-authored-by: Pavlos Rontidis * Fix typo in `src/enrichment_tables/memory/table.rs` * Add a how it works section for memory enrichment table * Add documentation above `as_sink` per discussion on the PR * Fix enrichment memory table tests * Fix spellcheck error * Update LICENSE-3rdparty * Fix cue formatting * Use NonZeroU64 new_unchecked to fix MSRV check --------- Co-authored-by: May Lee Co-authored-by: Pavlos Rontidis --- Cargo.lock | 35 +- Cargo.toml | 8 +- LICENSE-3rdparty.csv | 2 + .../21348_memory_enrichment_table.feature.md | 4 + src/api/schema/components/mod.rs | 9 +- src/config/builder.rs | 14 +- src/config/compiler.rs | 20 +- src/config/diff.rs | 8 +- src/config/enrichment_table.rs | 82 ++- src/config/graph.rs | 4 +- src/config/loading/config_builder.rs | 2 +- src/config/mod.rs | 15 +- src/config/sink.rs | 6 +- src/config/validation.rs | 8 + src/enrichment_tables/memory/config.rs | 137 ++++ .../memory/internal_events.rs | 154 +++++ src/enrichment_tables/memory/mod.rs | 8 + src/enrichment_tables/memory/table.rs | 617 ++++++++++++++++++ src/enrichment_tables/mod.rs | 11 + src/topology/builder.rs | 16 +- src/topology/running.rs | 135 +++- .../en/docs/reference/configuration/_index.md | 74 +++ website/cue/reference/configuration.cue | 95 ++- website/cue/reference/urls.cue | 2 + 24 files changed, 1429 insertions(+), 37 deletions(-) create mode 100644 changelog.d/21348_memory_enrichment_table.feature.md create mode 100644 src/enrichment_tables/memory/config.rs create mode 100644 src/enrichment_tables/memory/internal_events.rs create mode 100644 src/enrichment_tables/memory/mod.rs create mode 100644 src/enrichment_tables/memory/table.rs diff --git a/Cargo.lock b/Cargo.lock index 5db3d7c754..dfdcac6a86 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3497,6 +3497,28 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "evmap" +version = "10.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e3ea06a83f97d3dc2eb06e51e7a729b418f0717a5558a5c870e3d5156dc558d" +dependencies = [ + "hashbag", + "slab", + "smallvec", +] + +[[package]] +name = "evmap-derive" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "332b1937705b7ed2fce76837024e9ae6f41cd2ad18a32c052de081f89982561b" +dependencies = [ + "proc-macro2 1.0.93", + "quote 1.0.38", + "syn 1.0.109", +] + [[package]] name = "executor-trait" version = "2.1.0" @@ -4131,6 +4153,12 @@ version = "2.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74721d007512d0cb3338cd20f0654ac913920061a4c4d0d8708edb3f2a698c0c" +[[package]] +name = "hashbag" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98f494b2060b2a8f5e63379e1e487258e014cee1b1725a735816c0107a2e9d93" + [[package]] name = "hashbrown" version = "0.12.3" @@ -9788,9 +9816,9 @@ dependencies = [ [[package]] name = "thread_local" -version = "1.1.7" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" dependencies = [ "cfg-if", "once_cell", @@ -11004,6 +11032,8 @@ dependencies = [ "dyn-clone", "encoding_rs", "enum_dispatch", + "evmap", + "evmap-derive", "exitcode", "fakedata", "flate2", @@ -11106,6 +11136,7 @@ dependencies = [ "syslog", "tempfile", "test-generator", + "thread_local", "tikv-jemallocator", "tokio", "tokio-openssl", diff --git a/Cargo.toml b/Cargo.toml index c4ac0ffdf9..d2e01c3fa2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -311,6 +311,8 @@ dirs-next = { version = "2.0.0", default-features = false, optional = true } dyn-clone = { version = "1.0.17", default-features = false } encoding_rs = { version = "0.8.35", default-features = false, features = ["serde"] } enum_dispatch = { version = "0.3.13", default-features = false } +evmap = { version = "10.0.2", default-features = false, optional = true } +evmap-derive = { version = "0.2.0", default-features = false, optional = true } exitcode = { version = "1.1.2", default-features = false } flate2.workspace = true futures-util = { version = "0.3.29", default-features = false } @@ -372,6 +374,7 @@ tokio-tungstenite = { version = "0.20.1", default-features = false, features = [ toml.workspace = true tonic = { workspace = true, optional = true } hickory-proto = { workspace = true, optional = true } +thread_local = { version = "1.1.8", default-features = false, optional = true } typetag = { version = "0.2.19", default-features = false } url = { version = "2.5.4", default-features = false, features = ["serde"] } warp = { version = "0.3.7", default-features = false } @@ -525,9 +528,10 @@ protobuf-build = ["dep:tonic-build", "dep:prost-build"] gcp = ["dep:base64", "dep:goauth", "dep:smpl_jwt"] # Enrichment Tables -enrichment-tables = ["enrichment-tables-geoip", "enrichment-tables-mmdb"] +enrichment-tables = ["enrichment-tables-geoip", "enrichment-tables-mmdb", "enrichment-tables-memory"] enrichment-tables-geoip = ["dep:maxminddb"] enrichment-tables-mmdb = ["dep:maxminddb"] +enrichment-tables-memory = ["dep:evmap", "dep:evmap-derive", "dep:thread_local"] # Codecs codecs-syslog = ["vector-lib/syslog"] @@ -980,7 +984,7 @@ remap-benches = ["transforms-remap"] transform-benches = ["transforms-filter", "transforms-dedupe", "transforms-reduce", "transforms-route"] codecs-benches = [] loki-benches = ["sinks-loki"] -enrichment-tables-benches = ["enrichment-tables-geoip", "enrichment-tables-mmdb"] +enrichment-tables-benches = ["enrichment-tables-geoip", "enrichment-tables-mmdb", "enrichment-tables-memory"] proptest = ["dep:proptest", "dep:proptest-derive", "vrl/proptest"] [[bench]] diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 9e634db470..0f9b5e3963 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -216,6 +216,7 @@ error-code,https://github.com/DoumanAsh/error-code,BSL-1.0,Douman event-listener,https://github.com/smol-rs/event-listener,Apache-2.0 OR MIT,"Stjepan Glavina , John Nunley " event-listener-strategy,https://github.com/smol-rs/event-listener-strategy,Apache-2.0 OR MIT,John Nunley +evmap,https://github.com/jonhoo/rust-evmap,MIT OR Apache-2.0,Jon Gjengset executor-trait,https://github.com/amqp-rs/executor-trait,Apache-2.0 OR MIT,Marc-Antoine Perennou exitcode,https://github.com/benwilber/exitcode,Apache-2.0,Ben Wilber fakedata_generator,https://github.com/kevingimbel/fakedata_generator,MIT,Kevin Gimbel @@ -267,6 +268,7 @@ group,https://github.com/zkcrypto/group,MIT OR Apache-2.0,"Sean Bowe , Sean McArthur " half,https://github.com/starkat99/half-rs,MIT OR Apache-2.0,Kathryn Long hash_hasher,https://github.com/Fraser999/Hash-Hasher,Apache-2.0 OR MIT,Fraser Hutchison +hashbag,https://github.com/jonhoo/hashbag,MIT OR Apache-2.0,Jon Gjengset hashbrown,https://github.com/rust-lang/hashbrown,MIT OR Apache-2.0,Amanieu d'Antras headers,https://github.com/hyperium/headers,MIT,Sean McArthur heck,https://github.com/withoutboats/heck,MIT OR Apache-2.0,The heck Authors diff --git a/changelog.d/21348_memory_enrichment_table.feature.md b/changelog.d/21348_memory_enrichment_table.feature.md new file mode 100644 index 0000000000..9a87239968 --- /dev/null +++ b/changelog.d/21348_memory_enrichment_table.feature.md @@ -0,0 +1,4 @@ +Add a new type of `enrichment_table` - `memory`, which can also act as a sink, taking in all the +data and storing it per key, enabling it to be read from as all other enrichment tables. + +authors: esensar diff --git a/src/api/schema/components/mod.rs b/src/api/schema/components/mod.rs index 7398a7c215..5edb2885fe 100644 --- a/src/api/schema/components/mod.rs +++ b/src/api/schema/components/mod.rs @@ -301,7 +301,14 @@ pub fn update_config(config: &Config) { } // Sinks - for (component_key, sink) in config.sinks() { + let table_sinks = config + .enrichment_tables() + .filter_map(|(k, e)| e.as_sink().map(|s| (k, s))) + .collect::>(); + for (component_key, sink) in config + .sinks() + .chain(table_sinks.iter().map(|(key, sink)| (*key, sink))) + { new_components.insert( component_key.clone(), Component::Sink(sink::Sink(sink::Data { diff --git a/src/config/builder.rs b/src/config/builder.rs index 3a0413e38a..351122aa5c 100644 --- a/src/config/builder.rs +++ b/src/config/builder.rs @@ -38,7 +38,7 @@ pub struct ConfigBuilder { /// All configured enrichment tables. #[serde(default)] - pub enrichment_tables: IndexMap, + pub enrichment_tables: IndexMap>, /// All configured sources. #[serde(default)] @@ -106,6 +106,11 @@ impl From for ConfigBuilder { .map(|(key, sink)| (key, sink.map_inputs(ToString::to_string))) .collect(); + let enrichment_tables = enrichment_tables + .into_iter() + .map(|(key, table)| (key, table.map_inputs(ToString::to_string))) + .collect(); + let tests = tests.into_iter().map(TestDefinition::stringify).collect(); ConfigBuilder { @@ -145,11 +150,16 @@ impl ConfigBuilder { pub fn add_enrichment_table, E: Into>( &mut self, key: K, + inputs: &[&str], enrichment_table: E, ) { + let inputs = inputs + .iter() + .map(|value| value.to_string()) + .collect::>(); self.enrichment_tables.insert( ComponentKey::from(key.into()), - EnrichmentTableOuter::new(enrichment_table), + EnrichmentTableOuter::new(inputs, enrichment_table), ); } diff --git a/src/config/compiler.rs b/src/config/compiler.rs index 28015e01ab..ff661bcc9f 100644 --- a/src/config/compiler.rs +++ b/src/config/compiler.rs @@ -3,7 +3,7 @@ use super::{ OutputId, }; -use indexmap::IndexSet; +use indexmap::{IndexMap, IndexSet}; use vector_lib::id::Inputs; pub fn compile(mut builder: ConfigBuilder) -> Result<(Config, Vec), Vec> { @@ -52,8 +52,17 @@ pub fn compile(mut builder: ConfigBuilder) -> Result<(Config, Vec), Vec< graceful_shutdown_duration, allow_empty: _, } = builder; + let all_sinks = sinks + .clone() + .into_iter() + .chain( + enrichment_tables + .iter() + .filter_map(|(key, table)| table.as_sink().map(|s| (key.clone(), s))), + ) + .collect::>(); - let graph = match Graph::new(&sources, &transforms, &sinks, schema) { + let graph = match Graph::new(&sources, &transforms, &all_sinks, schema) { Ok(graph) => graph, Err(graph_errors) => { errors.extend(graph_errors); @@ -85,6 +94,13 @@ pub fn compile(mut builder: ConfigBuilder) -> Result<(Config, Vec), Vec< (key, transform.with_inputs(inputs)) }) .collect(); + let enrichment_tables = enrichment_tables + .into_iter() + .map(|(key, table)| { + let inputs = graph.inputs_for(&key); + (key, table.with_inputs(inputs)) + }) + .collect(); let tests = tests .into_iter() .map(|test| test.resolve_outputs(&graph)) diff --git a/src/config/diff.rs b/src/config/diff.rs index da5ed54fae..1a9097c71e 100644 --- a/src/config/diff.rs +++ b/src/config/diff.rs @@ -31,12 +31,16 @@ impl ConfigDiff { self.sources.flip(); self.transforms.flip(); self.sinks.flip(); + self.enrichment_tables.flip(); self } /// Checks whether or not the given component is present at all. pub fn contains(&self, key: &ComponentKey) -> bool { - self.sources.contains(key) || self.transforms.contains(key) || self.sinks.contains(key) + self.sources.contains(key) + || self.transforms.contains(key) + || self.sinks.contains(key) + || self.enrichment_tables.contains(key) } /// Checks whether or not the given component is changed. @@ -44,6 +48,7 @@ impl ConfigDiff { self.sources.is_changed(key) || self.transforms.is_changed(key) || self.sinks.is_changed(key) + || self.enrichment_tables.contains(key) } /// Checks whether or not the given component is removed. @@ -51,6 +56,7 @@ impl ConfigDiff { self.sources.is_removed(key) || self.transforms.is_removed(key) || self.sinks.is_removed(key) + || self.enrichment_tables.contains(key) } } diff --git a/src/config/enrichment_table.rs b/src/config/enrichment_table.rs index 5e2cd72a00..2193c3ef71 100644 --- a/src/config/enrichment_table.rs +++ b/src/config/enrichment_table.rs @@ -1,21 +1,91 @@ use enum_dispatch::enum_dispatch; +use serde::Serialize; use vector_lib::config::GlobalOptions; -use vector_lib::configurable::{configurable_component, NamedComponent}; +use vector_lib::configurable::{configurable_component, Configurable, NamedComponent, ToValue}; +use vector_lib::id::Inputs; use crate::enrichment_tables::EnrichmentTables; +use super::dot_graph::GraphConfig; +use super::{SinkConfig, SinkOuter}; + /// Fully resolved enrichment table component. #[configurable_component] #[derive(Clone, Debug)] -pub struct EnrichmentTableOuter { +pub struct EnrichmentTableOuter +where + T: Configurable + Serialize + 'static + ToValue + Clone, +{ #[serde(flatten)] pub inner: EnrichmentTables, + #[configurable(derived)] + #[serde(default, skip_serializing_if = "vector_lib::serde::is_default")] + pub graph: GraphConfig, + #[configurable(derived)] + #[serde( + default = "Inputs::::default", + skip_serializing_if = "Inputs::is_empty" + )] + pub inputs: Inputs, } -impl EnrichmentTableOuter { - pub fn new>(inner: I) -> Self { +impl EnrichmentTableOuter +where + T: Configurable + Serialize + 'static + ToValue + Clone, +{ + pub fn new(inputs: I, inner: IET) -> Self + where + I: IntoIterator, + IET: Into, + { Self { inner: inner.into(), + graph: Default::default(), + inputs: Inputs::from_iter(inputs), + } + } + + // Components are currently built in a way that they match exactly one of the roles (source, + // transform, sink, enrichment table). Due to specific requirements of the "memory" enrichment + // table, it has to fulfill 2 of these roles (sink and enrichment table). To reduce the impact + // of this very specific requirement, any enrichment table can now be optionally mapped into a + // sink, but this will only work for a "memory" enrichment table, since other tables will not + // have a "sink_config" present. + // This is also not ideal, since `SinkOuter` is not meant to represent the actual configuration, + // but it should just be a representation of that config used for deserialization. + // In the future, if more such components come up, it would be good to limit such "Outer" + // components to deserialization and build up the components and the topology in a more granular + // way, with each having "modules" for inputs (making them valid as sinks), for healthchecks, + // for providing outputs, etc. + pub fn as_sink(&self) -> Option> { + self.inner.sink_config().map(|sink| SinkOuter { + graph: self.graph.clone(), + inputs: self.inputs.clone(), + healthcheck_uri: None, + healthcheck: Default::default(), + buffer: Default::default(), + proxy: Default::default(), + inner: sink, + }) + } + + pub(super) fn map_inputs(self, f: impl Fn(&T) -> U) -> EnrichmentTableOuter + where + U: Configurable + Serialize + 'static + ToValue + Clone, + { + let inputs = self.inputs.iter().map(f).collect::>(); + self.with_inputs(inputs) + } + + pub(crate) fn with_inputs(self, inputs: I) -> EnrichmentTableOuter + where + I: IntoIterator, + U: Configurable + Serialize + 'static + ToValue + Clone, + { + EnrichmentTableOuter { + inputs: Inputs::from_iter(inputs), + inner: self.inner, + graph: self.graph, } } } @@ -36,4 +106,8 @@ pub trait EnrichmentTableConfig: NamedComponent + core::fmt::Debug + Send + Sync &self, globals: &GlobalOptions, ) -> crate::Result>; + + fn sink_config(&self) -> Option> { + None + } } diff --git a/src/config/graph.rs b/src/config/graph.rs index 70c59399ec..9ed1af6ff1 100644 --- a/src/config/graph.rs +++ b/src/config/graph.rs @@ -112,7 +112,7 @@ impl Graph { ); } - for (id, config) in sinks.iter() { + for (id, config) in sinks { graph.nodes.insert( id.clone(), Node::Sink { @@ -133,7 +133,7 @@ impl Graph { } } - for (id, config) in sinks.iter() { + for (id, config) in sinks { for input in config.inputs.iter() { if let Err(e) = graph.add_input(input, id, &available_inputs) { errors.push(e); diff --git a/src/config/loading/config_builder.rs b/src/config/loading/config_builder.rs index 1e40d38b98..4ef0010d6f 100644 --- a/src/config/loading/config_builder.rs +++ b/src/config/loading/config_builder.rs @@ -63,7 +63,7 @@ impl Process for ConfigBuilderLoader { } Some(ComponentHint::EnrichmentTable) => { self.builder.enrichment_tables.extend(deserialize_table::< - IndexMap, + IndexMap>, >(table)?); } Some(ComponentHint::Test) => { diff --git a/src/config/mod.rs b/src/config/mod.rs index 2326a8bc0e..bb3082e8ce 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -26,7 +26,7 @@ mod builder; mod cmd; mod compiler; mod diff; -mod dot_graph; +pub mod dot_graph; mod enrichment_table; pub mod format; mod graph; @@ -104,7 +104,7 @@ pub struct Config { sources: IndexMap, sinks: IndexMap>, transforms: IndexMap>, - pub enrichment_tables: IndexMap, + pub enrichment_tables: IndexMap>, tests: Vec, secret: IndexMap, pub graceful_shutdown_duration: Option, @@ -143,11 +143,22 @@ impl Config { self.sinks.get(id) } + pub fn enrichment_tables( + &self, + ) -> impl Iterator)> { + self.enrichment_tables.iter() + } + + pub fn enrichment_table(&self, id: &ComponentKey) -> Option<&EnrichmentTableOuter> { + self.enrichment_tables.get(id) + } + pub fn inputs_for_node(&self, id: &ComponentKey) -> Option<&[OutputId]> { self.transforms .get(id) .map(|t| &t.inputs[..]) .or_else(|| self.sinks.get(id).map(|s| &s.inputs[..])) + .or_else(|| self.enrichment_tables.get(id).map(|s| &s.inputs[..])) } pub fn propagate_acknowledgements(&mut self) -> Result<(), Vec> { diff --git a/src/config/sink.rs b/src/config/sink.rs index a322863f11..7c07575589 100644 --- a/src/config/sink.rs +++ b/src/config/sink.rs @@ -65,11 +65,11 @@ where /// This must be a valid URI, which requires at least the scheme and host. All other /// components -- port, path, etc -- are allowed as well. #[configurable(deprecated, metadata(docs::hidden), validation(format = "uri"))] - healthcheck_uri: Option, + pub healthcheck_uri: Option, #[configurable(derived, metadata(docs::advanced))] #[serde(default, deserialize_with = "crate::serde::bool_or_struct")] - healthcheck: SinkHealthcheckOptions, + pub healthcheck: SinkHealthcheckOptions, #[configurable(derived)] #[serde(default, skip_serializing_if = "vector_lib::serde::is_default")] @@ -77,7 +77,7 @@ where #[configurable(derived)] #[serde(default, skip_serializing_if = "vector_lib::serde::is_default")] - proxy: ProxyConfig, + pub proxy: ProxyConfig, #[serde(flatten)] #[configurable(metadata(docs::hidden))] diff --git a/src/config/validation.rs b/src/config/validation.rs index 56423e6aa7..48f8ebbe1b 100644 --- a/src/config/validation.rs +++ b/src/config/validation.rs @@ -349,6 +349,11 @@ pub fn warnings(config: &Config) -> Vec { .collect::>() }); + let table_sinks = config + .enrichment_tables + .iter() + .filter_map(|(key, table)| table.as_sink().map(|s| (key, s))) + .collect::>(); for (input_type, id) in transform_ids.chain(source_ids) { if !config .transforms @@ -358,6 +363,9 @@ pub fn warnings(config: &Config) -> Vec { .sinks .iter() .any(|(_, sink)| sink.inputs.contains(&id)) + && !table_sinks + .iter() + .any(|(_, sink)| sink.inputs.contains(&id)) { warnings.push(format!( "{} \"{}\" has no consumers", diff --git a/src/enrichment_tables/memory/config.rs b/src/enrichment_tables/memory/config.rs new file mode 100644 index 0000000000..d617dbd742 --- /dev/null +++ b/src/enrichment_tables/memory/config.rs @@ -0,0 +1,137 @@ +use std::num::NonZeroU64; +use std::sync::Arc; + +use crate::sinks::Healthcheck; +use crate::{config::SinkContext, enrichment_tables::memory::Memory}; +use async_trait::async_trait; +use futures::{future, FutureExt}; +use tokio::sync::Mutex; +use vector_lib::config::{AcknowledgementsConfig, Input}; +use vector_lib::enrichment::Table; +use vector_lib::{configurable::configurable_component, sink::VectorSink}; + +use crate::config::{EnrichmentTableConfig, SinkConfig}; + +use super::internal_events::InternalMetricsConfig; + +/// Configuration for the `memory` enrichment table. +#[configurable_component(enrichment_table("memory"))] +#[derive(Clone)] +pub struct MemoryConfig { + /// TTL (time-to-live in seconds) is used to limit the lifetime of data stored in the cache. + /// When TTL expires, data behind a specific key in the cache is removed. + /// TTL is reset when the key is replaced. + #[serde(default = "default_ttl")] + pub ttl: u64, + /// The scan interval used to look for expired records. This is provided + /// as an optimization to ensure that TTL is updated, but without doing + /// too many cache scans. + #[serde(default = "default_scan_interval")] + pub scan_interval: NonZeroU64, + /// The interval used for making writes visible in the table. + /// Longer intervals might get better performance, + /// but there is a longer delay before the data is visible in the table. + /// Since every TTL scan makes its changes visible, only use this value + /// if it is shorter than the `scan_interval`. + /// + /// By default, all writes are made visible immediately. + #[serde(skip_serializing_if = "vector_lib::serde::is_default")] + pub flush_interval: Option, + /// Maximum size of the table in bytes. All insertions that make + /// this table bigger than the maximum size are rejected. + /// + /// By default, there is no size limit. + #[serde(skip_serializing_if = "vector_lib::serde::is_default")] + pub max_byte_size: Option, + + /// Configuration of internal metrics + #[configurable(derived)] + #[serde(default)] + pub internal_metrics: InternalMetricsConfig, + + #[serde(skip)] + memory: Arc>>>, +} + +impl PartialEq for MemoryConfig { + fn eq(&self, other: &Self) -> bool { + self.ttl == other.ttl + && self.scan_interval == other.scan_interval + && self.flush_interval == other.flush_interval + } +} +impl Eq for MemoryConfig {} + +impl Default for MemoryConfig { + fn default() -> Self { + Self { + ttl: default_ttl(), + scan_interval: default_scan_interval(), + flush_interval: None, + memory: Arc::new(Mutex::new(None)), + max_byte_size: None, + internal_metrics: InternalMetricsConfig::default(), + } + } +} + +const fn default_ttl() -> u64 { + 600 +} + +const fn default_scan_interval() -> NonZeroU64 { + unsafe { NonZeroU64::new_unchecked(30) } +} + +impl MemoryConfig { + async fn get_or_build_memory(&self) -> Memory { + let mut boxed_memory = self.memory.lock().await; + *boxed_memory + .get_or_insert_with(|| Box::new(Memory::new(self.clone()))) + .clone() + } +} + +impl EnrichmentTableConfig for MemoryConfig { + async fn build( + &self, + _globals: &crate::config::GlobalOptions, + ) -> crate::Result> { + Ok(Box::new(self.get_or_build_memory().await)) + } + + fn sink_config(&self) -> Option> { + Some(Box::new(self.clone())) + } +} + +#[async_trait] +#[typetag::serde(name = "memory_enrichment_table")] +impl SinkConfig for MemoryConfig { + async fn build(&self, _cx: SinkContext) -> crate::Result<(VectorSink, Healthcheck)> { + let sink = VectorSink::from_event_streamsink(self.get_or_build_memory().await); + + Ok((sink, future::ok(()).boxed())) + } + + fn input(&self) -> Input { + Input::log() + } + + fn acknowledgements(&self) -> &AcknowledgementsConfig { + &AcknowledgementsConfig::DEFAULT + } +} + +impl std::fmt::Debug for MemoryConfig { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MemoryConfig") + .field("ttl", &self.ttl) + .field("scan_interval", &self.scan_interval) + .field("flush_interval", &self.flush_interval) + .field("max_byte_size", &self.max_byte_size) + .finish() + } +} + +impl_generate_config_from_default!(MemoryConfig); diff --git a/src/enrichment_tables/memory/internal_events.rs b/src/enrichment_tables/memory/internal_events.rs new file mode 100644 index 0000000000..7a95438938 --- /dev/null +++ b/src/enrichment_tables/memory/internal_events.rs @@ -0,0 +1,154 @@ +use metrics::{counter, gauge}; +use vector_lib::configurable::configurable_component; +use vector_lib::internal_event::InternalEvent; + +/// Configuration of internal metrics for enrichment memory table. +#[configurable_component] +#[derive(Clone, Debug, PartialEq, Eq, Default)] +#[serde(deny_unknown_fields)] +pub struct InternalMetricsConfig { + /// Determines whether to include the key tag on internal metrics. + /// + /// This is useful for distinguishing between different keys while monitoring. However, the tag's + /// cardinality is unbounded. + #[serde(default = "crate::serde::default_false")] + pub include_key_tag: bool, +} + +#[derive(Debug)] +pub(crate) struct MemoryEnrichmentTableRead<'a> { + pub key: &'a str, + pub include_key_metric_tag: bool, +} + +impl InternalEvent for MemoryEnrichmentTableRead<'_> { + fn emit(self) { + if self.include_key_metric_tag { + counter!( + "memory_enrichment_table_reads_total", + "key" => self.key.to_owned() + ) + .increment(1); + } else { + counter!("memory_enrichment_table_reads_total",).increment(1); + } + } + + fn name(&self) -> Option<&'static str> { + Some("MemoryEnrichmentTableRead") + } +} + +#[derive(Debug)] +pub(crate) struct MemoryEnrichmentTableInserted<'a> { + pub key: &'a str, + pub include_key_metric_tag: bool, +} + +impl InternalEvent for MemoryEnrichmentTableInserted<'_> { + fn emit(self) { + if self.include_key_metric_tag { + counter!( + "memory_enrichment_table_insertions_total", + "key" => self.key.to_owned() + ) + .increment(1); + } else { + counter!("memory_enrichment_table_insertions_total",).increment(1); + } + } + + fn name(&self) -> Option<&'static str> { + Some("MemoryEnrichmentTableInserted") + } +} + +#[derive(Debug)] +pub(crate) struct MemoryEnrichmentTableFlushed { + pub new_objects_count: usize, + pub new_byte_size: usize, +} + +impl InternalEvent for MemoryEnrichmentTableFlushed { + fn emit(self) { + counter!("memory_enrichment_table_flushes_total",).increment(1); + gauge!("memory_enrichment_table_objects_count",).set(self.new_objects_count as f64); + gauge!("memory_enrichment_table_byte_size",).set(self.new_byte_size as f64); + } + + fn name(&self) -> Option<&'static str> { + Some("MemoryEnrichmentTableFlushed") + } +} + +#[derive(Debug)] +pub(crate) struct MemoryEnrichmentTableTtlExpired<'a> { + pub key: &'a str, + pub include_key_metric_tag: bool, +} + +impl InternalEvent for MemoryEnrichmentTableTtlExpired<'_> { + fn emit(self) { + if self.include_key_metric_tag { + counter!( + "memory_enrichment_table_ttl_expirations", + "key" => self.key.to_owned() + ) + .increment(1); + } else { + counter!("memory_enrichment_table_ttl_expirations",).increment(1); + } + } + + fn name(&self) -> Option<&'static str> { + Some("MemoryEnrichmentTableTtlExpired") + } +} + +#[derive(Debug)] +pub(crate) struct MemoryEnrichmentTableReadFailed<'a> { + pub key: &'a str, + pub include_key_metric_tag: bool, +} + +impl InternalEvent for MemoryEnrichmentTableReadFailed<'_> { + fn emit(self) { + if self.include_key_metric_tag { + counter!( + "memory_enrichment_table_failed_reads", + "key" => self.key.to_owned() + ) + .increment(1); + } else { + counter!("memory_enrichment_table_failed_reads",).increment(1); + } + } + + fn name(&self) -> Option<&'static str> { + Some("MemoryEnrichmentTableReadFailed") + } +} + +#[derive(Debug)] +pub(crate) struct MemoryEnrichmentTableInsertFailed<'a> { + pub key: &'a str, + pub include_key_metric_tag: bool, +} + +impl InternalEvent for MemoryEnrichmentTableInsertFailed<'_> { + fn emit(self) { + if self.include_key_metric_tag { + counter!( + "memory_enrichment_table_failed_insertions", + "key" => self.key.to_owned() + ) + .increment(1); + } else { + counter!("memory_enrichment_table_failed_insertions",).increment(1); + } + } + + fn name(&self) -> Option<&'static str> { + Some("MemoryEnrichmentTableInsertFailed") + } +} diff --git a/src/enrichment_tables/memory/mod.rs b/src/enrichment_tables/memory/mod.rs new file mode 100644 index 0000000000..36a1ed878e --- /dev/null +++ b/src/enrichment_tables/memory/mod.rs @@ -0,0 +1,8 @@ +//! Handles enrichment tables for `type = memory`. + +mod config; +mod internal_events; +mod table; + +pub use config::*; +pub use table::*; diff --git a/src/enrichment_tables/memory/table.rs b/src/enrichment_tables/memory/table.rs new file mode 100644 index 0000000000..9c52149dad --- /dev/null +++ b/src/enrichment_tables/memory/table.rs @@ -0,0 +1,617 @@ +use crate::enrichment_tables::memory::internal_events::{ + MemoryEnrichmentTableFlushed, MemoryEnrichmentTableInsertFailed, MemoryEnrichmentTableInserted, + MemoryEnrichmentTableRead, MemoryEnrichmentTableReadFailed, MemoryEnrichmentTableTtlExpired, +}; +use crate::enrichment_tables::memory::MemoryConfig; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use evmap::shallow_copy::CopyValue; +use evmap::{self}; +use evmap_derive::ShallowCopy; +use thread_local::ThreadLocal; +use tokio::time::interval; +use tokio_stream::wrappers::IntervalStream; +use vector_lib::{ByteSizeOf, EstimatedJsonEncodedSizeOf}; + +use async_trait::async_trait; +use bytes::Bytes; +use futures::stream::BoxStream; +use tokio_stream::StreamExt; +use vector_lib::enrichment::{Case, Condition, IndexHandle, Table}; +use vector_lib::event::{Event, EventStatus, Finalizable}; +use vector_lib::internal_event::{ + ByteSize, BytesSent, CountByteSize, EventsSent, InternalEventHandle, Output, Protocol, +}; +use vector_lib::sink::StreamSink; +use vrl::value::{KeyString, ObjectMap, Value}; + +/// Single memory entry containing the value and TTL +#[derive(Clone, Eq, PartialEq, Hash, ShallowCopy)] +pub struct MemoryEntry { + value: String, + update_time: CopyValue, +} + +impl ByteSizeOf for MemoryEntry { + fn allocated_bytes(&self) -> usize { + self.value.size_of() + } +} + +impl MemoryEntry { + fn as_object_map(&self, now: Instant, total_ttl: u64, key: &str) -> Result { + let ttl = total_ttl.saturating_sub(now.duration_since(*self.update_time).as_secs()); + Ok(ObjectMap::from([ + ( + KeyString::from("key"), + Value::Bytes(Bytes::copy_from_slice(key.as_bytes())), + ), + ( + KeyString::from("value"), + serde_json::from_str::(&self.value) + .map_err(|_| "Failed to read value from memory!")?, + ), + ( + KeyString::from("ttl"), + Value::Integer(ttl.try_into().unwrap_or(i64::MAX)), + ), + ])) + } + + fn expired(&self, now: Instant, ttl: u64) -> bool { + now.duration_since(*self.update_time).as_secs() > ttl + } +} + +#[derive(Default)] +struct MemoryMetadata { + byte_size: u64, +} + +// Used to ensure that these 2 are locked together +struct MemoryWriter { + write_handle: evmap::WriteHandle, + metadata: MemoryMetadata, +} + +/// A struct that implements [vector_lib::enrichment::Table] to handle loading enrichment data from a memory structure. +pub struct Memory { + read_handle_factory: evmap::ReadHandleFactory, + read_handle: ThreadLocal>, + write_handle: Arc>, + config: MemoryConfig, +} + +impl Memory { + /// Creates a new [Memory] based on the provided config. + pub fn new(config: MemoryConfig) -> Self { + let (read_handle, write_handle) = evmap::new(); + Self { + config, + read_handle_factory: read_handle.factory(), + read_handle: ThreadLocal::new(), + write_handle: Arc::new(Mutex::new(MemoryWriter { + write_handle, + metadata: MemoryMetadata::default(), + })), + } + } + + fn get_read_handle(&self) -> &evmap::ReadHandle { + self.read_handle + .get_or(|| self.read_handle_factory.handle()) + } + + fn handle_value(&mut self, value: ObjectMap) { + let mut writer = self.write_handle.lock().expect("mutex poisoned"); + let now = Instant::now(); + + for (k, v) in value.into_iter() { + let new_entry_key = String::from(k); + let Ok(v) = serde_json::to_string(&v) else { + emit!(MemoryEnrichmentTableInsertFailed { + key: &new_entry_key, + include_key_metric_tag: self.config.internal_metrics.include_key_tag + }); + continue; + }; + let new_entry = MemoryEntry { + value: v, + update_time: now.into(), + }; + let new_entry_size = new_entry_key.size_of() + new_entry.size_of(); + if let Some(max_byte_size) = self.config.max_byte_size { + if writer + .metadata + .byte_size + .saturating_add(new_entry_size as u64) + > max_byte_size + { + // Reject new entries + emit!(MemoryEnrichmentTableInsertFailed { + key: &new_entry_key, + include_key_metric_tag: self.config.internal_metrics.include_key_tag + }); + continue; + } + } + writer.metadata.byte_size = writer + .metadata + .byte_size + .saturating_add(new_entry_size as u64); + emit!(MemoryEnrichmentTableInserted { + key: &new_entry_key, + include_key_metric_tag: self.config.internal_metrics.include_key_tag + }); + writer.write_handle.update(new_entry_key, new_entry); + } + + if self.config.flush_interval.is_none() { + writer.write_handle.refresh(); + } + } + + fn scan_and_mark_for_deletion(&mut self) -> bool { + let mut writer = self.write_handle.lock().expect("mutex poisoned"); + let now = Instant::now(); + + let mut needs_flush = false; + // Since evmap holds 2 separate maps for the data, we are free to directly remove + // elements via the writer, while we are iterating the reader + // Refresh will happen only after we manually invoke it after iteration + if let Some(reader) = self.get_read_handle().read() { + for (k, v) in reader.iter() { + if let Some(entry) = v.get_one() { + if entry.expired(now, self.config.ttl) { + // Byte size is not reduced at this point, because the actual deletion + // will only happen at refresh time + writer.write_handle.empty(k.clone()); + emit!(MemoryEnrichmentTableTtlExpired { + key: k, + include_key_metric_tag: self.config.internal_metrics.include_key_tag + }); + needs_flush = true; + } + } + } + }; + + needs_flush + } + + fn scan(&mut self) { + let needs_flush = self.scan_and_mark_for_deletion(); + if needs_flush { + self.flush(); + } + } + + fn flush(&mut self) { + let mut writer = self.write_handle.lock().expect("mutex poisoned"); + + writer.write_handle.refresh(); + if let Some(reader) = self.get_read_handle().read() { + let mut byte_size = 0; + for (k, v) in reader.iter() { + byte_size += k.size_of() + v.get_one().size_of(); + } + writer.metadata.byte_size = byte_size as u64; + emit!(MemoryEnrichmentTableFlushed { + new_objects_count: reader.len(), + new_byte_size: byte_size + }); + } + } +} + +impl Clone for Memory { + fn clone(&self) -> Self { + Self { + read_handle_factory: self.read_handle_factory.clone(), + read_handle: ThreadLocal::new(), + write_handle: Arc::clone(&self.write_handle), + config: self.config.clone(), + } + } +} + +impl Table for Memory { + fn find_table_row<'a>( + &self, + case: Case, + condition: &'a [Condition<'a>], + select: Option<&'a [String]>, + index: Option, + ) -> Result { + let mut rows = self.find_table_rows(case, condition, select, index)?; + + match rows.pop() { + Some(row) if rows.is_empty() => Ok(row), + Some(_) => Err("More than 1 row found".to_string()), + None => Err("Key not found".to_string()), + } + } + + fn find_table_rows<'a>( + &self, + _case: Case, + condition: &'a [Condition<'a>], + _select: Option<&'a [String]>, + _index: Option, + ) -> Result, String> { + match condition.first() { + Some(_) if condition.len() > 1 => Err("Only one condition is allowed".to_string()), + Some(Condition::Equals { value, .. }) => { + let key = value.to_string_lossy(); + match self.get_read_handle().get_one(key.as_ref()) { + Some(row) => { + emit!(MemoryEnrichmentTableRead { + key: &key, + include_key_metric_tag: self.config.internal_metrics.include_key_tag + }); + row.as_object_map(Instant::now(), self.config.ttl, &key) + .map(|r| vec![r]) + } + None => { + emit!(MemoryEnrichmentTableReadFailed { + key: &key, + include_key_metric_tag: self.config.internal_metrics.include_key_tag + }); + Ok(Default::default()) + } + } + } + Some(_) => Err("Only equality condition is allowed".to_string()), + None => Err("Key condition must be specified".to_string()), + } + } + + fn add_index(&mut self, _case: Case, fields: &[&str]) -> Result { + match fields.len() { + 0 => Err("Key field is required".to_string()), + 1 => Ok(IndexHandle(0)), + _ => Err("Only one field is allowed".to_string()), + } + } + + /// Returns a list of the field names that are in each index + fn index_fields(&self) -> Vec<(Case, Vec)> { + Vec::new() + } + + /// Doesn't need reload, data is written directly + fn needs_reload(&self) -> bool { + false + } +} + +impl std::fmt::Debug for Memory { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Memory {} row(s)", self.get_read_handle().len()) + } +} + +#[async_trait] +impl StreamSink for Memory { + async fn run(mut self: Box, mut input: BoxStream<'_, Event>) -> Result<(), ()> { + let events_sent = register!(EventsSent::from(Output(None))); + let bytes_sent = register!(BytesSent::from(Protocol("memory_enrichment_table".into(),))); + let mut flush_interval = IntervalStream::new(interval( + self.config + .flush_interval + .map(Duration::from_secs) + .unwrap_or(Duration::MAX), + )); + let mut scan_interval = IntervalStream::new(interval(Duration::from_secs( + self.config.scan_interval.into(), + ))); + + loop { + tokio::select! { + event = input.next() => { + let mut event = if let Some(event) = event { + event + } else { + break; + }; + let event_byte_size = event.estimated_json_encoded_size_of(); + + let finalizers = event.take_finalizers(); + + // Panic: This sink only accepts Logs, so this should never panic + let log = event.into_log(); + + if let (Value::Object(map), _) = log.into_parts() { + self.handle_value(map) + }; + + finalizers.update_status(EventStatus::Delivered); + events_sent.emit(CountByteSize(1, event_byte_size)); + bytes_sent.emit(ByteSize(event_byte_size.get())); + } + + Some(_) = flush_interval.next() => { + self.flush(); + } + + Some(_) = scan_interval.next() => { + self.scan(); + } + } + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use futures::future::ready; + use futures_util::stream; + use std::time::Duration; + + use vector_lib::sink::VectorSink; + + use super::*; + use crate::{ + event::{Event, LogEvent}, + test_util::components::{run_and_assert_sink_compliance, SINK_TAGS}, + }; + + fn build_memory_config(modfn: impl Fn(&mut MemoryConfig)) -> MemoryConfig { + let mut config = MemoryConfig::default(); + modfn(&mut config); + config + } + + #[test] + fn finds_row() { + let mut memory = Memory::new(Default::default()); + memory.handle_value(ObjectMap::from([("test_key".into(), Value::from(5))])); + + let condition = Condition::Equals { + field: "key", + value: Value::from("test_key"), + }; + + assert_eq!( + Ok(ObjectMap::from([ + ("key".into(), Value::from("test_key")), + ("ttl".into(), Value::from(memory.config.ttl)), + ("value".into(), Value::from(5)), + ])), + memory.find_table_row(Case::Sensitive, &[condition], None, None) + ); + } + + #[test] + fn calculates_ttl() { + let ttl = 100; + let secs_to_subtract = 10; + let memory = Memory::new(build_memory_config(|c| c.ttl = ttl)); + { + let mut handle = memory.write_handle.lock().unwrap(); + handle.write_handle.update( + "test_key".to_string(), + MemoryEntry { + value: "5".to_string(), + update_time: (Instant::now() - Duration::from_secs(secs_to_subtract)).into(), + }, + ); + handle.write_handle.refresh(); + } + + let condition = Condition::Equals { + field: "key", + value: Value::from("test_key"), + }; + + assert_eq!( + Ok(ObjectMap::from([ + ("key".into(), Value::from("test_key")), + ("ttl".into(), Value::from(ttl - secs_to_subtract)), + ("value".into(), Value::from(5)), + ])), + memory.find_table_row(Case::Sensitive, &[condition], None, None) + ); + } + + #[test] + fn removes_expired_records_on_scan_interval() { + let ttl = 100; + let mut memory = Memory::new(build_memory_config(|c| { + c.ttl = ttl; + })); + { + let mut handle = memory.write_handle.lock().unwrap(); + handle.write_handle.update( + "test_key".to_string(), + MemoryEntry { + value: "5".to_string(), + update_time: (Instant::now() - Duration::from_secs(ttl + 10)).into(), + }, + ); + handle.write_handle.refresh(); + } + + // Finds the value before scan + let condition = Condition::Equals { + field: "key", + value: Value::from("test_key"), + }; + assert_eq!( + Ok(ObjectMap::from([ + ("key".into(), Value::from("test_key")), + ("ttl".into(), Value::from(0)), + ("value".into(), Value::from(5)), + ])), + memory.find_table_row(Case::Sensitive, &[condition.clone()], None, None) + ); + + // Force scan + memory.scan(); + + // The value is not present anymore + assert!(memory + .find_table_rows(Case::Sensitive, &[condition], None, None) + .unwrap() + .pop() + .is_none()); + } + + #[test] + fn does_not_show_values_before_flush_interval() { + let ttl = 100; + let mut memory = Memory::new(build_memory_config(|c| { + c.ttl = ttl; + c.flush_interval = Some(10); + })); + memory.handle_value(ObjectMap::from([("test_key".into(), Value::from(5))])); + + let condition = Condition::Equals { + field: "key", + value: Value::from("test_key"), + }; + + assert!(memory + .find_table_rows(Case::Sensitive, &[condition], None, None) + .unwrap() + .pop() + .is_none()); + } + + #[test] + fn updates_ttl_on_value_replacement() { + let ttl = 100; + let mut memory = Memory::new(build_memory_config(|c| c.ttl = ttl)); + { + let mut handle = memory.write_handle.lock().unwrap(); + handle.write_handle.update( + "test_key".to_string(), + MemoryEntry { + value: "5".to_string(), + update_time: (Instant::now() - Duration::from_secs(ttl / 2)).into(), + }, + ); + handle.write_handle.refresh(); + } + let condition = Condition::Equals { + field: "key", + value: Value::from("test_key"), + }; + + assert_eq!( + Ok(ObjectMap::from([ + ("key".into(), Value::from("test_key")), + ("ttl".into(), Value::from(ttl / 2)), + ("value".into(), Value::from(5)), + ])), + memory.find_table_row(Case::Sensitive, &[condition.clone()], None, None) + ); + + memory.handle_value(ObjectMap::from([("test_key".into(), Value::from(5))])); + + assert_eq!( + Ok(ObjectMap::from([ + ("key".into(), Value::from("test_key")), + ("ttl".into(), Value::from(ttl)), + ("value".into(), Value::from(5)), + ])), + memory.find_table_row(Case::Sensitive, &[condition], None, None) + ); + } + + #[test] + fn ignores_all_values_over_byte_size_limit() { + let mut memory = Memory::new(build_memory_config(|c| { + c.max_byte_size = Some(1); + })); + memory.handle_value(ObjectMap::from([("test_key".into(), Value::from(5))])); + + let condition = Condition::Equals { + field: "key", + value: Value::from("test_key"), + }; + + assert!(memory + .find_table_rows(Case::Sensitive, &[condition], None, None) + .unwrap() + .pop() + .is_none()); + } + + #[test] + fn ignores_values_when_byte_size_limit_is_reached() { + let ttl = 100; + let mut memory = Memory::new(build_memory_config(|c| { + c.ttl = ttl; + c.max_byte_size = Some(150); + })); + memory.handle_value(ObjectMap::from([("test_key".into(), Value::from(5))])); + memory.handle_value(ObjectMap::from([("rejected_key".into(), Value::from(5))])); + + assert_eq!( + Ok(ObjectMap::from([ + ("key".into(), Value::from("test_key")), + ("ttl".into(), Value::from(ttl)), + ("value".into(), Value::from(5)), + ])), + memory.find_table_row( + Case::Sensitive, + &[Condition::Equals { + field: "key", + value: Value::from("test_key") + }], + None, + None + ) + ); + + assert!(memory + .find_table_rows( + Case::Sensitive, + &[Condition::Equals { + field: "key", + value: Value::from("rejected_key") + }], + None, + None + ) + .unwrap() + .pop() + .is_none()); + } + + #[test] + fn missing_key() { + let memory = Memory::new(Default::default()); + + let condition = Condition::Equals { + field: "key", + value: Value::from("test_key"), + }; + + assert!(memory + .find_table_rows(Case::Sensitive, &[condition], None, None) + .unwrap() + .pop() + .is_none()); + } + + #[tokio::test] + async fn sink_spec_compliance() { + let event = Event::Log(LogEvent::from(ObjectMap::from([( + "test_key".into(), + Value::from(5), + )]))); + + let memory = Memory::new(Default::default()); + + run_and_assert_sink_compliance( + VectorSink::from_event_streamsink(memory), + stream::once(ready(event)), + &SINK_TAGS, + ) + .await; + } +} diff --git a/src/enrichment_tables/mod.rs b/src/enrichment_tables/mod.rs index 97a93b0059..75b128e50b 100644 --- a/src/enrichment_tables/mod.rs +++ b/src/enrichment_tables/mod.rs @@ -1,4 +1,5 @@ //! Functionality to handle enrichment tables. +use crate::sinks::prelude::SinkConfig; use enum_dispatch::enum_dispatch; use vector_lib::configurable::{configurable_component, NamedComponent}; pub use vector_lib::enrichment::{Condition, IndexHandle, Table}; @@ -7,6 +8,9 @@ use crate::config::{EnrichmentTableConfig, GlobalOptions}; pub mod file; +#[cfg(feature = "enrichment-tables-memory")] +pub mod memory; + #[cfg(feature = "enrichment-tables-geoip")] pub mod geoip; @@ -22,6 +26,11 @@ pub enum EnrichmentTables { /// Exposes data from a static file as an enrichment table. File(file::FileConfig), + /// Exposes data from a memory cache as an enrichment table. The cache can be written to using + /// a sink. + #[cfg(feature = "enrichment-tables-memory")] + Memory(memory::MemoryConfig), + /// Exposes data from a [MaxMind][maxmind] [GeoIP2][geoip2] database as an enrichment table. /// /// [maxmind]: https://www.maxmind.com/ @@ -41,6 +50,8 @@ impl NamedComponent for EnrichmentTables { fn get_component_name(&self) -> &'static str { match self { Self::File(config) => config.get_component_name(), + #[cfg(feature = "enrichment-tables-memory")] + Self::Memory(config) => config.get_component_name(), #[cfg(feature = "enrichment-tables-geoip")] Self::Geoip(config) => config.get_component_name(), #[cfg(feature = "enrichment-tables-mmdb")] diff --git a/src/topology/builder.rs b/src/topology/builder.rs index 8602e5cce6..061f4ac3f0 100644 --- a/src/topology/builder.rs +++ b/src/topology/builder.rs @@ -155,7 +155,7 @@ impl<'a> Builder<'a> { let mut enrichment_tables = HashMap::new(); // Build enrichment tables - 'tables: for (name, table) in self.config.enrichment_tables.iter() { + 'tables: for (name, table_outer) in self.config.enrichment_tables.iter() { let table_name = name.to_string(); if ENRICHMENT_TABLES.needs_reload(&table_name) { let indexes = if !self.diff.enrichment_tables.is_added(name) { @@ -166,7 +166,7 @@ impl<'a> Builder<'a> { None }; - let mut table = match table.inner.build(&self.config.global).await { + let mut table = match table_outer.inner.build(&self.config.global).await { Ok(table) => table, Err(error) => { self.errors @@ -506,10 +506,22 @@ impl<'a> Builder<'a> { } async fn build_sinks(&mut self, enrichment_tables: &vector_lib::enrichment::TableRegistry) { + let table_sinks = self + .config + .enrichment_tables + .iter() + .filter_map(|(key, table)| table.as_sink().map(|s| (key, s))) + .collect::>(); for (key, sink) in self .config .sinks() .filter(|(key, _)| self.diff.sinks.contains_new(key)) + .chain( + table_sinks + .iter() + .map(|(key, sink)| (*key, sink)) + .filter(|(key, _)| self.diff.enrichment_tables.contains_new(key)), + ) { debug!(component = %key, "Building new sink."); diff --git a/src/topology/running.rs b/src/topology/running.rs index 796661b7b4..4ff91c510f 100644 --- a/src/topology/running.rs +++ b/src/topology/running.rs @@ -15,7 +15,10 @@ use super::{ BuiltBuffer, TaskHandle, }; use crate::{ - config::{ComponentKey, Config, ConfigDiff, HealthcheckOptions, Inputs, OutputId, Resource}, + config::{ + ComponentKey, Config, ConfigDiff, EnrichmentTableOuter, HealthcheckOptions, Inputs, + OutputId, Resource, + }, event::EventArray, extra_context::ExtraContext, shutdown::SourceShutdownCoordinator, @@ -419,7 +422,25 @@ impl RunningTopology { let remove_sink = diff .sinks .removed_and_changed() - .map(|key| (key, self.config.sink(key).unwrap().resources(key))); + .map(|key| { + ( + key, + self.config + .sink(key) + .map(|s| s.resources(key)) + .unwrap_or_default(), + ) + }) + .chain( + diff.enrichment_tables + .removed_and_changed() + .filter_map(|key| { + self.config + .enrichment_table(key) + .and_then(|t| t.as_sink()) + .map(|s| (key, s.resources(key))) + }), + ); let add_source = diff .sources .changed_and_added() @@ -427,7 +448,25 @@ impl RunningTopology { let add_sink = diff .sinks .changed_and_added() - .map(|key| (key, new_config.sink(key).unwrap().resources(key))); + .map(|key| { + ( + key, + new_config + .sink(key) + .map(|s| s.resources(key)) + .unwrap_or_default(), + ) + }) + .chain( + diff.enrichment_tables + .changed_and_added() + .filter_map(|key| { + self.config + .enrichment_table(key) + .and_then(|t| t.as_sink()) + .map(|s| (key, s.resources(key))) + }), + ); let conflicts = Resource::conflicts( remove_sink.map(|(key, value)| ((true, key), value)).chain( add_sink @@ -450,7 +489,17 @@ impl RunningTopology { .to_change .iter() .filter(|&key| { - self.config.sink(key).unwrap().buffer == new_config.sink(key).unwrap().buffer + self.config.sink(key).map(|s| s.buffer.clone()).or_else(|| { + self.config + .enrichment_table(key) + .and_then(EnrichmentTableOuter::as_sink) + .map(|s| s.buffer) + }) == new_config.sink(key).map(|s| s.buffer.clone()).or_else(|| { + self.config + .enrichment_table(key) + .and_then(EnrichmentTableOuter::as_sink) + .map(|s| s.buffer) + }) }) .cloned() .collect::>(); @@ -463,7 +512,18 @@ impl RunningTopology { .collect::>(); // First, we remove any inputs to removed sinks so they can naturally shut down. - for key in &diff.sinks.to_remove { + let removed_sinks = diff + .sinks + .to_remove + .iter() + .chain(diff.enrichment_tables.to_remove.iter().filter(|key| { + self.config + .enrichment_table(key) + .and_then(EnrichmentTableOuter::as_sink) + .is_some() + })) + .collect::>(); + for key in &removed_sinks { debug!(component = %key, "Removing sink."); self.remove_inputs(key, diff, new_config).await; } @@ -472,7 +532,18 @@ impl RunningTopology { // they can naturally shutdown and allow us to recover their buffers if possible. let mut buffer_tx = HashMap::new(); - for key in &diff.sinks.to_change { + let sinks_to_change = diff + .sinks + .to_change + .iter() + .chain(diff.enrichment_tables.to_change.iter().filter(|key| { + self.config + .enrichment_table(key) + .and_then(EnrichmentTableOuter::as_sink) + .is_some() + })) + .collect::>(); + for key in &sinks_to_change { debug!(component = %key, "Changing sink."); if reuse_buffers.contains(key) { self.detach_triggers @@ -491,7 +562,7 @@ impl RunningTopology { // basically a no-op since we're reusing the same buffer) than it is to pass around // info about which sinks are having their buffers reused and treat them differently // at other stages. - buffer_tx.insert(key.clone(), self.inputs.get(key).unwrap().clone()); + buffer_tx.insert((*key).clone(), self.inputs.get(key).unwrap().clone()); } self.remove_inputs(key, diff, new_config).await; } @@ -502,7 +573,7 @@ impl RunningTopology { // // If a sink we're removing isn't tying up any resource that a changed/added sink depends // on, we don't bother waiting for it to shutdown. - for key in &diff.sinks.to_remove { + for key in &removed_sinks { let previous = self.tasks.remove(key).unwrap(); if wait_for_sinks.contains(key) { debug!(message = "Waiting for sink to shutdown.", %key); @@ -513,7 +584,7 @@ impl RunningTopology { } let mut buffers = HashMap::::new(); - for key in &diff.sinks.to_change { + for key in &sinks_to_change { if wait_for_sinks.contains(key) { let previous = self.tasks.remove(key).unwrap(); debug!(message = "Waiting for sink to shutdown.", %key); @@ -533,7 +604,7 @@ impl RunningTopology { _ => unreachable!(), }; - buffers.insert(key.clone(), (tx, Arc::new(Mutex::new(Some(rx))))); + buffers.insert((*key).clone(), (tx, Arc::new(Mutex::new(Some(rx))))); } } } @@ -567,6 +638,17 @@ impl RunningTopology { self.inputs_tap_metadata.remove(key); } + let removed_sinks = diff.enrichment_tables.to_remove.iter().filter(|key| { + self.config + .enrichment_table(key) + .and_then(EnrichmentTableOuter::as_sink) + .is_some() + }); + for key in removed_sinks { + // Sinks only have inputs + self.inputs_tap_metadata.remove(key); + } + for key in diff.sources.changed_and_added() { if let Some(task) = new_pieces.tasks.get(key) { self.outputs_tap_metadata @@ -613,6 +695,15 @@ impl RunningTopology { debug!(component = %key, "Connecting inputs for sink."); self.setup_inputs(key, diff, new_pieces).await; } + let added_changed_tables: Vec<&ComponentKey> = diff + .enrichment_tables + .changed_and_added() + .filter(|k| new_pieces.tasks.contains_key(k)) + .collect(); + for key in added_changed_tables { + debug!(component = %key, "Connecting inputs for enrichment table sink."); + self.setup_inputs(key, diff, new_pieces).await; + } // We do a final pass here to reconnect unchanged components. // @@ -847,6 +938,30 @@ impl RunningTopology { trace!(message = "Spawning new sink.", key = %key); self.spawn_sink(key, &mut new_pieces); } + + let changed_tables: Vec<&ComponentKey> = diff + .enrichment_tables + .to_change + .iter() + .filter(|k| new_pieces.tasks.contains_key(k)) + .collect(); + + let added_tables: Vec<&ComponentKey> = diff + .enrichment_tables + .to_add + .iter() + .filter(|k| new_pieces.tasks.contains_key(k)) + .collect(); + + for key in changed_tables { + debug!(message = "Spawning changed enrichment table sink.", key = %key); + self.spawn_sink(key, &mut new_pieces); + } + + for key in added_tables { + debug!(message = "Spawning enrichment table new sink.", key = %key); + self.spawn_sink(key, &mut new_pieces); + } } fn spawn_sink(&mut self, key: &ComponentKey, new_pieces: &mut builder::TopologyPieces) { diff --git a/website/content/en/docs/reference/configuration/_index.md b/website/content/en/docs/reference/configuration/_index.md index acd0a3dcdc..b6effbcd91 100644 --- a/website/content/en/docs/reference/configuration/_index.md +++ b/website/content/en/docs/reference/configuration/_index.md @@ -435,6 +435,80 @@ sinks: inputs: ["app*", "system_logs"] ``` +### Enrichment tables + +#### Memory enrichment table + +Memory enrichment table has to be used as a sink to feed it data, which can then be queried like any +other enrichment table. The data has to conform to a specific format - the memory table will only +accept [VRL objects](/docs/reference/vrl/expressions/#object), where each key-value pair will be +stored as a separate entry in the table, associating the value with the key in the table. Value here +can be any VRL type. + +```yaml +enrichment_tables: + memory_table: + type: memory + ttl: 60 + flush_interval: 5 + inputs: ["cache_generator"] + +sources: + demo_logs_test: + type: "demo_logs" + format: "json" + +transforms: + demo_logs_processor: + type: "remap" + inputs: ["demo_logs_test"] + source: | + . = parse_json!(.message) + user_id = get!(., path: ["user-identifier"]) + + # Look for existing value in the table, using "user-identifier" as key + existing, err = get_enrichment_table_record("memory_table", { "key": user_id }) + + if err == null { + # Value found, just use the cached value + # In this case existing looks like this { "key": user_id, "value": {}, "ttl": 50 } + # Where value is the value we cached, ttl is the time left before this value is removed from + # the cache and key is the key we queried the table with + . = existing.value + .source = "cache" + } else { + # Do some processing, because we don't have this value in the table + .referer = parse_url!(.referer) + .referer.host = encode_punycode!(.referer.host) + .source = "transform" + } + + cache_generator: + type: "remap" + inputs: ["demo_logs_processor"] + source: | + existing, err = get_enrichment_table_record("memory_table", { "key": get!(., path: ["user-identifier"]) }) + if err != null { + # We don't have this key cached, so we need to prepare it for the table + data = . + # Since the memory enrichment table takes in all key value pairs it receives and stores them + # We want to produce an object that has the value of "user-identifier" as its key and + # rest of the object as its value + . = set!(value: {}, path: [get!(data, path: ["user-identifier"])], data: data) + } else { + . = {} + } + +# We can observe that after some time that some events have "source" set to "cache" +sinks: + console: + inputs: ["demo_logs_processor"] + target: "stdout" + type: "console" + encoding: + codec: "json" +``` + ## Sections {{< sections >}} diff --git a/website/cue/reference/configuration.cue b/website/cue/reference/configuration.cue index 277447950e..fe63239914 100644 --- a/website/cue/reference/configuration.cue +++ b/website/cue/reference/configuration.cue @@ -99,6 +99,7 @@ configuration: { * [CSV](\(urls.csv)) files * [MaxMind](\(urls.maxmind)) databases + * In-memory storage For the lookup in the enrichment tables to be as performant as possible, the data is indexed according to the fields that are used in the search. Note that indices can only be created for fields for which an @@ -116,9 +117,10 @@ configuration: { required: true type: string: { enum: { - "file": "Enrich data from a CSV file." - "geoip": "Enrich data from a [GeoIp](\(urls.maxmind_geoip2)) [MaxMind](\(urls.maxmind)) database." - "mmdb": "Enrich data from any [MaxMind](\(urls.maxmind)) database." + "file": "Enrich data from a CSV file." + "geoip": "Enrich data from a [GeoIp](\(urls.maxmind_geoip2)) [MaxMind](\(urls.maxmind)) database." + "mmdb": "Enrich data from any [MaxMind](\(urls.maxmind)) database." + "memory": "Enrich data from memory, which can be populated by using the table as a sink." } } } @@ -277,6 +279,93 @@ configuration: { } } } + type: object: options: { + memory: { + required: true + description: """ + Configuration options for in-memory enrichment table. + + This enrichment table only supports lookup with key field. + + To write data into this table, you have to define inputs to use this the table as a sink. + They are expected to produce objects, where each key-value pair is stored as a separate + record in the table. [Read more on how to use this component](\(urls.vector_enrichment_memory_how_it_works)). + """ + type: object: options: { + inputs: base.components.sinks.configuration.inputs + ttl: { + description: """ + TTL (time-to-live in seconds) is used to limit the lifetime of data stored in the cache. + When TTL expires, data behind a specific key in the cache is removed. + TTL is reset when the key is replaced. + """ + required: false + type: uint: { + default: 600 + examples: [3600, 21600] + } + } + scan_interval: { + description: """ + The scan interval used to look for expired records. This is provided + as an optimization to ensure that TTL is updated, but without doing + too many cache scans. + """ + required: false + type: uint: { + default: 30 + examples: [600, 60] + } + } + flush_interval: { + description: """ + The interval used for making writes visible in the table. + Longer intervals might get better performance, + but there is a longer delay before the data is visible in the table. + Since every TTL scan makes its changes visible, only use this value + if it is shorter than the `scan_interval`. + + By default, all writes are made visible immediately. + """ + required: false + type: uint: { + default: 0 + examples: [15, 30] + } + } + max_byte_size: { + description: """ + Maximum size of the table in bytes. All insertions that make this table bigger than the maximum size are rejected. + + By default, there is no size limit. + """ + required: false + type: uint: { + default: 0 + examples: [64000, 1000000000] + } + } + internal_metrics: { + description: """ + Configuration of internal metrics for enrichment memory table. + """ + required: false + type: object: options: { + include_key_tag: { + description: """ + Determines whether to include the key tag on internal metrics. + + This is useful for distinguishing between different keys while monitoring. However, the tag's + cardinality is unbounded. + """ + required: false + type: bool: default: false + } + } + } + } + } + } } schema: { common: false diff --git a/website/cue/reference/urls.cue b/website/cue/reference/urls.cue index 04e27fa6b0..e6d9edc931 100644 --- a/website/cue/reference/urls.cue +++ b/website/cue/reference/urls.cue @@ -560,6 +560,8 @@ urls: { vector_download: "/releases/latest/download/" vector_download_nightly: "/releases/nightly/download/" vector_enriching_transforms: "/components/?functions%5B%5D=enrich" + vector_enrichment_memory: "/docs/reference/configuration/global-options/#enrichment_tables.memory" + vector_enrichment_memory_how_it_works: "/docs/reference/configuration/#memory-enrichment-table" vector_file_source: "/docs/reference/configuration/sources/file/" vector_exec_source: "/docs/reference/configuration/sources/exec" vector_file_source: "/docs/reference/configuration/sources/file/" From 5bf6071554ed155817b986e75b8b7c1a2040ef98 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 23 Jan 2025 15:27:09 +0000 Subject: [PATCH 024/658] chore(deps): Bump proptest from 1.5.0 to 1.6.0 (#22172) * chore(deps): Bump proptest from 1.5.0 to 1.6.0 Bumps [proptest](https://github.com/proptest-rs/proptest) from 1.5.0 to 1.6.0. - [Release notes](https://github.com/proptest-rs/proptest/releases) - [Changelog](https://github.com/proptest-rs/proptest/blob/main/CHANGELOG.md) - [Commits](https://github.com/proptest-rs/proptest/commits) --- updated-dependencies: - dependency-name: proptest dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * dd-rust-license-tool write --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Pavlos Rontidis --- Cargo.lock | 8 ++++---- Cargo.toml | 2 +- LICENSE-3rdparty.csv | 1 - lib/vector-buffers/Cargo.toml | 2 +- lib/vector-core/Cargo.toml | 4 ++-- lib/vector-stream/Cargo.toml | 2 +- 6 files changed, 9 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dfdcac6a86..2ca453ed5b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7484,12 +7484,12 @@ dependencies = [ [[package]] name = "proptest" -version = "1.5.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4c2511913b88df1637da85cc8d96ec8e43a3f8bb8ccb71ee1ac240d6f3df58d" +checksum = "14cae93065090804185d3b75f0bf93b8eeda30c7a9b4a33d3bdb3988d6229e50" dependencies = [ - "bit-set 0.5.3", - "bit-vec 0.6.3", + "bit-set 0.8.0", + "bit-vec 0.8.0", "bitflags 2.8.0", "lazy_static", "num-traits", diff --git a/Cargo.toml b/Cargo.toml index d2e01c3fa2..34ded9f41d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -148,7 +148,7 @@ metrics = "0.24.1" metrics-tracing-context = { version = "0.17.0", default-features = false } metrics-util = { version = "0.18.0", default-features = false, features = ["registry"] } pin-project = { version = "1.1.8", default-features = false } -proptest = { version = "1.5" } +proptest = { version = "1.6" } proptest-derive = { version = "0.5.1" } prost = { version = "0.12", default-features = false, features = ["std"] } prost-build = { version = "0.12", default-features = false } diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 0f9b5e3963..08fa4e1b45 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -91,7 +91,6 @@ base64,https://github.com/marshallpierce/rust-base64,MIT OR Apache-2.0,Marshall base64-simd,https://github.com/Nugine/simd,MIT,The base64-simd Authors base64ct,https://github.com/RustCrypto/formats/tree/master/base64ct,Apache-2.0 OR MIT,RustCrypto Developers bit-set,https://github.com/contain-rs/bit-set,Apache-2.0 OR MIT,Alexis Beingessner -bit-set,https://github.com/contain-rs/bit-set,MIT OR Apache-2.0,Alexis Beingessner bit-vec,https://github.com/contain-rs/bit-vec,Apache-2.0 OR MIT,Alexis Beingessner bit-vec,https://github.com/contain-rs/bit-vec,MIT OR Apache-2.0,Alexis Beingessner bitflags,https://github.com/bitflags/bitflags,MIT OR Apache-2.0,The Rust Project Developers diff --git a/lib/vector-buffers/Cargo.toml b/lib/vector-buffers/Cargo.toml index b43f922148..daa96db19a 100644 --- a/lib/vector-buffers/Cargo.toml +++ b/lib/vector-buffers/Cargo.toml @@ -40,7 +40,7 @@ crossbeam-queue = "0.3.12" hdrhistogram = "7.5.4" metrics-tracing-context.workspace = true metrics-util = { workspace = true, features = ["debugging"] } -proptest = "1.5" +proptest = "1.6" quickcheck = "1.0" rand = "0.8.5" serde_yaml = { version = "0.9", default-features = false } diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml index 3c5c4d811c..2368e46a01 100644 --- a/lib/vector-core/Cargo.toml +++ b/lib/vector-core/Cargo.toml @@ -36,7 +36,7 @@ ordered-float = { version = "4.6.0", default-features = false } openssl = { version = "0.10.68", default-features = false, features = ["vendored"] } parking_lot = { version = "0.12.3", default-features = false } pin-project.workspace = true -proptest = { version = "1.5", optional = true } +proptest = { version = "1.6", optional = true } prost-types.workspace = true prost .workspace = true quanta = { version = "0.12.5", default-features = false } @@ -81,7 +81,7 @@ criterion = { version = "0.5.1", features = ["html_reports"] } env-test-util = "1.0.1" quickcheck = "1" quickcheck_macros = "1" -proptest = "1.5" +proptest = "1.6" similar-asserts = "1.6.1" tokio-test = "0.4.4" toml.workspace = true diff --git a/lib/vector-stream/Cargo.toml b/lib/vector-stream/Cargo.toml index 45b1e77aaa..052c8b0fed 100644 --- a/lib/vector-stream/Cargo.toml +++ b/lib/vector-stream/Cargo.toml @@ -19,6 +19,6 @@ vector-common = { path = "../vector-common" } vector-core = { path = "../vector-core" } [dev-dependencies] -proptest = "1.5" +proptest = "1.6" rand = "0.8.5" rand_distr = "0.4.3" From fd515ea86e3a3378c69953a4a59556045eb93fe4 Mon Sep 17 00:00:00 2001 From: Sainath Singineedi <44405294+sainad2222@users.noreply.github.com> Date: Fri, 24 Jan 2025 02:44:51 +0530 Subject: [PATCH 025/658] feat(new sink): Keep sink (#22072) * feat(new sink): Keep * feat(new sink): add changelog and cue references * docs: fix documention * docs: fix spell check for api key example * docs: add missing cue files * docs: add non base cue for keep sink * fix: fix name thing in services keep sink * fix: change how_it_works section * fix: indentation for urls * fix: spelling v2 * fix: cue formatting --- .github/actions/spelling/allow.txt | 3 + Cargo.toml | 2 + changelog.d/21965-add-keep-sink.feature.md | 3 + src/sinks/keep/config.rs | 175 +++++++++++ src/sinks/keep/encoder.rs | 50 +++ src/sinks/keep/mod.rs | 10 + src/sinks/keep/request_builder.rs | 48 +++ src/sinks/keep/service.rs | 33 ++ src/sinks/keep/sink.rs | 77 +++++ src/sinks/mod.rs | 2 + .../reference/configuration/sinks/keep.md | 14 + .../reference/components/sinks/base/keep.cue | 289 ++++++++++++++++++ .../cue/reference/components/sinks/keep.cue | 95 ++++++ website/cue/reference/services/keep.cue | 10 + website/cue/reference/urls.cue | 2 + website/data/redirects.yaml | 1 + 16 files changed, 814 insertions(+) create mode 100644 changelog.d/21965-add-keep-sink.feature.md create mode 100644 src/sinks/keep/config.rs create mode 100644 src/sinks/keep/encoder.rs create mode 100644 src/sinks/keep/mod.rs create mode 100644 src/sinks/keep/request_builder.rs create mode 100644 src/sinks/keep/service.rs create mode 100644 src/sinks/keep/sink.rs create mode 100644 website/content/en/docs/reference/configuration/sinks/keep.md create mode 100644 website/cue/reference/components/sinks/base/keep.cue create mode 100644 website/cue/reference/components/sinks/keep.cue create mode 100644 website/cue/reference/services/keep.cue diff --git a/.github/actions/spelling/allow.txt b/.github/actions/spelling/allow.txt index f0216c0af3..3abfa0f9aa 100644 --- a/.github/actions/spelling/allow.txt +++ b/.github/actions/spelling/allow.txt @@ -7,6 +7,7 @@ Airis Airpad Alcatel Alexey +alertmanager Alibaba Allfine Allview @@ -360,6 +361,8 @@ jsonnet jsontag jvm kenton +keepappkey +keephq konqueror kube kubeadm diff --git a/Cargo.toml b/Cargo.toml index 34ded9f41d..27c4f6eb9e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -726,6 +726,7 @@ sinks-logs = [ "sinks-humio", "sinks-influxdb", "sinks-kafka", + "sinks-keep", "sinks-loki", "sinks-mezmo", "sinks-mqtt", @@ -792,6 +793,7 @@ sinks-http = [] sinks-humio = ["sinks-splunk_hec", "transforms-metric_to_log"] sinks-influxdb = [] sinks-kafka = ["dep:rdkafka"] +sinks-keep = [] sinks-mezmo = [] sinks-loki = ["loki-logproto"] sinks-mqtt = ["dep:rumqttc"] diff --git a/changelog.d/21965-add-keep-sink.feature.md b/changelog.d/21965-add-keep-sink.feature.md new file mode 100644 index 0000000000..00328d73e8 --- /dev/null +++ b/changelog.d/21965-add-keep-sink.feature.md @@ -0,0 +1,3 @@ +A new sink for keep was added + +authors: sainad2222 diff --git a/src/sinks/keep/config.rs b/src/sinks/keep/config.rs new file mode 100644 index 0000000000..b5bc91578a --- /dev/null +++ b/src/sinks/keep/config.rs @@ -0,0 +1,175 @@ +//! Configuration for the `keep` sink. + +use bytes::Bytes; +use futures::FutureExt; +use http::{Request, StatusCode, Uri}; +use vector_lib::configurable::configurable_component; +use vector_lib::sensitive_string::SensitiveString; +use vrl::value::Kind; + +use crate::{ + http::HttpClient, + sinks::{ + prelude::*, + util::{ + http::{http_response_retry_logic, HttpService}, + BatchConfig, BoxedRawValue, + }, + }, +}; + +use super::{ + encoder::KeepEncoder, request_builder::KeepRequestBuilder, service::KeepSvcRequestBuilder, + sink::KeepSink, +}; + +pub(super) const HTTP_HEADER_KEEP_API_KEY: &str = "x-api-key"; + +/// Configuration for the `keep` sink. +#[configurable_component(sink("keep", "Deliver log events to Keep."))] +#[derive(Clone, Debug)] +pub struct KeepConfig { + /// Keeps endpoint to send logs to + #[serde(default = "default_endpoint")] + #[configurable(metadata( + docs::examples = "https://backend.keep.com:8081/alerts/event/vectordev?provider_id=test", + ))] + #[configurable(validation(format = "uri"))] + pub(super) endpoint: String, + + /// The API key that is used to authenticate against Keep. + #[configurable(metadata(docs::examples = "${KEEP_API_KEY}"))] + #[configurable(metadata(docs::examples = "keepappkey"))] + api_key: SensitiveString, + + #[configurable(derived)] + #[serde(default)] + batch: BatchConfig, + + #[configurable(derived)] + #[serde(default)] + request: TowerRequestConfig, + + #[configurable(derived)] + #[serde(default, skip_serializing_if = "crate::serde::is_default")] + encoding: Transformer, + + #[configurable(derived)] + #[serde( + default, + deserialize_with = "crate::serde::bool_or_struct", + skip_serializing_if = "crate::serde::is_default" + )] + acknowledgements: AcknowledgementsConfig, +} + +fn default_endpoint() -> String { + "http://localhost:8080/alerts/event/vectordev?provider_id=test".to_string() +} + +#[derive(Clone, Copy, Debug, Default)] +struct KeepDefaultBatchSettings; + +impl SinkBatchSettings for KeepDefaultBatchSettings { + const MAX_EVENTS: Option = None; + const MAX_BYTES: Option = Some(100_000); + const TIMEOUT_SECS: f64 = 1.0; +} + +impl GenerateConfig for KeepConfig { + fn generate_config() -> toml::Value { + toml::from_str( + r#"api_key = "${KEEP_API_KEY}" + "#, + ) + .unwrap() + } +} + +#[async_trait::async_trait] +#[typetag::serde(name = "keep")] +impl SinkConfig for KeepConfig { + async fn build(&self, cx: SinkContext) -> crate::Result<(VectorSink, Healthcheck)> { + let batch_settings = self.batch.validate()?.into_batcher_settings()?; + + let request_builder = KeepRequestBuilder { + encoder: KeepEncoder { + transformer: self.encoding.clone(), + }, + // TODO: add compression support + compression: Compression::None, + }; + + let uri: Uri = self.endpoint.clone().try_into()?; + let keep_service_request_builder = KeepSvcRequestBuilder { + uri: uri.clone(), + api_key: self.api_key.clone(), + }; + + let client = HttpClient::new(None, cx.proxy())?; + + let service = HttpService::new(client.clone(), keep_service_request_builder); + + let request_limits = self.request.into_settings(); + + let service = ServiceBuilder::new() + .settings(request_limits, http_response_retry_logic()) + .service(service); + + let sink = KeepSink::new(service, batch_settings, request_builder); + + let healthcheck = healthcheck(uri, self.api_key.clone(), client).boxed(); + + Ok((VectorSink::from_event_streamsink(sink), healthcheck)) + } + + fn input(&self) -> Input { + let requirement = Requirement::empty().optional_meaning("timestamp", Kind::timestamp()); + + Input::log().with_schema_requirement(requirement) + } + + fn acknowledgements(&self) -> &AcknowledgementsConfig { + &self.acknowledgements + } +} + +async fn healthcheck(uri: Uri, api_key: SensitiveString, client: HttpClient) -> crate::Result<()> { + let request = Request::post(uri).header(HTTP_HEADER_KEEP_API_KEY, api_key.inner()); + let body = crate::serde::json::to_bytes(&Vec::::new()) + .unwrap() + .freeze(); + let req: Request = request.body(body)?; + let req = req.map(hyper::Body::from); + + let res = client.send(req).await?; + + let status = res.status(); + let body = hyper::body::to_bytes(res.into_body()).await?; + + match status { + StatusCode::OK => Ok(()), // Healthcheck passed + StatusCode::BAD_REQUEST => Ok(()), // Healthcheck failed due to client error but is still considered valid + StatusCode::ACCEPTED => Ok(()), // Consider healthcheck passed if server accepted request + StatusCode::UNAUTHORIZED => { + // Handle unauthorized errors + let json: serde_json::Value = serde_json::from_slice(&body[..])?; + let message = json + .as_object() + .and_then(|o| o.get("error")) + .and_then(|s| s.as_str()) + .unwrap_or("Token is not valid, 401 returned.") + .to_string(); + Err(message.into()) + } + _ => { + // Handle other unexpected statuses + let body = String::from_utf8_lossy(&body[..]); + Err(format!( + "Server returned unexpected error status: {} body: {}", + status, body + ) + .into()) + } + } +} diff --git a/src/sinks/keep/encoder.rs b/src/sinks/keep/encoder.rs new file mode 100644 index 0000000000..95e44ea170 --- /dev/null +++ b/src/sinks/keep/encoder.rs @@ -0,0 +1,50 @@ +//! Encoding for the `keep` sink. + +use bytes::Bytes; +use serde_json::{json, to_vec}; +use std::io; + +use crate::sinks::{ + prelude::*, + util::encoding::{write_all, Encoder as SinkEncoder}, +}; + +pub(super) struct KeepEncoder { + pub(super) transformer: Transformer, +} + +impl SinkEncoder> for KeepEncoder { + fn encode_input( + &self, + events: Vec, + writer: &mut dyn io::Write, + ) -> io::Result<(usize, GroupedCountByteSize)> { + let mut byte_size = telemetry().create_request_count_byte_size(); + let n_events = events.len(); + let mut json_events: Vec = Vec::with_capacity(n_events); + + for mut event in events { + self.transformer.transform(&mut event); + + byte_size.add_event(&event, event.estimated_json_encoded_size_of()); + + let mut data = json!(event.as_log()); + if let Some(message) = data.get("message") { + if let Some(message_str) = message.as_str() { + // Parse the JSON string in `message` + let parsed_message: serde_json::Value = serde_json::from_str(message_str)?; + + // Reassign the parsed JSON back to `message` + data["message"] = parsed_message; + } + } + data["keep_source_type"] = json!(event.source_id()); + + json_events.push(data); + } + + let body = Bytes::from(to_vec(&serde_json::Value::Array(json_events))?); + + write_all(writer, n_events, body.as_ref()).map(|()| (body.len(), byte_size)) + } +} diff --git a/src/sinks/keep/mod.rs b/src/sinks/keep/mod.rs new file mode 100644 index 0000000000..594b5fa963 --- /dev/null +++ b/src/sinks/keep/mod.rs @@ -0,0 +1,10 @@ +//! The Keep [`vector_lib::sink::VectorSink`]. +//! +//! This module contains the [`vector_lib::sink::VectorSink`] instance that is responsible for +//! taking a stream of [`vector_lib::event::Event`]s and forwarding them to the Keep service. + +mod config; +mod encoder; +mod request_builder; +mod service; +mod sink; diff --git a/src/sinks/keep/request_builder.rs b/src/sinks/keep/request_builder.rs new file mode 100644 index 0000000000..5c58d3fee8 --- /dev/null +++ b/src/sinks/keep/request_builder.rs @@ -0,0 +1,48 @@ +//! `RequestBuilder` implementation for the `keep` sink. + +use bytes::Bytes; +use std::io; + +use crate::sinks::{prelude::*, util::http::HttpRequest}; + +use super::encoder::KeepEncoder; + +pub(super) struct KeepRequestBuilder { + pub(super) encoder: KeepEncoder, + pub(super) compression: Compression, +} + +impl RequestBuilder> for KeepRequestBuilder { + type Metadata = EventFinalizers; + type Events = Vec; + type Encoder = KeepEncoder; + type Payload = Bytes; + type Request = HttpRequest<()>; + type Error = io::Error; + + fn compression(&self) -> Compression { + self.compression + } + + fn encoder(&self) -> &Self::Encoder { + &self.encoder + } + + fn split_input( + &self, + mut events: Vec, + ) -> (Self::Metadata, RequestMetadataBuilder, Self::Events) { + let finalizers = events.take_finalizers(); + let builder = RequestMetadataBuilder::from_events(&events); + (finalizers, builder, events) + } + + fn build_request( + &self, + metadata: Self::Metadata, + request_metadata: RequestMetadata, + payload: EncodeResult, + ) -> Self::Request { + HttpRequest::new(payload.into_payload(), metadata, request_metadata, ()) + } +} diff --git a/src/sinks/keep/service.rs b/src/sinks/keep/service.rs new file mode 100644 index 0000000000..89b9ebaacf --- /dev/null +++ b/src/sinks/keep/service.rs @@ -0,0 +1,33 @@ +//! Service implementation for the `keep` sink. + +use bytes::Bytes; +use http::{Request, Uri}; +use vector_lib::sensitive_string::SensitiveString; + +use crate::sinks::{ + util::http::{HttpRequest, HttpServiceRequestBuilder}, + HTTPRequestBuilderSnafu, +}; +use snafu::ResultExt; + +use super::config::HTTP_HEADER_KEEP_API_KEY; + +#[derive(Debug, Clone)] +pub(super) struct KeepSvcRequestBuilder { + pub(super) uri: Uri, + pub(super) api_key: SensitiveString, +} + +impl HttpServiceRequestBuilder<()> for KeepSvcRequestBuilder { + fn build(&self, mut request: HttpRequest<()>) -> Result, crate::Error> { + let builder = + Request::post(&self.uri).header(HTTP_HEADER_KEEP_API_KEY, self.api_key.inner()); + + let builder = builder.header("Content-Type".to_string(), "application/json".to_string()); + + builder + .body(request.take_payload()) + .context(HTTPRequestBuilderSnafu) + .map_err(Into::into) + } +} diff --git a/src/sinks/keep/sink.rs b/src/sinks/keep/sink.rs new file mode 100644 index 0000000000..0a990767f9 --- /dev/null +++ b/src/sinks/keep/sink.rs @@ -0,0 +1,77 @@ +//! Implementation of the `keep` sink. + +use crate::sinks::{ + prelude::*, + util::http::{HttpJsonBatchSizer, HttpRequest}, +}; + +use super::request_builder::KeepRequestBuilder; + +pub(super) struct KeepSink { + service: S, + batch_settings: BatcherSettings, + request_builder: KeepRequestBuilder, +} + +impl KeepSink +where + S: Service> + Send + 'static, + S::Future: Send + 'static, + S::Response: DriverResponse + Send + 'static, + S::Error: std::fmt::Debug + Into + Send, +{ + /// Creates a new `keep`. + pub(super) const fn new( + service: S, + batch_settings: BatcherSettings, + request_builder: KeepRequestBuilder, + ) -> Self { + Self { + service, + batch_settings, + request_builder, + } + } + + async fn run_inner(self: Box, input: BoxStream<'_, Event>) -> Result<(), ()> { + input + // Batch the input stream with size calculation based on the estimated encoded json size + .batched(self.batch_settings.as_item_size_config(HttpJsonBatchSizer)) + // Build requests with default concurrency limit. + .request_builder( + default_request_builder_concurrency_limit(), + self.request_builder, + ) + // Filter out any errors that occurred in the request building. + .filter_map(|request| async move { + match request { + Err(error) => { + emit!(SinkRequestBuildError { error }); + None + } + Ok(req) => Some(req), + } + }) + // Generate the driver that will send requests and handle retries, + // event finalization, and logging/internal metric reporting. + .into_driver(self.service) + .run() + .await + } +} + +#[async_trait::async_trait] +impl StreamSink for KeepSink +where + S: Service> + Send + 'static, + S::Future: Send + 'static, + S::Response: DriverResponse + Send + 'static, + S::Error: std::fmt::Debug + Into + Send, +{ + async fn run( + self: Box, + input: futures_util::stream::BoxStream<'_, Event>, + ) -> Result<(), ()> { + self.run_inner(input).await + } +} diff --git a/src/sinks/mod.rs b/src/sinks/mod.rs index d3c1e1982a..3cf5613324 100644 --- a/src/sinks/mod.rs +++ b/src/sinks/mod.rs @@ -70,6 +70,8 @@ pub mod humio; pub mod influxdb; #[cfg(feature = "sinks-kafka")] pub mod kafka; +#[cfg(feature = "sinks-keep")] +pub mod keep; #[cfg(feature = "sinks-loki")] pub mod loki; #[cfg(feature = "sinks-mezmo")] diff --git a/website/content/en/docs/reference/configuration/sinks/keep.md b/website/content/en/docs/reference/configuration/sinks/keep.md new file mode 100644 index 0000000000..1c212c9567 --- /dev/null +++ b/website/content/en/docs/reference/configuration/sinks/keep.md @@ -0,0 +1,14 @@ +--- +title: Keep +description: Deliver log events to [Keep](https://www.keephq.dev) +component_kind: sink +layout: component +tags: ["keep", "component", "sink", "logs"] +--- + +{{/* +This doc is generated using: + +1. The template in layouts/docs/component.html +2. The relevant CUE data in cue/reference/components/... +*/}} diff --git a/website/cue/reference/components/sinks/base/keep.cue b/website/cue/reference/components/sinks/base/keep.cue new file mode 100644 index 0000000000..52d905438c --- /dev/null +++ b/website/cue/reference/components/sinks/base/keep.cue @@ -0,0 +1,289 @@ +package metadata + +base: components: sinks: keep: configuration: { + acknowledgements: { + description: """ + Controls how acknowledgements are handled for this sink. + + See [End-to-end Acknowledgements][e2e_acks] for more information on how event acknowledgement is handled. + + [e2e_acks]: https://vector.dev/docs/about/under-the-hood/architecture/end-to-end-acknowledgements/ + """ + required: false + type: object: options: enabled: { + description: """ + Whether or not end-to-end acknowledgements are enabled. + + When enabled for a sink, any source connected to that sink where the source supports + end-to-end acknowledgements as well, waits for events to be acknowledged by **all + connected** sinks before acknowledging them at the source. + + Enabling or disabling acknowledgements at the sink level takes precedence over any global + [`acknowledgements`][global_acks] configuration. + + [global_acks]: https://vector.dev/docs/reference/configuration/global-options/#acknowledgements + """ + required: false + type: bool: {} + } + } + api_key: { + description: "The API key that is used to authenticate against Keep." + required: true + type: string: examples: ["${KEEP_API_KEY}", "keepappkey"] + } + batch: { + description: "Event batching behavior." + required: false + type: object: options: { + max_bytes: { + description: """ + The maximum size of a batch that is processed by a sink. + + This is based on the uncompressed size of the batched events, before they are + serialized/compressed. + """ + required: false + type: uint: { + default: 100000 + unit: "bytes" + } + } + max_events: { + description: "The maximum size of a batch before it is flushed." + required: false + type: uint: unit: "events" + } + timeout_secs: { + description: "The maximum age of a batch before it is flushed." + required: false + type: float: { + default: 1.0 + unit: "seconds" + } + } + } + } + encoding: { + description: "Transformations to prepare an event for serialization." + required: false + type: object: options: { + except_fields: { + description: "List of fields that are excluded from the encoded event." + required: false + type: array: items: type: string: {} + } + only_fields: { + description: "List of fields that are included in the encoded event." + required: false + type: array: items: type: string: {} + } + timestamp_format: { + description: "Format used for timestamp fields." + required: false + type: string: enum: { + rfc3339: "Represent the timestamp as a RFC 3339 timestamp." + unix: "Represent the timestamp as a Unix timestamp." + unix_float: "Represent the timestamp as a Unix timestamp in floating point." + unix_ms: "Represent the timestamp as a Unix timestamp in milliseconds." + unix_ns: "Represent the timestamp as a Unix timestamp in nanoseconds." + unix_us: "Represent the timestamp as a Unix timestamp in microseconds" + } + } + } + } + endpoint: { + description: "Keeps endpoint to send logs to" + required: false + type: string: { + default: "http://localhost:8080/alerts/event/vectordev?provider_id=test" + examples: ["https://backend.keep.com:8081/alerts/event/vectordev?provider_id=test"] + } + } + request: { + description: """ + Middleware settings for outbound requests. + + Various settings can be configured, such as concurrency and rate limits, timeouts, and retry behavior. + + Note that the retry backoff policy follows the Fibonacci sequence. + """ + required: false + type: object: options: { + adaptive_concurrency: { + description: """ + Configuration of adaptive concurrency parameters. + + These parameters typically do not require changes from the default, and incorrect values can lead to meta-stable or + unstable performance and sink behavior. Proceed with caution. + """ + required: false + type: object: options: { + decrease_ratio: { + description: """ + The fraction of the current value to set the new concurrency limit when decreasing the limit. + + Valid values are greater than `0` and less than `1`. Smaller values cause the algorithm to scale back rapidly + when latency increases. + + **Note**: The new limit is rounded down after applying this ratio. + """ + required: false + type: float: default: 0.9 + } + ewma_alpha: { + description: """ + The weighting of new measurements compared to older measurements. + + Valid values are greater than `0` and less than `1`. + + ARC uses an exponentially weighted moving average (EWMA) of past RTT measurements as a reference to compare with + the current RTT. Smaller values cause this reference to adjust more slowly, which may be useful if a service has + unusually high response variability. + """ + required: false + type: float: default: 0.4 + } + initial_concurrency: { + description: """ + The initial concurrency limit to use. If not specified, the initial limit is 1 (no concurrency). + + Datadog recommends setting this value to your service's average limit if you're seeing that it takes a + long time to ramp up adaptive concurrency after a restart. You can find this value by looking at the + `adaptive_concurrency_limit` metric. + """ + required: false + type: uint: default: 1 + } + max_concurrency_limit: { + description: """ + The maximum concurrency limit. + + The adaptive request concurrency limit does not go above this bound. This is put in place as a safeguard. + """ + required: false + type: uint: default: 200 + } + rtt_deviation_scale: { + description: """ + Scale of RTT deviations which are not considered anomalous. + + Valid values are greater than or equal to `0`, and we expect reasonable values to range from `1.0` to `3.0`. + + When calculating the past RTT average, we also compute a secondary “deviation” value that indicates how variable + those values are. We use that deviation when comparing the past RTT average to the current measurements, so we + can ignore increases in RTT that are within an expected range. This factor is used to scale up the deviation to + an appropriate range. Larger values cause the algorithm to ignore larger increases in the RTT. + """ + required: false + type: float: default: 2.5 + } + } + } + concurrency: { + description: """ + Configuration for outbound request concurrency. + + This can be set either to one of the below enum values or to a positive integer, which denotes + a fixed concurrency limit. + """ + required: false + type: { + string: { + default: "adaptive" + enum: { + adaptive: """ + Concurrency is managed by Vector's [Adaptive Request Concurrency][arc] feature. + + [arc]: https://vector.dev/docs/about/under-the-hood/networking/arc/ + """ + none: """ + A fixed concurrency of 1. + + Only one request can be outstanding at any given time. + """ + } + } + uint: {} + } + } + rate_limit_duration_secs: { + description: "The time window used for the `rate_limit_num` option." + required: false + type: uint: { + default: 1 + unit: "seconds" + } + } + rate_limit_num: { + description: "The maximum number of requests allowed within the `rate_limit_duration_secs` time window." + required: false + type: uint: { + default: 9223372036854775807 + unit: "requests" + } + } + retry_attempts: { + description: "The maximum number of retries to make for failed requests." + required: false + type: uint: { + default: 9223372036854775807 + unit: "retries" + } + } + retry_initial_backoff_secs: { + description: """ + The amount of time to wait before attempting the first retry for a failed request. + + After the first retry has failed, the fibonacci sequence is used to select future backoffs. + """ + required: false + type: uint: { + default: 1 + unit: "seconds" + } + } + retry_jitter_mode: { + description: "The jitter mode to use for retry backoff behavior." + required: false + type: string: { + default: "Full" + enum: { + Full: """ + Full jitter. + + The random delay is anywhere from 0 up to the maximum current delay calculated by the backoff + strategy. + + Incorporating full jitter into your backoff strategy can greatly reduce the likelihood + of creating accidental denial of service (DoS) conditions against your own systems when + many clients are recovering from a failure state. + """ + None: "No jitter." + } + } + } + retry_max_duration_secs: { + description: "The maximum amount of time to wait between retries." + required: false + type: uint: { + default: 30 + unit: "seconds" + } + } + timeout_secs: { + description: """ + The time a request can take before being aborted. + + Datadog highly recommends that you do not lower this value below the service's internal timeout, as this could + create orphaned requests, pile on retries, and result in duplicate data downstream. + """ + required: false + type: uint: { + default: 60 + unit: "seconds" + } + } + } + } +} diff --git a/website/cue/reference/components/sinks/keep.cue b/website/cue/reference/components/sinks/keep.cue new file mode 100644 index 0000000000..8c823e09b3 --- /dev/null +++ b/website/cue/reference/components/sinks/keep.cue @@ -0,0 +1,95 @@ +package metadata + +components: sinks: keep: { + title: "Keep" + + classes: { + commonly_used: false + delivery: "at_least_once" + development: "beta" + egress_method: "batch" + service_providers: ["Keep"] + stateful: false + } + + features: { + acknowledgements: true + auto_generated: true + healthcheck: enabled: true + send: { + batch: { + enabled: true + common: false + max_events: 1000 + max_bytes: 1_048_576 + timeout_secs: 1.0 + } + compression: { + enabled: false + } + encoding: { + enabled: true + codec: enabled: false + } + proxy: enabled: true + request: { + enabled: true + headers: true + } + tls: { + enabled: true + can_verify_certificate: true + can_verify_hostname: true + enabled_default: true + enabled_by_scheme: true + } + to: { + service: services.keep + + interface: { + socket: { + api: { + title: "Keep API" + url: urls.keep + } + direction: "outgoing" + protocols: ["http"] + ssl: "required" + } + } + } + } + } + + support: { + requirements: [] + warnings: [] + notices: [] + } + + configuration: base.components.sinks.keep.configuration + + input: { + logs: true + metrics: null + traces: false + } + + how_it_works: { + setup: { + title: "Setup" + body: """ + 1. Register for a free account at [platform.keephq.dev](\(urls.keep_platform)) + + 2. Go to providers tab and setup vector as a provider + """ + } + + configuration: { + title: "Configuration" + body: """ + In vector configuration source name needs to be "prometheus_alertmanager" + """ + } + } +} diff --git a/website/cue/reference/services/keep.cue b/website/cue/reference/services/keep.cue new file mode 100644 index 0000000000..db78d849d7 --- /dev/null +++ b/website/cue/reference/services/keep.cue @@ -0,0 +1,10 @@ +package metadata + +services: keep: { + name: "Keep" + thing: "\(name) alerts" + url: urls.keep + versions: null + + description: "[Keep](\(urls.keep)) is an open-source AIOps platform." +} diff --git a/website/cue/reference/urls.cue b/website/cue/reference/urls.cue index e6d9edc931..8ec9a5761a 100644 --- a/website/cue/reference/urls.cue +++ b/website/cue/reference/urls.cue @@ -293,6 +293,8 @@ urls: { kafka_partitioning_docs: "https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-Partitioningandbootstrapping" kafka_protocol: "https://kafka.apache.org/protocol" kafka_sasl: "https://docs.confluent.io/current/kafka/authentication_sasl/index.html" + keep: "https://keephq.dev" + keep_platform: "https://platform.keephq.dev" klog: "\(github)/kubernetes/klog" kubectl: "\(kubernetes)/docs/reference/kubectl/overview/" kubernetes: "https://kubernetes.io" diff --git a/website/data/redirects.yaml b/website/data/redirects.yaml index c72f6d5b33..e9e502526a 100644 --- a/website/data/redirects.yaml +++ b/website/data/redirects.yaml @@ -28,6 +28,7 @@ sinks: - influxdb_logs - influxdb_metrics - kafka +- keep - mezmo - loki - nats From e382afe60cc42bbb6855c28cfab59015832e230f Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Mon, 27 Jan 2025 11:09:18 -0500 Subject: [PATCH 026/658] fix(ci): fix summary paths (#22305) --- .github/workflows/regression.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index f6b69c446d..6b1ced9961 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -551,11 +551,10 @@ jobs: uses: actions/download-artifact@v4 with: name: capture-artifacts - path: downloads - name: Display Markdown Summary run: | - unzip downloads/capture-artifacts -d results + unzip capture-artifacts -d results REPORT_MD=results/report.md if [ -f ${REPORT_MD} ]; then cat ${REPORT_MD} >> $GITHUB_STEP_SUMMARY From cbcbeb3f36fb22efc7ff856bc967b82ebda641ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josu=C3=A9=20AGBEKODO?= <70853334+20agbekodo@users.noreply.github.com> Date: Mon, 27 Jan 2025 17:09:34 +0100 Subject: [PATCH 027/658] feat(datadog service): support filtering on ddtags and ddsource in datadog search syntax (#22281) * feat(datadog service): support filtering on ddtags in datadog search syntax * feat(datadog service):support filtering on ddsource in datadog search syntax * update changelog --- ...earch_syntax_support_ddtags.enhancement.md | 6 + src/conditions/datadog_search.rs | 470 ++++++++++++++++-- 2 files changed, 432 insertions(+), 44 deletions(-) create mode 100644 changelog.d/datadog_search_syntax_support_ddtags.enhancement.md diff --git a/changelog.d/datadog_search_syntax_support_ddtags.enhancement.md b/changelog.d/datadog_search_syntax_support_ddtags.enhancement.md new file mode 100644 index 0000000000..ce522e072b --- /dev/null +++ b/changelog.d/datadog_search_syntax_support_ddtags.enhancement.md @@ -0,0 +1,6 @@ +In datadog search syntax, allow the following queries to match on several fields (OR condition): + +- `tags` will lookup the fields `tags` and `ddtags` +- `source` will lookup the fields `source` and `ddsource` + +authors: 20agbekodo diff --git a/src/conditions/datadog_search.rs b/src/conditions/datadog_search.rs index 87bf97affe..83ec3e0092 100644 --- a/src/conditions/datadog_search.rs +++ b/src/conditions/datadog_search.rs @@ -86,22 +86,20 @@ impl Filter for EventFilter { Field::Tag(tag) => { let starts_with = format!("{}:", tag); - any_string_match("tags", move |value| { + any_string_match_multiple(vec!["ddtags", "tags"], move |value| { value == tag || value.starts_with(&starts_with) }) } // Literal field 'tags' needs to be compared by key. Field::Reserved(field) if field == "tags" => { - any_string_match("tags", move |value| value == field) + any_string_match_multiple(vec!["ddtags", "tags"], move |value| value == field) } - Field::Default(f) | Field::Attribute(f) | Field::Reserved(f) => { - Run::boxed(move |log: &LogEvent| { - log.parse_path_and_get_value(f.as_str()) - .ok() - .flatten() - .is_some() - }) + // A literal "source" field should string match in "source" and "ddsource" fields (OR condition). + Field::Reserved(field) if field == "source" => { + exists_match_multiple(vec!["ddsource", "source"]) } + + Field::Default(f) | Field::Attribute(f) | Field::Reserved(f) => exists_match(f), }) } @@ -121,7 +119,7 @@ impl Filter for EventFilter { Field::Reserved(field) if field == "tags" => { let to_match = to_match.to_owned(); - array_match(field, move |values| { + array_match_multiple(vec!["ddtags", "tags"], move |values| { values.contains(&Value::Bytes(Bytes::copy_from_slice(to_match.as_bytes()))) }) } @@ -129,7 +127,15 @@ impl Filter for EventFilter { Field::Tag(tag) => { let value_bytes = Value::Bytes(format!("{}:{}", tag, to_match).into()); - array_match("tags", move |values| values.contains(&value_bytes)) + array_match_multiple(vec!["ddtags", "tags"], move |values| { + values.contains(&value_bytes) + }) + } + // A literal "source" field should string match in "source" and "ddsource" fields (OR condition). + Field::Reserved(field) if field == "source" => { + let to_match = to_match.to_owned(); + + string_match_multiple(vec!["ddsource", "source"], move |value| value == to_match) } // Reserved values are matched by string equality. Field::Reserved(field) => { @@ -162,8 +168,19 @@ impl Filter for EventFilter { Field::Tag(tag) => { let starts_with = format!("{}:{}", tag, prefix); - any_string_match("tags", move |value| value.starts_with(&starts_with)) + any_string_match_multiple(vec!["ddtags", "tags"], move |value| { + value.starts_with(&starts_with) + }) + } + // A literal "source" field should string match in "source" and "ddsource" fields (OR condition). + Field::Reserved(field) if field == "source" => { + let prefix = prefix.to_owned(); + + string_match_multiple(vec!["ddsource", "source"], move |value| { + value.starts_with(&prefix) + }) } + // All other field types are compared by complete value. Field::Reserved(field) | Field::Attribute(field) => { let prefix = prefix.to_owned(); @@ -187,7 +204,13 @@ impl Filter for EventFilter { Field::Tag(tag) => { let re = wildcard_regex(&format!("{}:{}", tag, wildcard)); - any_string_match("tags", move |value| re.is_match(&value)) + any_string_match_multiple(vec!["ddtags", "tags"], move |value| re.is_match(&value)) + } + // A literal "source" field should string match in "source" and "ddsource" fields (OR condition). + Field::Reserved(field) if field == "source" => { + let re = wildcard_regex(wildcard); + + string_match_multiple(vec!["ddsource", "source"], move |value| re.is_match(&value)) } Field::Reserved(field) | Field::Attribute(field) => { let re = wildcard_regex(wildcard); @@ -277,19 +300,30 @@ impl Filter for EventFilter { }) } // Tag values need extracting by "key:value" to be compared. - Field::Tag(tag) => any_string_match("tags", move |value| match value.split_once(':') { - Some((t, lhs)) if t == tag => { - let lhs = Cow::from(lhs); + Field::Tag(tag) => any_string_match_multiple(vec!["ddtags", "tags"], move |value| { + match value.split_once(':') { + Some((t, lhs)) if t == tag => { + let lhs = Cow::from(lhs); - match comparator { - Comparison::Lt => lhs < rhs, - Comparison::Lte => lhs <= rhs, - Comparison::Gt => lhs > rhs, - Comparison::Gte => lhs >= rhs, + match comparator { + Comparison::Lt => lhs < rhs, + Comparison::Lte => lhs <= rhs, + Comparison::Gt => lhs > rhs, + Comparison::Gte => lhs >= rhs, + } } + _ => false, } - _ => false, }), + // A literal "source" field should string match in "source" and "ddsource" fields (OR condition). + Field::Reserved(field) if field == "source" => { + string_match_multiple(vec!["ddsource", "source"], move |lhs| match comparator { + Comparison::Lt => lhs < rhs, + Comparison::Lte => lhs <= rhs, + Comparison::Gt => lhs > rhs, + Comparison::Gte => lhs >= rhs, + }) + } // All other tag types are compared by string. Field::Default(field) | Field::Reserved(field) => { string_match(field, move |lhs| match comparator { @@ -303,6 +337,21 @@ impl Filter for EventFilter { } } +// Returns a `Matcher` that returns true if the field exists. +fn exists_match(field: S) -> Box> +where + S: Into, +{ + let field = field.into(); + + Run::boxed(move |log: &LogEvent| { + log.parse_path_and_get_value(field.as_str()) + .ok() + .flatten() + .is_some() + }) +} + /// Returns a `Matcher` that returns true if the field resolves to a string, /// numeric, or boolean which matches the provided `func`. fn simple_scalar_match(field: S, func: F) -> Box> @@ -340,46 +389,71 @@ where }) } -/// Returns a `Matcher` that returns true if the log event resolves to an array, where -/// the vector of `Value`s the array contains matches the provided `func`. -fn array_match(field: S, func: F) -> Box> +// Returns a `Matcher` that returns true if any provided field exists. +fn exists_match_multiple(fields: Vec) -> Box> where - S: Into, - F: Fn(&Vec) -> bool + Send + Sync + Clone + 'static, + S: Into + Clone + Send + Sync + 'static, { - let field = field.into(); - Run::boxed(move |log: &LogEvent| { - match log.parse_path_and_get_value(field.as_str()).ok().flatten() { - Some(Value::Array(values)) => func(values), - _ => false, - } + fields + .iter() + .any(|field| exists_match(field.clone()).run(log)) }) } -/// Returns a `Matcher` that returns true if the log event resolves to an array, where -/// at least one `Value` it contains matches the provided `func`. -fn any_match(field: S, func: F) -> Box> +/// Returns a `Matcher` that returns true if any provided field resolves to a string which +/// matches the provided `func`. +fn string_match_multiple(fields: Vec, func: F) -> Box> where - S: Into, - F: Fn(&Value) -> bool + Send + Sync + Clone + 'static, + S: Into + Clone + Send + Sync + 'static, + F: Fn(Cow) -> bool + Send + Sync + Clone + 'static, { - array_match(field, move |values| values.iter().any(&func)) + Run::boxed(move |log: &LogEvent| { + fields + .iter() + .any(|field| string_match(field.clone(), func.clone()).run(log)) + }) } -/// Returns a `Matcher` that returns true if the log event resolves to an array of strings, -/// where at least one string matches the provided `func`. -fn any_string_match(field: S, func: F) -> Box> +fn any_string_match_multiple(fields: Vec, func: F) -> Box> where - S: Into, + S: Into + Clone + Send + Sync + 'static, F: Fn(Cow) -> bool + Send + Sync + Clone + 'static, { - any_match(field, move |value| { + any_match_multiple(fields, move |value| { let bytes = value.coerce_to_bytes(); func(String::from_utf8_lossy(&bytes)) }) } +/// Returns a `Matcher` that returns true if any provided field of the log event resolves to an array, where +/// at least one `Value` it contains matches the provided `func`. +fn any_match_multiple(fields: Vec, func: F) -> Box> +where + S: Into + Clone + Send + Sync + 'static, + F: Fn(&Value) -> bool + Send + Sync + Clone + 'static, +{ + array_match_multiple(fields, move |values| values.iter().any(&func)) +} + +/// Returns a `Matcher` that returns true if any provided field of the log event resolves to an array, where +/// the vector of `Value`s the array contains matches the provided `func`. +fn array_match_multiple(fields: Vec, func: F) -> Box> +where + S: Into + Clone + Send + Sync + 'static, + F: Fn(&Vec) -> bool + Send + Sync + Clone + 'static, +{ + Run::boxed(move |log: &LogEvent| { + fields.iter().any(|field| { + let field = field.clone().into(); + match log.parse_path_and_get_value(field.as_str()).ok().flatten() { + Some(Value::Array(values)) => func(values), + _ => false, + } + }) + }) +} + #[cfg(test)] mod test { use super::*; @@ -1187,6 +1261,314 @@ mod test { log_event!["field" => false, "field2" => "value2"], log_event!["field" => true, "field2" => "value2"], ), + // tags checks with 'ddtags' (DD Agent Source naming) + + // Tag exists. + ( + "_exists_:a", // Source + log_event!["ddtags" => vec!["a:foo"]], // Pass + log_event!["ddtags" => vec!["b:foo"]], // Fail + ), + // Tag exists with - in name. + ( + "_exists_:a-b", // Source + log_event!["ddtags" => vec!["a-b:foo"]], // Pass + log_event!["ddtags" => vec!["ab:foo"]], // Fail + ), + // Tag exists (negate). + ( + "NOT _exists_:a", + log_event!["ddtags" => vec!["b:foo"]], + log_event!("ddtags" => vec!["a:foo"]), + ), + // Tag exists (negate w/-). + ( + "-_exists_:a", + log_event!["ddtags" => vec!["b:foo"]], + log_event!["ddtags" => vec!["a:foo"]], + ), + // Tag doesn't exist. + ( + "_missing_:a", + log_event![], + log_event!["ddtags" => vec!["a:foo"]], + ), + // Tag doesn't exist (negate). + ( + "NOT _missing_:a", + log_event!["ddtags" => vec!["a:foo"]], + log_event![], + ), + // Tag doesn't exist (negate w/-). + ( + "-_missing_:a", + log_event!["ddtags" => vec!["a:foo"]], + log_event![], + ), + // Tag match. + ( + "a:bla", + log_event!["ddtags" => vec!["a:bla"]], + log_event!["ddtags" => vec!["b:bla"]], + ), + // Tag match (negate). + ( + "NOT a:bla", + log_event!["ddtags" => vec!["b:bla"]], + log_event!["ddtags" => vec!["a:bla"]], + ), + // Reserved tag match (negate). + ( + "NOT host:foo", + log_event!["ddtags" => vec!["host:fo o"]], + log_event!["host" => "foo"], + ), + // Tag match (negate w/-). + ( + "-a:bla", + log_event!["ddtags" => vec!["b:bla"]], + log_event!["ddtags" => vec!["a:bla"]], + ), + // Quoted tag match. + ( + r#"a:"bla""#, + log_event!["ddtags" => vec!["a:bla"]], + log_event!["a" => "bla"], + ), + // Quoted tag match (negate). + ( + r#"NOT a:"bla""#, + log_event!["a" => "bla"], + log_event!["ddtags" => vec!["a:bla"]], + ), + // Quoted tag match (negate w/-). + ( + r#"-a:"bla""#, + log_event!["a" => "bla"], + log_event!["ddtags" => vec!["a:bla"]], + ), + // String attribute match. + ( + "@a:bla", + log_event!["a" => "bla"], + log_event!["ddtags" => vec!["a:bla"]], + ), + // String attribute match (negate). + ( + "NOT @a:bla", + log_event!["ddtags" => vec!["a:bla"]], + log_event!["a" => "bla"], + ), + // String attribute match (negate w/-). + ( + "-@a:bla", + log_event!["ddtags" => vec!["a:bla"]], + log_event!["a" => "bla"], + ), + // Quoted attribute match. + ( + r#"@a:"bla""#, + log_event!["a" => "bla"], + log_event!["ddtags" => vec!["a:bla"]], + ), + // Quoted attribute match (negate). + ( + r#"NOT @a:"bla""#, + log_event!["ddtags" => vec!["a:bla"]], + log_event!["a" => "bla"], + ), + // Quoted attribute match (negate w/-). + ( + r#"-@a:"bla""#, + log_event!["ddtags" => vec!["a:bla"]], + log_event!["a" => "bla"], + ), + // Integer attribute match. + ( + "@a:200", + log_event!["a" => 200], + log_event!["ddtags" => vec!["a:200"]], + ), + // Float attribute match. + ( + "@a:0.75", + log_event!["a" => 0.75], + log_event!["ddtags" => vec!["a:0.75"]], + ), + ( + "a:*bla", + log_event!["ddtags" => vec!["a:foobla"]], + log_event!["ddtags" => vec!["a:blafoo"]], + ), + // Wildcard prefix - tag (negate). + ( + "NOT a:*bla", + log_event!["ddtags" => vec!["a:blafoo"]], + log_event!["ddtags" => vec!["a:foobla"]], + ), + // Wildcard prefix - tag (negate w/-). + ( + "-a:*bla", + log_event!["ddtags" => vec!["a:blafoo"]], + log_event!["ddtags" => vec!["a:foobla"]], + ), + // Wildcard suffix - tag. + ( + "b:bla*", + log_event!["ddtags" => vec!["b:blabop"]], + log_event!["ddtags" => vec!["b:bopbla"]], + ), + // Wildcard suffix - tag (negate). + ( + "NOT b:bla*", + log_event!["ddtags" => vec!["b:bopbla"]], + log_event!["ddtags" => vec!["b:blabop"]], + ), + // Wildcard suffix - tag (negate w/-). + ( + "-b:bla*", + log_event!["ddtags" => vec!["b:bopbla"]], + log_event!["ddtags" => vec!["b:blabop"]], + ), + // Multiple wildcards - tag. + ( + "c:*b*la*", + log_event!["ddtags" => vec!["c:foobla"]], + log_event!["custom" => r#"{"title" => "foobla"}"#], + ), + // Multiple wildcards - tag (negate). + ( + "NOT c:*b*la*", + log_event!["custom" => r#"{"title" => "foobla"}"#], + log_event!["ddtags" => vec!["c:foobla"]], + ), + // Multiple wildcards - tag (negate w/-). + ( + "-c:*b*la*", + log_event!["custom" => r#"{"title" => "foobla"}"#], + log_event!["ddtags" => vec!["c:foobla"]], + ), + // Wildcard prefix - attribute. + ( + "@a:*bla", + log_event!["a" => "foobla"], + log_event!["ddtags" => vec!["a:foobla"]], + ), + // Wildcard prefix - attribute (negate). + ( + "NOT @a:*bla", + log_event!["ddtags" => vec!["a:foobla"]], + log_event!["a" => "foobla"], + ), + // Wildcard prefix - attribute (negate w/-). + ( + "-@a:*bla", + log_event!["ddtags" => vec!["a:foobla"]], + log_event!["a" => "foobla"], + ), + // Wildcard suffix - attribute. + ( + "@b:bla*", + log_event!["b" => "blabop"], + log_event!["ddtags" => vec!["b:blabop"]], + ), + // Wildcard suffix - attribute (negate). + ( + "NOT @b:bla*", + log_event!["ddtags" => vec!["b:blabop"]], + log_event!["b" => "blabop"], + ), + // Wildcard suffix - attribute (negate w/-). + ( + "-@b:bla*", + log_event!["ddtags" => vec!["b:blabop"]], + log_event!["b" => "blabop"], + ), + // Multiple wildcards - attribute. + ( + "@c:*b*la*", + log_event!["c" => "foobla"], + log_event!["ddtags" => vec!["c:foobla"]], + ), + // Multiple wildcards - attribute (negate). + ( + "NOT @c:*b*la*", + log_event!["ddtags" => vec!["c:foobla"]], + log_event!["c" => "foobla"], + ), + // Multiple wildcards - attribute (negate w/-). + ( + "-@c:*b*la*", + log_event!["ddtags" => vec!["c:foobla"]], + log_event!["c" => "foobla"], + ), + // Special case for tags. + ( + "tags:a", + log_event!["ddtags" => vec!["a", "b", "c"]], + log_event!["ddtags" => vec!["d", "e", "f"]], + ), + // Special case for tags (negate). + ( + "NOT tags:a", + log_event!["ddtags" => vec!["d", "e", "f"]], + log_event!["ddtags" => vec!["a", "b", "c"]], + ), + // Special case for tags (negate w/-). + ( + "-tags:a", + log_event!["ddtags" => vec!["d", "e", "f"]], + log_event!["ddtags" => vec!["a", "b", "c"]], + ), + // Special case: 'source' looks up on 'source' and 'ddsource' (OR condition) + // source + ( + "source:foo", + log_event!["source" => "foo"], + log_event!["tags" => vec!["source:foo"]], + ), + ( + "source:foo", + log_event!["source" => "foo"], + log_event!["source" => "foobar"], + ), + ( + "source:foo", + log_event!["source" => "foo"], + log_event!["source" => r#"{"value": "foo"}"#], + ), + // ddsource + ( + "source:foo", + log_event!["ddsource" => "foo"], + log_event!["tags" => vec!["ddsource:foo"]], + ), + ( + "source:foo", + log_event!["ddsource" => "foo"], + log_event!["ddsource" => "foobar"], + ), + ( + "source:foo", + log_event!["ddsource" => "foo"], + log_event!["ddsource" => r#"{"value": "foo"}"#], + ), + // both source and ddsource + ( + "source:foo", + log_event!["source" => "foo", "ddsource" => "foo"], + log_event!["source" => "foobar", "ddsource" => "foobar"], + ), + ( + "source:foo", + log_event!["source" => "foo", "ddsource" => "foobar"], + log_event!["source" => "foobar", "ddsource" => "foobar"], + ), + ( + "source:foo", + log_event!["source" => "foobar", "ddsource" => "foo"], + log_event!["source" => "foobar", "ddsource" => "foobar"], + ), ] } From ffd359786b8b1cd6d78d59e29dce3c5c1b276aeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ensar=20Saraj=C4=8Di=C4=87?= Date: Mon, 27 Jan 2025 17:52:47 +0100 Subject: [PATCH 028/658] fix(enriching): fix flush metrics for `memory` enrichment table (#22296) * fix(enriching): fix flush metrics for `memory` enrichment table When `flush_interval` is not set for `memory` enrichment table (when data is flushed on every update) flush metrics are not properly updated. This changes that scenario to use the same flushing method, to ensure that object count, size and flush count metrics are properly updated. * Add tests for metrics --- src/enrichment_tables/memory/table.rs | 256 +++++++++++++++++++++++--- 1 file changed, 234 insertions(+), 22 deletions(-) diff --git a/src/enrichment_tables/memory/table.rs b/src/enrichment_tables/memory/table.rs index 9c52149dad..6c2bd1c611 100644 --- a/src/enrichment_tables/memory/table.rs +++ b/src/enrichment_tables/memory/table.rs @@ -3,7 +3,7 @@ use crate::enrichment_tables::memory::internal_events::{ MemoryEnrichmentTableRead, MemoryEnrichmentTableReadFailed, MemoryEnrichmentTableTtlExpired, }; use crate::enrichment_tables::memory::MemoryConfig; -use std::sync::{Arc, Mutex}; +use std::sync::{Arc, Mutex, MutexGuard}; use std::time::{Duration, Instant}; use evmap::shallow_copy::CopyValue; @@ -103,7 +103,7 @@ impl Memory { .get_or(|| self.read_handle_factory.handle()) } - fn handle_value(&mut self, value: ObjectMap) { + fn handle_value(&self, value: ObjectMap) { let mut writer = self.write_handle.lock().expect("mutex poisoned"); let now = Instant::now(); @@ -148,12 +148,11 @@ impl Memory { } if self.config.flush_interval.is_none() { - writer.write_handle.refresh(); + self.flush(writer); } } - fn scan_and_mark_for_deletion(&mut self) -> bool { - let mut writer = self.write_handle.lock().expect("mutex poisoned"); + fn scan_and_mark_for_deletion(&self, writer: &mut MutexGuard<'_, MemoryWriter>) -> bool { let now = Instant::now(); let mut needs_flush = false; @@ -180,16 +179,14 @@ impl Memory { needs_flush } - fn scan(&mut self) { - let needs_flush = self.scan_and_mark_for_deletion(); + fn scan(&self, mut writer: MutexGuard<'_, MemoryWriter>) { + let needs_flush = self.scan_and_mark_for_deletion(&mut writer); if needs_flush { - self.flush(); + self.flush(writer); } } - fn flush(&mut self) { - let mut writer = self.write_handle.lock().expect("mutex poisoned"); - + fn flush(&self, mut writer: MutexGuard<'_, MemoryWriter>) { writer.write_handle.refresh(); if let Some(reader) = self.get_read_handle().read() { let mut byte_size = 0; @@ -332,11 +329,13 @@ impl StreamSink for Memory { } Some(_) = flush_interval.next() => { - self.flush(); + let writer = self.write_handle.lock().expect("mutex poisoned"); + self.flush(writer); } Some(_) = scan_interval.next() => { - self.scan(); + let writer = self.write_handle.lock().expect("mutex poisoned"); + self.scan(writer); } } } @@ -346,14 +345,19 @@ impl StreamSink for Memory { #[cfg(test)] mod tests { - use futures::future::ready; + use futures::{future::ready, StreamExt}; use futures_util::stream; use std::time::Duration; - use vector_lib::sink::VectorSink; + use vector_lib::{ + event::{EventContainer, MetricValue}, + metrics::Controller, + sink::VectorSink, + }; use super::*; use crate::{ + enrichment_tables::memory::internal_events::InternalMetricsConfig, event::{Event, LogEvent}, test_util::components::{run_and_assert_sink_compliance, SINK_TAGS}, }; @@ -366,7 +370,7 @@ mod tests { #[test] fn finds_row() { - let mut memory = Memory::new(Default::default()); + let memory = Memory::new(Default::default()); memory.handle_value(ObjectMap::from([("test_key".into(), Value::from(5))])); let condition = Condition::Equals { @@ -419,7 +423,7 @@ mod tests { #[test] fn removes_expired_records_on_scan_interval() { let ttl = 100; - let mut memory = Memory::new(build_memory_config(|c| { + let memory = Memory::new(build_memory_config(|c| { c.ttl = ttl; })); { @@ -449,7 +453,8 @@ mod tests { ); // Force scan - memory.scan(); + let writer = memory.write_handle.lock().unwrap(); + memory.scan(writer); // The value is not present anymore assert!(memory @@ -462,7 +467,7 @@ mod tests { #[test] fn does_not_show_values_before_flush_interval() { let ttl = 100; - let mut memory = Memory::new(build_memory_config(|c| { + let memory = Memory::new(build_memory_config(|c| { c.ttl = ttl; c.flush_interval = Some(10); })); @@ -483,7 +488,7 @@ mod tests { #[test] fn updates_ttl_on_value_replacement() { let ttl = 100; - let mut memory = Memory::new(build_memory_config(|c| c.ttl = ttl)); + let memory = Memory::new(build_memory_config(|c| c.ttl = ttl)); { let mut handle = memory.write_handle.lock().unwrap(); handle.write_handle.update( @@ -523,7 +528,7 @@ mod tests { #[test] fn ignores_all_values_over_byte_size_limit() { - let mut memory = Memory::new(build_memory_config(|c| { + let memory = Memory::new(build_memory_config(|c| { c.max_byte_size = Some(1); })); memory.handle_value(ObjectMap::from([("test_key".into(), Value::from(5))])); @@ -543,7 +548,7 @@ mod tests { #[test] fn ignores_values_when_byte_size_limit_is_reached() { let ttl = 100; - let mut memory = Memory::new(build_memory_config(|c| { + let memory = Memory::new(build_memory_config(|c| { c.ttl = ttl; c.max_byte_size = Some(150); })); @@ -614,4 +619,211 @@ mod tests { ) .await; } + + #[tokio::test] + async fn flush_metrics_without_interval() { + let event = Event::Log(LogEvent::from(ObjectMap::from([( + "test_key".into(), + Value::from(5), + )]))); + + let memory = Memory::new(Default::default()); + + run_and_assert_sink_compliance( + VectorSink::from_event_streamsink(memory), + stream::once(ready(event)), + &SINK_TAGS, + ) + .await; + + let metrics = Controller::get().unwrap().capture_metrics(); + let insertions_counter = metrics + .iter() + .find(|m| { + matches!(m.value(), MetricValue::Counter { .. }) + && m.name() == "memory_enrichment_table_insertions_total" + }) + .expect("Insertions metric is missing!"); + let MetricValue::Counter { + value: insertions_count, + } = insertions_counter.value() + else { + unreachable!(); + }; + let flushes_counter = metrics + .iter() + .find(|m| { + matches!(m.value(), MetricValue::Counter { .. }) + && m.name() == "memory_enrichment_table_flushes_total" + }) + .expect("Flushes metric is missing!"); + let MetricValue::Counter { + value: flushes_count, + } = flushes_counter.value() + else { + unreachable!(); + }; + let object_count_gauge = metrics + .iter() + .find(|m| { + matches!(m.value(), MetricValue::Gauge { .. }) + && m.name() == "memory_enrichment_table_objects_count" + }) + .expect("Object count metric is missing!"); + let MetricValue::Gauge { + value: object_count, + } = object_count_gauge.value() + else { + unreachable!(); + }; + let byte_size_gauge = metrics + .iter() + .find(|m| { + matches!(m.value(), MetricValue::Gauge { .. }) + && m.name() == "memory_enrichment_table_byte_size" + }) + .expect("Byte size metric is missing!"); + assert_eq!(*insertions_count, 1.0); + assert_eq!(*flushes_count, 1.0); + assert_eq!(*object_count, 1.0); + assert!(!byte_size_gauge.is_empty()); + } + + #[tokio::test] + async fn flush_metrics_with_interval() { + let event = Event::Log(LogEvent::from(ObjectMap::from([( + "test_key".into(), + Value::from(5), + )]))); + + let memory = Memory::new(build_memory_config(|c| { + c.flush_interval = Some(1); + })); + + run_and_assert_sink_compliance( + VectorSink::from_event_streamsink(memory), + stream::iter(vec![event.clone(), event]).flat_map(|e| { + stream::once(async move { + tokio::time::sleep(Duration::from_millis(600)).await; + e + }) + }), + &SINK_TAGS, + ) + .await; + + let metrics = Controller::get().unwrap().capture_metrics(); + let insertions_counter = metrics + .iter() + .find(|m| { + matches!(m.value(), MetricValue::Counter { .. }) + && m.name() == "memory_enrichment_table_insertions_total" + }) + .expect("Insertions metric is missing!"); + let MetricValue::Counter { + value: insertions_count, + } = insertions_counter.value() + else { + unreachable!(); + }; + let flushes_counter = metrics + .iter() + .find(|m| { + matches!(m.value(), MetricValue::Counter { .. }) + && m.name() == "memory_enrichment_table_flushes_total" + }) + .expect("Flushes metric is missing!"); + let MetricValue::Counter { + value: flushes_count, + } = flushes_counter.value() + else { + unreachable!(); + }; + let object_count_gauge = metrics + .iter() + .find(|m| { + matches!(m.value(), MetricValue::Gauge { .. }) + && m.name() == "memory_enrichment_table_objects_count" + }) + .expect("Object count metric is missing!"); + let MetricValue::Gauge { + value: object_count, + } = object_count_gauge.value() + else { + unreachable!(); + }; + let byte_size_gauge = metrics + .iter() + .find(|m| { + matches!(m.value(), MetricValue::Gauge { .. }) + && m.name() == "memory_enrichment_table_byte_size" + }) + .expect("Byte size metric is missing!"); + + assert_eq!(*insertions_count, 2.0); + // One is done right away and the next one after the interval + assert_eq!(*flushes_count, 2.0); + assert_eq!(*object_count, 1.0); + assert!(!byte_size_gauge.is_empty()); + } + + #[tokio::test] + async fn flush_metrics_with_key() { + let event = Event::Log(LogEvent::from(ObjectMap::from([( + "test_key".into(), + Value::from(5), + )]))); + + let memory = Memory::new(build_memory_config(|c| { + c.internal_metrics = InternalMetricsConfig { + include_key_tag: true, + }; + })); + + run_and_assert_sink_compliance( + VectorSink::from_event_streamsink(memory), + stream::once(ready(event)), + &SINK_TAGS, + ) + .await; + + let metrics = Controller::get().unwrap().capture_metrics(); + let insertions_counter = metrics + .iter() + .find(|m| { + matches!(m.value(), MetricValue::Counter { .. }) + && m.name() == "memory_enrichment_table_insertions_total" + }) + .expect("Insertions metric is missing!"); + + assert!(insertions_counter.tag_matches("key", "test_key")); + } + + #[tokio::test] + async fn flush_metrics_without_key() { + let event = Event::Log(LogEvent::from(ObjectMap::from([( + "test_key".into(), + Value::from(5), + )]))); + + let memory = Memory::new(Default::default()); + + run_and_assert_sink_compliance( + VectorSink::from_event_streamsink(memory), + stream::once(ready(event)), + &SINK_TAGS, + ) + .await; + + let metrics = Controller::get().unwrap().capture_metrics(); + let insertions_counter = metrics + .iter() + .find(|m| { + matches!(m.value(), MetricValue::Counter { .. }) + && m.name() == "memory_enrichment_table_insertions_total" + }) + .expect("Insertions metric is missing!"); + + assert!(insertions_counter.tag_value("key").is_none()); + } } From 6fa2099880417573436e72f8626855266d79d9e3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Jan 2025 18:16:51 +0000 Subject: [PATCH 029/658] chore(ci): Bump docker/build-push-action from 6.12.0 to 6.13.0 (#22306) Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 6.12.0 to 6.13.0. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/v6.12.0...v6.13.0) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/environment.yml | 2 +- .github/workflows/regression.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/environment.yml b/.github/workflows/environment.yml index 31691d0f2f..e3f8ac5e0d 100644 --- a/.github/workflows/environment.yml +++ b/.github/workflows/environment.yml @@ -62,7 +62,7 @@ jobs: org.opencontainers.image.title=Vector development environment org.opencontainers.image.url=https://github.com/vectordotdev/vector - name: Build and push - uses: docker/build-push-action@v6.12.0 + uses: docker/build-push-action@v6.13.0 with: context: . file: ./scripts/environment/Dockerfile diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 6b1ced9961..251e9cda0b 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -204,7 +204,7 @@ jobs: uses: docker/setup-buildx-action@v3.8.0 - name: Build 'vector' target image - uses: docker/build-push-action@v6.12.0 + uses: docker/build-push-action@v6.13.0 with: context: baseline-vector/ cache-from: type=gha @@ -243,7 +243,7 @@ jobs: uses: docker/setup-buildx-action@v3.8.0 - name: Build 'vector' target image - uses: docker/build-push-action@v6.12.0 + uses: docker/build-push-action@v6.13.0 with: context: comparison-vector/ cache-from: type=gha From 559cb4667f43cca891189b7179e0f37c778d5c49 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Jan 2025 14:10:57 -0500 Subject: [PATCH 030/658] chore(deps): Bump vrl from `c0245e1` to `2ccb98e` (#22303) Bumps [vrl](https://github.com/vectordotdev/vrl) from `c0245e1` to `2ccb98e`. - [Commits](https://github.com/vectordotdev/vrl/compare/c0245e16a4ef23c3306f7ddabde0d8869e754423...2ccb98e471e650b908409eb11a0cae5a0d0bdf21) --- updated-dependencies: - dependency-name: vrl dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2ca453ed5b..3b7c30abaa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4583,7 +4583,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.5.8", + "socket2 0.4.10", "tokio", "tower-service", "tracing 0.1.41", @@ -7619,7 +7619,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e9552f850d5f0964a4e4d0bf306459ac29323ddfbae05e35a7c0d35cb0803cc5" dependencies = [ "anyhow", - "itertools 0.13.0", + "itertools 0.11.0", "proc-macro2 1.0.93", "quote 1.0.38", "syn 2.0.96", @@ -11528,7 +11528,7 @@ checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" [[package]] name = "vrl" version = "0.21.0" -source = "git+https://github.com/vectordotdev/vrl?branch=main#c0245e16a4ef23c3306f7ddabde0d8869e754423" +source = "git+https://github.com/vectordotdev/vrl?branch=main#2ccb98e471e650b908409eb11a0cae5a0d0bdf21" dependencies = [ "aes", "aes-siv", From f151cab1fa156a53b6b44db245deedad8eef6ab8 Mon Sep 17 00:00:00 2001 From: Johannes Geiger Date: Mon, 27 Jan 2025 19:25:29 +0000 Subject: [PATCH 031/658] feat (sink: aws_cloudwatch_logs) #11185: Allow specifying a KMS key and tags for newly created AWS CloudWatch log groups. (#22274) * #11185: Allow specifying a KMS key and tags for newly created AWS CloudWatch log groups. * #11185: Checklist. * #11185: Rustdocs. * #11185: Unrelated changes to the Story it seems, but still present. * #11185: Revert unrelated changes. * #11185: Rustdocs. --- Cargo.lock | 23 +++ Cargo.toml | 3 +- LICENSE-3rdparty.csv | 1 + changelog.d/11185.feature.md | 3 + scripts/integration/aws/compose.yaml | 2 +- scripts/integration/aws/test.yaml | 1 + src/sinks/aws_cloudwatch_logs/config.rs | 21 +++ .../aws_cloudwatch_logs/integration_tests.rs | 135 +++++++++++++++++- src/sinks/aws_cloudwatch_logs/request.rs | 22 ++- src/sinks/aws_cloudwatch_logs/service.rs | 9 ++ .../sinks/base/aws_cloudwatch_logs.cue | 23 +++ 11 files changed, 234 insertions(+), 9 deletions(-) create mode 100644 changelog.d/11185.feature.md diff --git a/Cargo.lock b/Cargo.lock index 3b7c30abaa..d91a2e6c0c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -969,6 +969,28 @@ dependencies = [ "tracing 0.1.41", ] +[[package]] +name = "aws-sdk-kms" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "374513bec90ddc64288c1f1e5cb4b95e6df6e42a47d9e332a713f97f66cf9043" +dependencies = [ + "aws-credential-types", + "aws-http", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes 1.9.0", + "http 0.2.9", + "regex", + "tracing 0.1.41", +] + [[package]] name = "aws-sdk-s3" version = "1.4.0" @@ -10992,6 +11014,7 @@ dependencies = [ "aws-sdk-elasticsearch", "aws-sdk-firehose", "aws-sdk-kinesis", + "aws-sdk-kms", "aws-sdk-s3", "aws-sdk-secretsmanager", "aws-sdk-sns", diff --git a/Cargo.toml b/Cargo.toml index 27c4f6eb9e..87e354bb34 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -220,6 +220,7 @@ aws-sdk-sqs = { version = "1.3.0", default-features = false, features = ["behavi aws-sdk-sns = { version = "1.6.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } aws-sdk-cloudwatch = { version = "1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } aws-sdk-cloudwatchlogs = { version = "1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } +aws-sdk-kms = { version = "1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } aws-sdk-elasticsearch = { version = "1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } aws-sdk-firehose = { version = "1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } aws-sdk-kinesis = { version = "1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } @@ -763,7 +764,7 @@ sinks-metrics = [ sinks-amqp = ["lapin"] sinks-appsignal = [] -sinks-aws_cloudwatch_logs = ["aws-core", "dep:aws-sdk-cloudwatchlogs"] +sinks-aws_cloudwatch_logs = ["aws-core", "dep:aws-sdk-cloudwatchlogs", "dep:aws-sdk-kms"] sinks-aws_cloudwatch_metrics = ["aws-core", "dep:aws-sdk-cloudwatch"] sinks-aws_kinesis_firehose = ["aws-core", "dep:aws-sdk-firehose"] sinks-aws_kinesis_streams = ["aws-core", "dep:aws-sdk-kinesis"] diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 08fa4e1b45..7c4bd3eb70 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -55,6 +55,7 @@ aws-sdk-cloudwatch,https://github.com/awslabs/aws-sdk-rust,Apache-2.0,"AWS Rust aws-sdk-cloudwatchlogs,https://github.com/awslabs/aws-sdk-rust,Apache-2.0,"AWS Rust SDK Team , Russell Cohen " aws-sdk-firehose,https://github.com/awslabs/aws-sdk-rust,Apache-2.0,"AWS Rust SDK Team , Russell Cohen " aws-sdk-kinesis,https://github.com/awslabs/aws-sdk-rust,Apache-2.0,"AWS Rust SDK Team , Russell Cohen " +aws-sdk-kms,https://github.com/awslabs/aws-sdk-rust,Apache-2.0,"AWS Rust SDK Team , Russell Cohen " aws-sdk-s3,https://github.com/awslabs/aws-sdk-rust,Apache-2.0,"AWS Rust SDK Team , Russell Cohen " aws-sdk-secretsmanager,https://github.com/awslabs/aws-sdk-rust,Apache-2.0,"AWS Rust SDK Team , Russell Cohen " aws-sdk-sns,https://github.com/awslabs/aws-sdk-rust,Apache-2.0,"AWS Rust SDK Team , Russell Cohen " diff --git a/changelog.d/11185.feature.md b/changelog.d/11185.feature.md new file mode 100644 index 0000000000..79c4d89e5e --- /dev/null +++ b/changelog.d/11185.feature.md @@ -0,0 +1,3 @@ +Allows users to specify a KMS key and tags for newly created AWS CloudWatch log groups. + +authors: johannesfloriangeiger diff --git a/scripts/integration/aws/compose.yaml b/scripts/integration/aws/compose.yaml index c45cbe7f2e..c7337dd4bc 100644 --- a/scripts/integration/aws/compose.yaml +++ b/scripts/integration/aws/compose.yaml @@ -6,7 +6,7 @@ services: mock-localstack: image: docker.io/localstack/localstack:3 environment: - - SERVICES=kinesis,s3,cloudwatch,es,firehose,sqs,sns,logs + - SERVICES=kinesis,s3,cloudwatch,es,firehose,kms,sqs,sns,logs mock-ecs: image: docker.io/amazon/amazon-ecs-local-container-endpoints:latest volumes: diff --git a/scripts/integration/aws/test.yaml b/scripts/integration/aws/test.yaml index 71ff65cae1..ba1d901dfc 100644 --- a/scripts/integration/aws/test.yaml +++ b/scripts/integration/aws/test.yaml @@ -11,6 +11,7 @@ env: ECS_ADDRESS: http://mock-ecs ELASTICSEARCH_ADDRESS: http://mock-localstack:4566 KINESIS_ADDRESS: http://mock-localstack:4566 + KMS_ADDRESS: http://mock-localstack:4566 S3_ADDRESS: http://mock-localstack:4566 SQS_ADDRESS: http://mock-localstack:4566 SNS_ADDRESS: http://mock-localstack:4566 diff --git a/src/sinks/aws_cloudwatch_logs/config.rs b/src/sinks/aws_cloudwatch_logs/config.rs index d4a1dd4f39..db7e1d5d4d 100644 --- a/src/sinks/aws_cloudwatch_logs/config.rs +++ b/src/sinks/aws_cloudwatch_logs/config.rs @@ -1,6 +1,7 @@ use aws_sdk_cloudwatchlogs::Client as CloudwatchLogsClient; use futures::FutureExt; use serde::{de, Deserialize, Deserializer}; +use std::collections::HashMap; use tower::ServiceBuilder; use vector_lib::codecs::JsonSerializerConfig; use vector_lib::configurable::configurable_component; @@ -164,6 +165,24 @@ pub struct CloudwatchLogsSinkConfig { skip_serializing_if = "crate::serde::is_default" )] pub acknowledgements: AcknowledgementsConfig, + + /// The [ARN][arn] (Amazon Resource Name) of the [KMS key][kms_key] to use when encrypting log data. + /// + /// [arn]: https://docs.aws.amazon.com/IAM/latest/UserGuide/reference-arns.html + /// [kms_key]: https://docs.aws.amazon.com/kms/latest/developerguide/overview.html + #[configurable(derived)] + #[serde(default)] + pub kms_key: Option, + + /// The Key-value pairs to be applied as [tags][tags] to the log group and stream. + /// + /// [tags]: https://docs.aws.amazon.com/whitepapers/latest/tagging-best-practices/what-are-tags.html + #[configurable(derived)] + #[serde(default)] + #[configurable(metadata( + docs::additional_props_description = "A tag represented as a key-value pair" + ))] + pub tags: Option>, } impl CloudwatchLogsSinkConfig { @@ -248,6 +267,8 @@ fn default_config(encoding: EncodingConfig) -> CloudwatchLogsSinkConfig { assume_role: Default::default(), auth: Default::default(), acknowledgements: Default::default(), + kms_key: Default::default(), + tags: Default::default(), } } diff --git a/src/sinks/aws_cloudwatch_logs/integration_tests.rs b/src/sinks/aws_cloudwatch_logs/integration_tests.rs index a2c2ff4c8c..437408e51f 100644 --- a/src/sinks/aws_cloudwatch_logs/integration_tests.rs +++ b/src/sinks/aws_cloudwatch_logs/integration_tests.rs @@ -1,7 +1,9 @@ +use std::collections::HashMap; use std::convert::TryFrom; use aws_config::Region; use aws_sdk_cloudwatchlogs::Client as CloudwatchLogsClient; +use aws_sdk_kms::Client as KMSClient; use chrono::Duration; use futures::{stream, StreamExt}; use similar_asserts::assert_eq; @@ -9,7 +11,7 @@ use vector_lib::codecs::TextSerializerConfig; use vector_lib::lookup; use super::*; -use crate::aws::create_client; +use crate::aws::{create_client, ClientBuilder}; use crate::aws::{AwsAuthentication, RegionOrEndpoint}; use crate::sinks::aws_cloudwatch_logs::config::CloudwatchLogsClientBuilder; use crate::{ @@ -29,6 +31,20 @@ fn cloudwatch_address() -> String { std::env::var("CLOUDWATCH_ADDRESS").unwrap_or_else(|_| "http://localhost:4566".into()) } +fn kms_address() -> String { + std::env::var("KMS_ADDRESS").unwrap_or_else(|_| "http://localhost:4566".into()) +} + +struct KMSClientBuilder; + +impl ClientBuilder for KMSClientBuilder { + type Client = aws_sdk_kms::client::Client; + + fn build(&self, config: &aws_types::SdkConfig) -> Self::Client { + aws_sdk_kms::client::Client::new(config) + } +} + #[tokio::test] async fn cloudwatch_insert_log_event() { trace_init(); @@ -51,6 +67,8 @@ async fn cloudwatch_insert_log_event() { assume_role: None, auth: Default::default(), acknowledgements: Default::default(), + kms_key: None, + tags: None, }; let (sink, _) = config.build(SinkContext::default()).await.unwrap(); @@ -102,6 +120,8 @@ async fn cloudwatch_insert_log_events_sorted() { assume_role: None, auth: Default::default(), acknowledgements: Default::default(), + kms_key: None, + tags: None, }; let (sink, _) = config.build(SinkContext::default()).await.unwrap(); @@ -178,6 +198,8 @@ async fn cloudwatch_insert_out_of_range_timestamp() { assume_role: None, auth: Default::default(), acknowledgements: Default::default(), + kms_key: None, + tags: None, }; let (sink, _) = config.build(SinkContext::default()).await.unwrap(); @@ -255,6 +277,8 @@ async fn cloudwatch_dynamic_group_and_stream_creation() { assume_role: None, auth: Default::default(), acknowledgements: Default::default(), + kms_key: None, + tags: None, }; let (sink, _) = config.build(SinkContext::default()).await.unwrap(); @@ -284,6 +308,90 @@ async fn cloudwatch_dynamic_group_and_stream_creation() { assert_eq!(output_lines.sort(), input_lines.sort()); } +#[tokio::test] +async fn cloudwatch_dynamic_group_and_stream_creation_with_kms_key_and_tags() { + trace_init(); + + let stream_name = gen_name(); + let group_name = gen_name(); + + let config = CloudwatchLogsSinkConfig { + stream_name: Template::try_from(stream_name.as_str()).unwrap(), + group_name: Template::try_from(group_name.as_str()).unwrap(), + region: RegionOrEndpoint::with_both("us-east-1", cloudwatch_address().as_str()), + encoding: TextSerializerConfig::default().into(), + create_missing_group: true, + create_missing_stream: true, + retention: Default::default(), + compression: Default::default(), + batch: Default::default(), + request: Default::default(), + tls: Default::default(), + assume_role: None, + auth: Default::default(), + acknowledgements: Default::default(), + kms_key: Some( + create_kms_client_test() + .await + .create_key() + .send() + .await + .unwrap() + .key_metadata() + .unwrap() + .key_id() + .parse() + .unwrap(), + ), + tags: Some(HashMap::from_iter(vec![( + "key".to_string(), + "value".to_string(), + )])), + }; + + let (sink, _) = config.build(SinkContext::default()).await.unwrap(); + + let timestamp = chrono::Utc::now(); + + let (mut input_lines, events) = random_lines_with_stream(100, 11, None); + run_and_assert_sink_compliance(sink, events, &AWS_SINK_TAGS).await; + + let response = create_client_test() + .await + .get_log_events() + .log_stream_name(stream_name) + .log_group_name(group_name.clone()) + .start_time(timestamp.timestamp_millis()) + .send() + .await + .unwrap(); + + let events = response.events.unwrap(); + + let mut output_lines = events + .into_iter() + .map(|e| e.message.unwrap()) + .collect::>(); + + assert_eq!(output_lines.sort(), input_lines.sort()); + + let log_group = create_client_test() + .await + .describe_log_groups() + .log_group_name_pattern(group_name.clone()) + .limit(1) + .send() + .await + .unwrap() + .log_groups() + .first() + .unwrap() + .clone(); + + let kms_key = log_group.kms_key_id().unwrap(); + assert_eq!(kms_key, config.kms_key.unwrap()); +} + #[tokio::test] async fn cloudwatch_insert_log_event_batched() { trace_init(); @@ -311,6 +419,8 @@ async fn cloudwatch_insert_log_event_batched() { assume_role: None, auth: Default::default(), acknowledgements: Default::default(), + kms_key: None, + tags: None, }; let (sink, _) = config.build(SinkContext::default()).await.unwrap(); @@ -362,6 +472,8 @@ async fn cloudwatch_insert_log_event_partitioned() { assume_role: None, auth: Default::default(), acknowledgements: Default::default(), + kms_key: None, + tags: None, }; let (sink, _) = config.build(SinkContext::default()).await.unwrap(); @@ -455,6 +567,8 @@ async fn cloudwatch_healthcheck() { assume_role: None, auth: Default::default(), acknowledgements: Default::default(), + kms_key: None, + tags: None, }; let client = config.create_client(&ProxyConfig::default()).await.unwrap(); @@ -480,6 +594,25 @@ async fn create_client_test() -> CloudwatchLogsClient { .unwrap() } +async fn create_kms_client_test() -> KMSClient { + let auth = AwsAuthentication::test_auth(); + let region = Some(Region::new("us-east-1")); + let endpoint = Some(kms_address()); + let proxy = ProxyConfig::default(); + + create_client::( + &KMSClientBuilder {}, + &auth, + region, + endpoint, + &proxy, + None, + None, + ) + .await + .unwrap() +} + async fn ensure_group() { let client = create_client_test().await; _ = client diff --git a/src/sinks/aws_cloudwatch_logs/request.rs b/src/sinks/aws_cloudwatch_logs/request.rs index 60d9e88a6b..e052c05d38 100644 --- a/src/sinks/aws_cloudwatch_logs/request.rs +++ b/src/sinks/aws_cloudwatch_logs/request.rs @@ -1,9 +1,3 @@ -use std::{ - future::Future, - pin::Pin, - task::{ready, Context, Poll}, -}; - use aws_sdk_cloudwatchlogs::{ operation::{ create_log_group::CreateLogGroupError, @@ -19,6 +13,12 @@ use aws_smithy_runtime_api::client::{orchestrator::HttpResponse, result::SdkErro use futures::{future::BoxFuture, FutureExt}; use http::{header::HeaderName, HeaderValue}; use indexmap::IndexMap; +use std::collections::HashMap; +use std::{ + future::Future, + pin::Pin, + task::{ready, Context, Poll}, +}; use tokio::sync::oneshot; use crate::sinks::aws_cloudwatch_logs::config::Retention; @@ -40,6 +40,8 @@ struct Client { group_name: String, headers: IndexMap, retention_days: u32, + kms_key: Option, + tags: Option>, } type ClientResult = BoxFuture<'static, Result>>; @@ -63,6 +65,8 @@ impl CloudwatchFuture { create_missing_group: bool, create_missing_stream: bool, retention: Retention, + kms_key: Option, + tags: Option>, mut events: Vec>, token: Option, token_tx: oneshot::Sender>, @@ -74,6 +78,8 @@ impl CloudwatchFuture { group_name, headers, retention_days, + kms_key, + tags, }; let state = if let Some(token) = token { @@ -288,10 +294,14 @@ impl Client { pub fn create_log_group(&self) -> ClientResult<(), CreateLogGroupError> { let client = self.client.clone(); let group_name = self.group_name.clone(); + let kms_key = self.kms_key.clone(); + let tags = self.tags.clone(); Box::pin(async move { client .create_log_group() .log_group_name(group_name) + .set_kms_key_id(kms_key) + .set_tags(tags) .send() .await?; Ok(()) diff --git a/src/sinks/aws_cloudwatch_logs/service.rs b/src/sinks/aws_cloudwatch_logs/service.rs index ab8ea09daf..f914029dbc 100644 --- a/src/sinks/aws_cloudwatch_logs/service.rs +++ b/src/sinks/aws_cloudwatch_logs/service.rs @@ -240,6 +240,9 @@ impl CloudwatchLogsSvc { let retention = config.retention.clone(); + let kms_key = config.kms_key.clone(); + let tags = config.tags.clone(); + CloudwatchLogsSvc { headers, client, @@ -248,6 +251,8 @@ impl CloudwatchLogsSvc { create_missing_group, create_missing_stream, retention, + kms_key, + tags, token: None, token_rx: None, } @@ -322,6 +327,8 @@ impl Service> for CloudwatchLogsSvc { self.create_missing_group, self.create_missing_stream, self.retention.clone(), + self.kms_key.clone(), + self.tags.clone(), event_batches, self.token.take(), tx, @@ -340,6 +347,8 @@ pub struct CloudwatchLogsSvc { create_missing_group: bool, create_missing_stream: bool, retention: Retention, + kms_key: Option, + tags: Option>, token: Option, token_rx: Option>>, } diff --git a/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue b/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue index be85c3848e..80a44689b2 100644 --- a/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue +++ b/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue @@ -592,6 +592,16 @@ base: components: sinks: aws_cloudwatch_logs: configuration: { syntax: "template" } } + kms_key: { + description: """ + The [ARN][arn] (Amazon Resource Name) of the [KMS key][kms_key] to use when encrypting log data. + + [arn]: https://docs.aws.amazon.com/IAM/latest/UserGuide/reference-arns.html + [kms_key]: https://docs.aws.amazon.com/kms/latest/developerguide/overview.html + """ + required: false + type: string: {} + } region: { description: """ The [AWS region][aws_region] of the target service. @@ -828,6 +838,19 @@ base: components: sinks: aws_cloudwatch_logs: configuration: { syntax: "template" } } + tags: { + description: """ + The Key-value pairs to be applied as [tags][tags] to the log group and stream. + + [tags]: https://docs.aws.amazon.com/whitepapers/latest/tagging-best-practices/what-are-tags.html + """ + required: false + type: object: options: "*": { + description: "A tag represented as a key-value pair" + required: true + type: string: {} + } + } tls: { description: "TLS configuration." required: false From 47e348cc8cab3bcbd8d6cf6937f9d0e2e329e080 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ensar=20Saraj=C4=8Di=C4=87?= Date: Tue, 28 Jan 2025 17:03:20 +0100 Subject: [PATCH 032/658] feat(tag_cardinality_limit transform): enable per metric limits for `tag_cardinality_limit` (#22077) * feat(tag_cardinality_limit transform): enable per metric limits for `tag_cardinality_limit` This adds the ability to add per metric limits in `tag_cardinality_limit` transform, besides the global configuration that applies to all metrics. It supports matching metrics by name and optionally by namespace too. Closes: #15743 * Add changelog entry * Fix typo in changelog entry * Generate docs using `make generate-component-docs` * Use HashMap instead of Vec for `per_metric_limits` * Remove debugging logs from `tag_cardinality_limit` * Change `&Option` into `Option<&..>` in `tag_cardinality_limit` --- ...tag_cardinality_limit_transform.feature.md | 3 + .../tag_cardinality_limit/config.rs | 37 ++- src/transforms/tag_cardinality_limit/mod.rs | 88 ++++-- src/transforms/tag_cardinality_limit/tests.rs | 251 +++++++++++++++++- .../transforms/base/tag_cardinality_limit.cue | 65 +++++ 5 files changed, 410 insertions(+), 34 deletions(-) create mode 100644 changelog.d/22077_per_metric_limits_for_tag_cardinality_limit_transform.feature.md diff --git a/changelog.d/22077_per_metric_limits_for_tag_cardinality_limit_transform.feature.md b/changelog.d/22077_per_metric_limits_for_tag_cardinality_limit_transform.feature.md new file mode 100644 index 0000000000..57a96a6f2f --- /dev/null +++ b/changelog.d/22077_per_metric_limits_for_tag_cardinality_limit_transform.feature.md @@ -0,0 +1,3 @@ +The `tag_cardinality_limit` transform now supports customizing limits for specific metrics, matched by metric name and optionally its namespace. + +authors: esensar diff --git a/src/transforms/tag_cardinality_limit/config.rs b/src/transforms/tag_cardinality_limit/config.rs index 6e52396656..3a348e8e28 100644 --- a/src/transforms/tag_cardinality_limit/config.rs +++ b/src/transforms/tag_cardinality_limit/config.rs @@ -16,6 +16,22 @@ use vector_lib::configurable::configurable_component; ))] #[derive(Clone, Debug)] pub struct TagCardinalityLimitConfig { + #[serde(flatten)] + pub global: TagCardinalityLimitInnerConfig, + + /// Tag cardinality limits configuration per metric name. + #[configurable( + derived, + metadata(docs::additional_props_description = "An individual metric configuration.") + )] + #[serde(default)] + pub per_metric_limits: HashMap, +} + +/// Configuration for the `tag_cardinality_limit` transform for a specific group of metrics. +#[configurable_component] +#[derive(Clone, Debug)] +pub struct TagCardinalityLimitInnerConfig { /// How many distinct values to accept for any given key. #[serde(default = "default_value_limit")] pub value_limit: usize, @@ -77,6 +93,18 @@ pub enum LimitExceededAction { DropEvent, } +/// Tag cardinality limit configuration per metric name. +#[configurable_component] +#[derive(Clone, Debug)] +pub struct PerMetricConfig { + /// Namespace of the metric this configuration refers to. + #[serde(default)] + pub namespace: Option, + + #[serde(flatten)] + pub config: TagCardinalityLimitInnerConfig, +} + const fn default_limit_exceeded_action() -> LimitExceededAction { LimitExceededAction::DropTag } @@ -92,9 +120,12 @@ pub(crate) const fn default_cache_size() -> usize { impl GenerateConfig for TagCardinalityLimitConfig { fn generate_config() -> toml::Value { toml::Value::try_from(Self { - mode: Mode::Exact, - value_limit: default_value_limit(), - limit_exceeded_action: default_limit_exceeded_action(), + global: TagCardinalityLimitInnerConfig { + mode: Mode::Exact, + value_limit: default_value_limit(), + limit_exceeded_action: default_limit_exceeded_action(), + }, + per_metric_limits: HashMap::default(), }) .unwrap() } diff --git a/src/transforms/tag_cardinality_limit/mod.rs b/src/transforms/tag_cardinality_limit/mod.rs index 14b18b457a..bf543ae86e 100644 --- a/src/transforms/tag_cardinality_limit/mod.rs +++ b/src/transforms/tag_cardinality_limit/mod.rs @@ -19,13 +19,15 @@ mod tag_value_set; mod tests; use crate::event::metric::TagValueSet; -pub use config::TagCardinalityLimitConfig; +pub use config::{TagCardinalityLimitConfig, TagCardinalityLimitInnerConfig}; use tag_value_set::AcceptedTagValueSet; +type MetricId = (Option, String); + #[derive(Debug)] pub struct TagCardinalityLimit { config: TagCardinalityLimitConfig, - accepted_tags: HashMap, + accepted_tags: HashMap, HashMap>, } impl TagCardinalityLimit { @@ -36,6 +38,24 @@ impl TagCardinalityLimit { } } + fn get_config_for_metric( + &self, + metric_key: Option<&MetricId>, + ) -> &TagCardinalityLimitInnerConfig { + match metric_key { + Some(id) => self + .config + .per_metric_limits + .iter() + .find(|(name, config)| { + **name == id.1 && (config.namespace.is_none() || config.namespace == id.0) + }) + .map(|(_, c)| &c.config) + .unwrap_or(&self.config.global), + None => &self.config.global, + } + } + /// Takes in key and a value corresponding to a tag on an incoming Metric /// Event. If that value is already part of set of accepted values for that /// key, then simply returns true. If that value is not yet part of the @@ -44,10 +64,17 @@ impl TagCardinalityLimit { /// for the key and returns true, otherwise returns false. A false return /// value indicates to the caller that the value is not accepted for this /// key, and the configured limit_exceeded_action should be taken. - fn try_accept_tag(&mut self, key: &str, value: &TagValueSet) -> bool { - let tag_value_set = self.accepted_tags.entry_ref(key).or_insert_with(|| { - AcceptedTagValueSet::new(self.config.value_limit, &self.config.mode) - }); + fn try_accept_tag( + &mut self, + metric_key: Option<&MetricId>, + key: &str, + value: &TagValueSet, + ) -> bool { + let config = self.get_config_for_metric(metric_key).clone(); + let metric_accepted_tags = self.accepted_tags.entry(metric_key.cloned()).or_default(); + let tag_value_set = metric_accepted_tags + .entry_ref(key) + .or_insert_with(|| AcceptedTagValueSet::new(config.value_limit, &config.mode)); if tag_value_set.contains(value) { // Tag value has already been accepted, nothing more to do. @@ -55,11 +82,11 @@ impl TagCardinalityLimit { } // Tag value not yet part of the accepted set. - if tag_value_set.len() < self.config.value_limit { + if tag_value_set.len() < config.value_limit { // accept the new value tag_value_set.insert(value.clone()); - if tag_value_set.len() == self.config.value_limit { + if tag_value_set.len() == config.value_limit { emit!(TagCardinalityValueLimitReached { key }); } @@ -72,34 +99,57 @@ impl TagCardinalityLimit { /// Checks if recording a key and value corresponding to a tag on an incoming Metric would /// exceed the cardinality limit. - fn tag_limit_exceeded(&self, key: &str, value: &TagValueSet) -> bool { + fn tag_limit_exceeded( + &self, + metric_key: Option<&MetricId>, + key: &str, + value: &TagValueSet, + ) -> bool { self.accepted_tags - .get(key) - .map(|value_set| { - !value_set.contains(value) && value_set.len() >= self.config.value_limit + .get(&metric_key.cloned()) + .and_then(|metric_accepted_tags| { + metric_accepted_tags.get(key).map(|value_set| { + !value_set.contains(value) + && value_set.len() >= self.get_config_for_metric(metric_key).value_limit + }) }) .unwrap_or(false) } /// Record a key and value corresponding to a tag on an incoming Metric. - fn record_tag_value(&mut self, key: &str, value: &TagValueSet) { - self.accepted_tags + fn record_tag_value(&mut self, metric_key: Option<&MetricId>, key: &str, value: &TagValueSet) { + let config = self.get_config_for_metric(metric_key).clone(); + let metric_accepted_tags = self.accepted_tags.entry(metric_key.cloned()).or_default(); + metric_accepted_tags .entry_ref(key) - .or_insert_with(|| AcceptedTagValueSet::new(self.config.value_limit, &self.config.mode)) + .or_insert_with(|| AcceptedTagValueSet::new(config.value_limit, &config.mode)) .insert(value.clone()); } fn transform_one(&mut self, mut event: Event) -> Option { let metric = event.as_mut_metric(); let metric_name = metric.name().to_string(); + let metric_namespace = metric.namespace().map(|n| n.to_string()); + let has_per_metric_config = self.config.per_metric_limits.iter().any(|(name, config)| { + *name == metric_name + && (config.namespace.is_none() || config.namespace == metric_namespace) + }); + let metric_key = if has_per_metric_config { + Some((metric_namespace, metric_name.clone())) + } else { + None + }; if let Some(tags_map) = metric.tags_mut() { - match self.config.limit_exceeded_action { + match self + .get_config_for_metric(metric_key.as_ref()) + .limit_exceeded_action + { LimitExceededAction::DropEvent => { // This needs to check all the tags, to ensure that the ordering of tag names // doesn't change the behavior of the check. for (key, value) in tags_map.iter_sets() { - if self.tag_limit_exceeded(key, value) { + if self.tag_limit_exceeded(metric_key.as_ref(), key, value) { emit!(TagCardinalityLimitRejectingEvent { metric_name: &metric_name, tag_key: key, @@ -109,12 +159,12 @@ impl TagCardinalityLimit { } } for (key, value) in tags_map.iter_sets() { - self.record_tag_value(key, value); + self.record_tag_value(metric_key.as_ref(), key, value); } } LimitExceededAction::DropTag => { tags_map.retain(|key, value| { - if self.try_accept_tag(key, value) { + if self.try_accept_tag(metric_key.as_ref(), key, value) { true } else { emit!(TagCardinalityLimitRejectingTag { diff --git a/src/transforms/tag_cardinality_limit/tests.rs b/src/transforms/tag_cardinality_limit/tests.rs index d4e63b0820..8e573cc959 100644 --- a/src/transforms/tag_cardinality_limit/tests.rs +++ b/src/transforms/tag_cardinality_limit/tests.rs @@ -1,5 +1,7 @@ +use std::collections::HashMap; use std::sync::Arc; +use config::PerMetricConfig; use vector_lib::config::ComponentKey; use vector_lib::config::OutputId; use vector_lib::event::EventMetadata; @@ -24,12 +26,12 @@ fn generate_config() { crate::test_util::test_generate_config::(); } -fn make_metric(tags: MetricTags) -> Event { +fn make_metric_with_name(tags: MetricTags, name: &str) -> Event { let event_metadata = EventMetadata::default().with_source_type("unit_test_stream"); Event::Metric( Metric::new_with_metadata( - "event", + name, metric::MetricKind::Incremental, metric::MetricValue::Counter { value: 1.0 }, event_metadata, @@ -38,27 +40,69 @@ fn make_metric(tags: MetricTags) -> Event { ) } -const fn make_transform_hashset( +fn make_metric(tags: MetricTags) -> Event { + make_metric_with_name(tags, "event") +} + +fn make_transform_hashset( + value_limit: usize, + limit_exceeded_action: LimitExceededAction, +) -> TagCardinalityLimitConfig { + TagCardinalityLimitConfig { + global: TagCardinalityLimitInnerConfig { + value_limit, + limit_exceeded_action, + mode: Mode::Exact, + }, + per_metric_limits: HashMap::new(), + } +} + +fn make_transform_bloom( + value_limit: usize, + limit_exceeded_action: LimitExceededAction, +) -> TagCardinalityLimitConfig { + TagCardinalityLimitConfig { + global: TagCardinalityLimitInnerConfig { + value_limit, + limit_exceeded_action, + mode: Mode::Probabilistic(BloomFilterConfig { + cache_size_per_key: default_cache_size(), + }), + }, + per_metric_limits: HashMap::new(), + } +} + +const fn make_transform_hashset_with_per_metric_limits( value_limit: usize, limit_exceeded_action: LimitExceededAction, + per_metric_limits: HashMap, ) -> TagCardinalityLimitConfig { TagCardinalityLimitConfig { - value_limit, - limit_exceeded_action, - mode: Mode::Exact, + global: TagCardinalityLimitInnerConfig { + value_limit, + limit_exceeded_action, + mode: Mode::Exact, + }, + per_metric_limits, } } -const fn make_transform_bloom( +const fn make_transform_bloom_with_per_metric_limits( value_limit: usize, limit_exceeded_action: LimitExceededAction, + per_metric_limits: HashMap, ) -> TagCardinalityLimitConfig { TagCardinalityLimitConfig { - value_limit, - limit_exceeded_action, - mode: Mode::Probabilistic(BloomFilterConfig { - cache_size_per_key: default_cache_size(), - }), + global: TagCardinalityLimitInnerConfig { + value_limit, + limit_exceeded_action, + mode: Mode::Probabilistic(BloomFilterConfig { + cache_size_per_key: default_cache_size(), + }), + }, + per_metric_limits, } } @@ -359,3 +403,186 @@ fn drop_event_checks_all_tags(make_tags: impl Fn(&str, &str) -> MetricTags) { assert_eq!(new_event3, None); assert_eq!(new_event4, Some(event4)); } + +#[tokio::test] +async fn tag_cardinality_limit_separate_value_limit_per_metric_name_hashset() { + separate_value_limit_per_metric_name(make_transform_hashset_with_per_metric_limits( + 2, + LimitExceededAction::DropTag, + HashMap::from([ + ( + "metricA".to_string(), + PerMetricConfig { + namespace: None, + config: make_transform_hashset(1, LimitExceededAction::DropTag).global, + }, + ), + ( + "metricB".to_string(), + PerMetricConfig { + namespace: None, + config: make_transform_hashset(5, LimitExceededAction::DropTag).global, + }, + ), + ]), + )) + .await; +} + +#[tokio::test] +async fn tag_cardinality_limit_separate_value_limit_per_metric_name_bloom() { + separate_value_limit_per_metric_name(make_transform_bloom_with_per_metric_limits( + 2, + LimitExceededAction::DropTag, + HashMap::from([ + ( + "metricA".to_string(), + PerMetricConfig { + namespace: None, + config: make_transform_bloom(1, LimitExceededAction::DropTag).global, + }, + ), + ( + "metricB".to_string(), + PerMetricConfig { + namespace: None, + config: make_transform_bloom(5, LimitExceededAction::DropTag).global, + }, + ), + ]), + )) + .await; +} + +/// Test that hitting the value limit on one tag does not affect the ability to take new +/// values for other tags. +async fn separate_value_limit_per_metric_name(config: TagCardinalityLimitConfig) { + assert_transform_compliance(async move { + let mut event_a1 = + make_metric_with_name(metric_tags!("tag1" => "val1", "tag2" => "val1"), "metricA"); + + // The limit for tag1 should already be reached here + let mut event_a2 = + make_metric_with_name(metric_tags!("tag1" => "val2", "tag2" => "val1"), "metricA"); + + // The limit for tag2 should be reached here + let mut event_a3 = + make_metric_with_name(metric_tags!("tag1" => "val1", "tag2" => "val2"), "metricA"); + + // MetricB should have all of its tags kept due to higher limit + let mut event_b1 = + make_metric_with_name(metric_tags!("tag1" => "val1", "tag2" => "val1"), "metricB"); + let mut event_b2 = + make_metric_with_name(metric_tags!("tag1" => "val2", "tag2" => "val1"), "metricB"); + let mut event_b3 = + make_metric_with_name(metric_tags!("tag1" => "val1", "tag2" => "val2"), "metricB"); + + // MetricC has no specific config, so it uses the global config, which allows 2 values + let mut event_c1 = + make_metric_with_name(metric_tags!("tag1" => "val1", "tag2" => "val1"), "metricC"); + let mut event_c2 = + make_metric_with_name(metric_tags!("tag1" => "val2", "tag2" => "val2"), "metricC"); + // The limit for tag2 should be reached here + let mut event_c3 = + make_metric_with_name(metric_tags!("tag1" => "val1", "tag2" => "val3"), "metricC"); + + let (tx, rx) = mpsc::channel(1); + let (topology, mut out) = create_topology(ReceiverStream::new(rx), config).await; + + let events = vec![ + &mut event_a1, + &mut event_a2, + &mut event_a3, + &mut event_b1, + &mut event_b2, + &mut event_b3, + &mut event_c1, + &mut event_c2, + &mut event_c3, + ]; + + for event in &events { + tx.send((*event).clone()).await.unwrap(); + } + + let new_event_a1 = out.recv().await; + let new_event_a2 = out.recv().await; + let new_event_a3 = out.recv().await; + let new_event_b1 = out.recv().await; + let new_event_b2 = out.recv().await; + let new_event_b3 = out.recv().await; + let new_event_c1 = out.recv().await; + let new_event_c2 = out.recv().await; + let new_event_c3 = out.recv().await; + + drop(tx); + topology.stop().await; + + for event in events { + event.set_source_id(Arc::new(ComponentKey::from("in"))); + event.set_upstream_id(Arc::new(OutputId::from("transform"))); + event.metadata_mut().set_schema_definition(&Arc::new( + Definition::new_with_default_metadata(Kind::any_object(), [LogNamespace::Legacy]), + )); + } + + assert_eq!(new_event_a1, Some(event_a1)); + // The second event should have been modified to remove "tag1" + let new_event_a2 = new_event_a2.unwrap(); + assert!(!new_event_a2 + .as_metric() + .tags() + .unwrap() + .contains_key("tag1")); + assert_eq!( + "val1", + new_event_a2 + .as_metric() + .tags() + .unwrap() + .get("tag2") + .unwrap() + ); + + // The third event should have been modified to remove "tag2" + let new_event_a3 = new_event_a3.unwrap(); + assert!(!new_event_a3 + .as_metric() + .tags() + .unwrap() + .contains_key("tag2")); + assert_eq!( + "val1", + new_event_a3 + .as_metric() + .tags() + .unwrap() + .get("tag1") + .unwrap() + ); + + assert_eq!(new_event_b1, Some(event_b1)); + assert_eq!(new_event_b2, Some(event_b2)); + assert_eq!(new_event_b3, Some(event_b3)); + + assert_eq!(new_event_c1, Some(event_c1)); + assert_eq!(new_event_c2, Some(event_c2)); + // The third event should have been modified to remove "tag2" + let new_event_c3 = new_event_c3.unwrap(); + assert!(!new_event_c3 + .as_metric() + .tags() + .unwrap() + .contains_key("tag2")); + assert_eq!( + "val1", + new_event_c3 + .as_metric() + .tags() + .unwrap() + .get("tag1") + .unwrap() + ); + }) + .await; +} diff --git a/website/cue/reference/components/transforms/base/tag_cardinality_limit.cue b/website/cue/reference/components/transforms/base/tag_cardinality_limit.cue index 8e961d886b..5680d03c2c 100644 --- a/website/cue/reference/components/transforms/base/tag_cardinality_limit.cue +++ b/website/cue/reference/components/transforms/base/tag_cardinality_limit.cue @@ -46,6 +46,71 @@ base: components: transforms: tag_cardinality_limit: configuration: { """ } } + per_metric_limits: { + description: "Tag cardinality limits configuration per metric name." + required: false + type: object: options: "*": { + description: "An individual metric configuration." + required: true + type: object: options: { + cache_size_per_key: { + description: """ + The size of the cache for detecting duplicate tags, in bytes. + + The larger the cache size, the less likely it is to have a false positive, or a case where + we allow a new value for tag even after we have reached the configured limits. + """ + relevant_when: "mode = \"probabilistic\"" + required: false + type: uint: default: 5120 + } + limit_exceeded_action: { + description: """ + Possible actions to take when an event arrives that would exceed the cardinality limit for one + or more of its tags. + """ + required: false + type: string: { + default: "drop_tag" + enum: { + drop_event: "Drop the entire event itself." + drop_tag: "Drop the tag(s) that would exceed the configured limit." + } + } + } + mode: { + description: "Controls the approach taken for tracking tag cardinality." + required: true + type: string: enum: { + exact: """ + Tracks cardinality exactly. + + This mode has higher memory requirements than `probabilistic`, but never falsely outputs + metrics with new tags after the limit has been hit. + """ + probabilistic: """ + Tracks cardinality probabilistically. + + This mode has lower memory requirements than `exact`, but may occasionally allow metric + events to pass through the transform even when they contain new tags that exceed the + configured limit. The rate at which this happens can be controlled by changing the value of + `cache_size_per_key`. + """ + } + } + namespace: { + description: "Namespace of the metric this configuration refers to." + required: false + type: string: {} + } + value_limit: { + description: "How many distinct values to accept for any given key." + required: false + type: uint: default: 500 + } + } + } + } value_limit: { description: "How many distinct values to accept for any given key." required: false From 1af287f1f05050e0382d3dac6efb888508dce390 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 28 Jan 2025 15:46:45 -0500 Subject: [PATCH 033/658] chore(dev): populated the new github 'type' field for features (#22315) * chore(dev): populated the new github 'type' field for features * fix silly typo --- .github/ISSUE_TEMPLATE/feature.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/ISSUE_TEMPLATE/feature.yml b/.github/ISSUE_TEMPLATE/feature.yml index 3675a2173a..ed43f6891b 100644 --- a/.github/ISSUE_TEMPLATE/feature.yml +++ b/.github/ISSUE_TEMPLATE/feature.yml @@ -1,4 +1,5 @@ name: Feature +type: 'Feature' description: 🚀 Suggest a new feature. labels: - 'type: feature' From 9f9a573fca334c42f92bca7752ca00c458c3e106 Mon Sep 17 00:00:00 2001 From: Geoffrey Oxberry Date: Wed, 29 Jan 2025 09:47:10 -0800 Subject: [PATCH 034/658] chore(ci): update smp to 0.20.2 (#22318) As part of keeping Vector's `smp` up-to-date with latest stable, this commit updates the `smp` version used in CI from 0.20.1 to 0.20.2. Signed-off-by: Geoffrey M. Oxberry --- .github/workflows/regression.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 251e9cda0b..aeca1ca70b 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -105,7 +105,7 @@ jobs: - name: Set SMP version id: experimental-meta run: | - export SMP_CRATE_VERSION="0.20.1" + export SMP_CRATE_VERSION="0.20.2" echo "smp crate version: ${SMP_CRATE_VERSION}" echo "SMP_CRATE_VERSION=${SMP_CRATE_VERSION}" >> $GITHUB_OUTPUT From cb09d9a0cd73e3deb53bd440cd2e6b1e648c377a Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Wed, 29 Jan 2025 17:06:54 -0500 Subject: [PATCH 035/658] add debugging statement --- .github/workflows/regression.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index aeca1ca70b..949b833304 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -554,6 +554,7 @@ jobs: - name: Display Markdown Summary run: | + ls -R unzip capture-artifacts -d results REPORT_MD=results/report.md if [ -f ${REPORT_MD} ]; then From d49cf33748ad82292198888687b6b50a01d8500f Mon Sep 17 00:00:00 2001 From: Jesse Szwedko Date: Wed, 29 Jan 2025 14:45:44 -0800 Subject: [PATCH 036/658] chore(ci): Swap traditional file generator for logrotate_fs for file_to_blackhole test (#22285) * chore(ci): Swap traditional file generator for logrotate_fs for file_to_blackhole test The current config fails since limits started being applied to the lading container. We think this is a good time to update to the preferred generator anyway. Signed-off-by: Jesse Szwedko * Fix whitespace Signed-off-by: Jesse Szwedko --------- Signed-off-by: Jesse Szwedko --- .../cases/file_to_blackhole/lading/lading.yaml | 15 +++++++++------ .../cases/file_to_blackhole/vector/vector.yaml | 2 +- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/regression/cases/file_to_blackhole/lading/lading.yaml b/regression/cases/file_to_blackhole/lading/lading.yaml index 2b718c6b55..5017287363 100644 --- a/regression/cases/file_to_blackhole/lading/lading.yaml +++ b/regression/cases/file_to_blackhole/lading/lading.yaml @@ -1,14 +1,17 @@ generator: - file_gen: - traditional: + logrotate_fs: seed: [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131] - path_template: "/tmp/file-gen-%NNN%.log" - duplicates: 4 + load_profile: + constant: 100MiB + concurrent_logs: 4 + maximum_bytes_per_log: 100MiB + total_rotations: 5 + max_depth: 0 variant: "ascii" - bytes_per_second: "100Mb" - maximum_bytes_per_file: "100Mb" - maximum_prebuild_cache_size_bytes: "400Mb" + maximum_prebuild_cache_size_bytes: 250MiB + mount_point: /smp-shared blackhole: - tcp: diff --git a/regression/cases/file_to_blackhole/vector/vector.yaml b/regression/cases/file_to_blackhole/vector/vector.yaml index 48edf30e0e..377ae72154 100644 --- a/regression/cases/file_to_blackhole/vector/vector.yaml +++ b/regression/cases/file_to_blackhole/vector/vector.yaml @@ -11,7 +11,7 @@ sources: file: type: "file" include: - - "/tmp/file-gen-*.log" + - "/smp-shared/*.log" ## ## Sinks From 79852bbd794338c1ceda3f2c22fcb9eb175f930e Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Thu, 30 Jan 2025 09:57:25 -0500 Subject: [PATCH 037/658] fix(ci): make some final regression steps optional (#22324) --- .github/workflows/regression.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 949b833304..4b5141432b 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -548,11 +548,13 @@ jobs: FAILED: ${{ contains(needs.*.result, 'failure') }} steps: - name: Download capture-artifacts + continue-on-error: true uses: actions/download-artifact@v4 with: name: capture-artifacts - name: Display Markdown Summary + continue-on-error: true run: | ls -R unzip capture-artifacts -d results From 0431a079273432e69cb5a1fe9add9c85ca4e1edc Mon Sep 17 00:00:00 2001 From: Huang Chen-Yi Date: Fri, 31 Jan 2025 03:23:43 +0800 Subject: [PATCH 038/658] docs(external): fix `enrichment_table` and `secret` docs (#22319) * Add relevant notice for csv enrichment table * Try to fix the type for secret and description * Cue format --- website/cue/reference/configuration.cue | 229 +++++++++++------------- 1 file changed, 101 insertions(+), 128 deletions(-) diff --git a/website/cue/reference/configuration.cue b/website/cue/reference/configuration.cue index fe63239914..4833c3c060 100644 --- a/website/cue/reference/configuration.cue +++ b/website/cue/reference/configuration.cue @@ -159,9 +159,10 @@ configuration: { } delimiter: { - description: "The delimiter used to separate fields in each row of the CSV file." - common: false - required: false + description: "The delimiter used to separate fields in each row of the CSV file." + common: false + required: false + relevant_when: "type = \"csv\"" type: string: { default: "," examples: [":"] @@ -176,8 +177,9 @@ configuration: { If you set it to `false`, there are no headers and the columns are referred to by their numerical index. """ - required: false - common: false + required: false + relevant_when: "type = \"csv\"" + common: false type: bool: default: true } } @@ -571,148 +573,119 @@ configuration: { Configuration options to retrieve secrets from external backend in order to avoid storing secrets in plaintext in Vector config. Multiple backends can be configured. Use `SECRET[.]` to tell Vector to retrieve the secret. This placeholder is replaced by the secret retrieved from the relevant backend. + + When `type` is `exec`, the provided command will be run and provided a list of + secrets to fetch, determined from the configuration file, on stdin as JSON in the format: + + ```json + {"version": "1.0", "secrets": ["secret1", "secret2"]} + ``` + + The executable is expected to respond with the values of these secrets on stdout, also as JSON, in the format: + + ```json + { + "secret1": {"value": "secret_value", "error": null}, + "secret2": {"value": null, "error": "could not fetch the secret"} + } + ``` + If an `error` is returned for any secrets, or if the command exits with a non-zero status code, + Vector will log the errors and exit. + + Otherwise, the secret must be a JSON text string with key/value pairs. For example: + ```json + { + "username": "test", + "password": "example-password" + } + ``` + + If an error occurred while reading the file or retrieving the secrets, Vector logs the error and exits. + + Secrets are loaded when Vector starts or if Vector receives a `SIGHUP` signal triggering its + configuration reload process. + """ required: false type: object: options: { - file: { - required: true + type: { description: """ - Retrieve secrets from a file path. - - The secret must be a JSON text string with key/value pairs. For example: - ```json - { - "username": "test", - "password": "example-password" - } - ``` - - If an error occurs while reading the file, Vector will log the error and exit. - - Secrets are loaded when Vector starts or if Vector receives a `SIGHUP` signal triggering its - configuration reload process. + The type of secret backend to use. """ - type: object: options: { - path: { - description: "The file path to read." - required: true - type: string: { - examples: ["/path/to/secret.json"] - } + required: true + type: string: { + enum: { + "file": "Retrieve secrets from a file path." + "directory": "Retrieve secrets from file contents in a directory." + "exec": "Run a local command to retrieve secrets." + "aws_secrets_manager": "Retrieve secrets from AWS Secrets Manager." + "test": "Secret backend for test." } } } - directory: { - required: true + path: { description: """ - Retrieve secrets from file contents in a directory. - - The directory must contain files with names corresponding to secret keys. + When type is `file`, this value is the file path to read. - If an error occurs while reading the file, Vector will log the error and exit. - - Secrets are loaded when Vector starts or if Vector receives a `SIGHUP` signal triggering its - configuration reload process. + When type is `directory`, this value is the path of the directory with secrets. """ - type: object: options: { - path: { - description: """ - The path of the directory with secrets. - """ - required: true - type: string: { - examples: [ - "${CREDENTIALS_DIRECTORY}", // https://systemd.io/CREDENTIALS - "/path/to/secrets-directory", - ] - } - } - remove_trailing_whitespace: { - description: """ - Remove trailing whitespace from file contents. - """ - required: false - type: bool: default: false - } + required: true + relevant_when: "type = \"file\" or type = \"directory\"" + type: string: { + examples: [ + "/path/to/secret.json", + "${CREDENTIALS_DIRECTORY}", // https://systemd.io/CREDENTIALS + "/path/to/secrets-directory", + ] } } - exec: { - required: true + remove_trailing_whitespace: { description: """ - Run a local command to retrieve secrets. - - The provided command will be run and provided a list of - secrets to fetch, determined from the configuration file, on stdin as JSON in the format: - - ```json - {"version": "1.0", "secrets": ["secret1", "secret2"]} - ``` - - The executable is expected to respond with the values of these secrets on stdout, also as JSON, in the format: - - ```json - { - "secret1": {"value": "secret_value", "error": null}, - "secret2": {"value": null, "error": "could not fetch the secret"} - } - ``` - - If an `error` is returned for any secrets, or if the command exits with a non-zero status code, - Vector will log the errors and exit. - - Secrets are loaded when Vector starts or if Vector receives a `SIGHUP` signal triggering its - configuration reload process. + Remove trailing whitespace from file contents. """ - type: object: options: { - command: { - description: """ - The command to be run, plus any arguments required. - """ - required: true - type: array: { - examples: [["/path/to/get-secret", "-s"], ["/path/to/vault-wrapper"]] - items: type: string: {} - } - } - timeout: { - description: "The amount of time Vector will wait for the command to complete." - required: false - common: false - type: uint: { - default: 5 - unit: "seconds" - } - } + required: false + relevant_when: "type = \"directory\"" + type: bool: default: false + } + replacement: { + description: """ + Fixed value to replace all secrets with. + """ + required: false + relevant_when: "type = \"test\"" + type: string: { + examples: ["test"] } } - aws_secrets_manager: { - required: true + command: { description: """ - Retrieve secrets from AWS Secrets Manager. - - The secret must be a JSON text string with key/value pairs. For example: - ```json - { - "username": "test", - "password": "example-password" - } - ``` - - If an error occurred retrieving the secrets, Vector logs the error and exits. - - Secrets are loaded when Vector starts or if Vector receives a `SIGHUP` signal triggering its - configuration reload process. + The command to be run, plus any arguments required. """ - type: object: options: { - secret_id: { - description: """ - The ID of the secret to be retrieved. - """ - required: true - type: string: { - examples: ["/secret/foo-bar"] - } - } + required: true + relevant_when: "type = \"exec\"" + type: array: { + examples: [["/path/to/get-secret", "-s"], ["/path/to/vault-wrapper"]] + items: type: string: {} + } + } + timeout: { + description: "The amount of time Vector will wait for the command to complete." + required: false + relevant_when: "type = \"exec\"" + common: false + type: uint: { + default: 5 + unit: "seconds" + } + } + secret_id: { + description: """ + The ID of the secret to be retrieved. + """ + required: true + relevant_when: "type = \"aws_secrets_manager\"" + type: string: { + examples: ["/secret/foo-bar"] } } } From acd7dd25d1192b344285c73a1541b42909d1af7d Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Thu, 30 Jan 2025 15:42:49 -0500 Subject: [PATCH 039/658] docs(external): fix getting starting guide rendering (#22332) * docs(external): fix getting starting guide rendering * update weights --- website/content/en/guides/_index.md | 1 + website/content/en/guides/getting-started/_index.md | 6 ++++++ 2 files changed, 7 insertions(+) create mode 100644 website/content/en/guides/getting-started/_index.md diff --git a/website/content/en/guides/_index.md b/website/content/en/guides/_index.md index 12bc11d63b..0e170f7e68 100644 --- a/website/content/en/guides/_index.md +++ b/website/content/en/guides/_index.md @@ -4,4 +4,5 @@ description: Thoughtful guides to help you get the most out of Vector. Created a short: Guides tags: ["guides", "guide"] aliases: ["/docs/setup/guides"] +weight: 2 --- diff --git a/website/content/en/guides/getting-started/_index.md b/website/content/en/guides/getting-started/_index.md new file mode 100644 index 0000000000..8fe4b3c3c4 --- /dev/null +++ b/website/content/en/guides/getting-started/_index.md @@ -0,0 +1,6 @@ +--- +title: Getting Started Guides +description: Getting Started +weight: 1 +tags: [ "getting started", "guides", "guide" ] +--- From 0d57e2aed0b300a8fbe9626283582578484c1620 Mon Sep 17 00:00:00 2001 From: ArunPiduguDD Date: Fri, 31 Jan 2025 09:51:51 -0500 Subject: [PATCH 040/658] =?UTF-8?q?enhancement(gcp=5Fchronicle=20sink):=20?= =?UTF-8?q?Add=20default=20fallback=20logic=20if=20log=5Ftype=20template?= =?UTF-8?q?=20cannot=20be=20resolved=20fo=E2=80=A6=20(#22323)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add default fallback logic if log_type template cannot be resolved for Google Chronicle sink * modify error message * fix changelog * format * update docs * add suggested changes * update docs again --- ...index_google_chronicle_sink.enhancement.md | 3 ++ .../gcp_chronicle/chronicle_unstructured.rs | 6 +++ src/sinks/gcp_chronicle/partitioner.rs | 44 ++++++++++++++----- .../sinks/base/gcp_chronicle_unstructured.cue | 5 +++ 4 files changed, 47 insertions(+), 11 deletions(-) create mode 100644 changelog.d/add_default_fallback_index_google_chronicle_sink.enhancement.md diff --git a/changelog.d/add_default_fallback_index_google_chronicle_sink.enhancement.md b/changelog.d/add_default_fallback_index_google_chronicle_sink.enhancement.md new file mode 100644 index 0000000000..04d2a76a82 --- /dev/null +++ b/changelog.d/add_default_fallback_index_google_chronicle_sink.enhancement.md @@ -0,0 +1,3 @@ +Add an option to Google Chronicle sink to set a fallback index if the provided template in the `log_type` field cannot be resolved + +authors: ArunPiduguDD diff --git a/src/sinks/gcp_chronicle/chronicle_unstructured.rs b/src/sinks/gcp_chronicle/chronicle_unstructured.rs index 2c30b157ec..537ce9dde6 100644 --- a/src/sinks/gcp_chronicle/chronicle_unstructured.rs +++ b/src/sinks/gcp_chronicle/chronicle_unstructured.rs @@ -237,6 +237,10 @@ pub struct ChronicleUnstructuredConfig { #[configurable(metadata(docs::examples = "WINDOWS_DNS", docs::examples = "{{ log_type }}"))] pub log_type: Template, + /// The default `log_type` to attach to events if the template in `log_type` cannot be resolved. + #[configurable(metadata(docs::examples = "VECTOR_DEV"))] + pub fallback_log_type: Option, + #[configurable(derived)] #[serde( default, @@ -261,6 +265,7 @@ impl GenerateConfig for ChronicleUnstructuredConfig { namespace = "namespace" compression = "gzip" log_type = "log_type" + fallback_log_type = "VECTOR_DEV" encoding.codec = "text" "#}) .unwrap() @@ -355,6 +360,7 @@ impl ChronicleUnstructuredConfig { fn partitioner(&self) -> crate::Result { Ok(ChroniclePartitioner::new( self.log_type.clone(), + self.fallback_log_type.clone(), self.namespace.clone(), )) } diff --git a/src/sinks/gcp_chronicle/partitioner.rs b/src/sinks/gcp_chronicle/partitioner.rs index 35ee79cf96..24181a458d 100644 --- a/src/sinks/gcp_chronicle/partitioner.rs +++ b/src/sinks/gcp_chronicle/partitioner.rs @@ -9,11 +9,23 @@ pub struct ChroniclePartitionKey { } /// Partitions items based on the generated key for the given event. -pub struct ChroniclePartitioner(Template, Option