diff --git a/.dockerignore b/.dockerignore index f8bae8f065c5c..02c60ccd5e905 100644 --- a/.dockerignore +++ b/.dockerignore @@ -10,3 +10,6 @@ !rust-toolchain.toml !scripts !vdev + +# Exclude target directories to avoid copying build artifacts +target diff --git a/.github/ISSUE_TEMPLATE/minor-release.md b/.github/ISSUE_TEMPLATE/minor-release.md index d83bc945879a3..bcecf75087bc9 100644 --- a/.github/ISSUE_TEMPLATE/minor-release.md +++ b/.github/ISSUE_TEMPLATE/minor-release.md @@ -11,10 +11,11 @@ labels: "domain: releasing" Note the preparation steps are now automated. First, alter/create release.env ```shell +#!/usr/bin/env bash export NEW_VECTOR_VERSION= # replace this with the actual new version (e.g.: 0.50.0) export NEW_VRL_VERSION= # replace this with the actual new VRL version (e.g.: 0.30.0) export MINOR_VERSION=$(echo "$NEW_VECTOR_VERSION" | cut -d. -f2) -export PREP_BRANCH=prepare-v-0-"${MINOR_VERSION}"-"${NEW_VECTOR_VERSION}"-website +export PREP_BRANCH=prepare-v-"${NEW_VECTOR_VERSION//./-}"-website export RELEASE_BRANCH=v0."${MINOR_VERSION}" ``` @@ -24,7 +25,7 @@ and then source it by running `source ./release.env` ## 1. Manual Steps -- [ ] Cut a new release of [VRL](https://github.com/vectordotdev/vrl) if needed +- [ ] Cut a new release of [VRL](https://github.com/vectordotdev/vrl) if needed. - VRL release steps: https://github.com/vectordotdev/vrl/blob/main/release/README.md ## 2. Automated Steps @@ -37,12 +38,11 @@ cargo vdev release prepare --version "${NEW_VECTOR_VERSION}" --vrl-version "${NE Automated steps include: - [ ] Create a new release branch from master to freeze commits - - `git fetch && git checkout origin/master && git checkout -b "{RELEASE_BRANCH}" && git push -u` + - `git fetch && git checkout origin/master && git checkout -b "${RELEASE_BRANCH}" && git push -u` - [ ] Create a new release preparation branch from `master` - `git checkout -b "${PREP_BRANCH}" && git push -u` - [ ] Pin VRL to latest released version rather than `main` -- [ ] Check if there is a newer version of [Alpine](https://alpinelinux.org/releases/) or - [Debian](https://www.debian.org/releases/) available to update the release images in +- [ ] Check if there is a newer version of [Alpine](https://alpinelinux.org/releases/) or [Debian](https://www.debian.org/releases/) available to update the release images in `distribution/docker/`. Update if so. - [ ] Run `cargo vdev build release-cue` to generate a new cue file for the release - [ ] Copy VRL changelogs from the VRL version in the last Vector release as a new changelog entry @@ -60,47 +60,44 @@ Automated steps include: - [ ] Edit `website/cue/reference/releases/"${NEW_VECTOR_VERSION}".cue` - [ ] Add description key to the generated cue file with a description of the release (see previous releases for examples). - - [ ] Ensure any breaking changes are highlighted in the release upgrade guide - - [ ] Ensure any deprecations are highlighted in the release upgrade guide - - [ ] Review generated changelog entries to ensure they are understandable to end-users + - [ ] Ensure any breaking changes are highlighted in the release upgrade guide. + - [ ] Ensure any deprecations are highlighted in the release upgrade guide. + - [ ] Review generated changelog entries to ensure they are understandable to end-users. - [ ] Ensure the date matches the scheduled release date. - [ ] Add a link to pending deprecation items from [DEPRECATIONS.md](https://github.com/vectordotdev/vector/blob/master/docs/DEPRECATIONS.md). -- [ ] PR review & approval +- [ ] PR review & approval. # On the day of release -- [ ] Make sure the release branch is in sync with origin/master and has only one squashed commit with all commits from the prepare branch. If you made a PR from the prepare branch into the release branch this should already be the case +- [ ] Make sure the release branch is in sync with origin/master and has only one squashed commit with all commits from the prepare branch. If you made a PR from the prepare branch into the release branch this should already be the case. - [ ] `git checkout "${RELEASE_BRANCH}"` - - [ ] `git show --stat HEAD` - This should show the squashed prepare commit + - [ ] `git show --stat HEAD` - This should show the squashed prepare commit. - [ ] Ensure release date in `website/cue/reference/releases/0.XX.Y.cue` matches current date. - - If this needs to be updated commit and squash it in the release branch + - If this needs to be updated commit and squash it in the release branch. - Follow these steps if the release branch needs to be updated - - [ ] Rebase the release preparation branch on the release branch + - [ ] Rebase the release preparation branch on the release branch. - [ ] Squash the release preparation commits (but not the cherry-picked commits!) to a single commit. This makes it easier to cherry-pick to master after the release. - - [ ] Merge release preparation branch into the release branch + - [ ] Merge release preparation branch into the release branch. - `git switch "${RELEASE_BRANCH}" && git merge --ff-only "${PREP_BRANCH}"` - [ ] Tag new release - [ ] `git tag v"${NEW_VECTOR_VERSION}" -a -m v"${NEW_VECTOR_VERSION}"` - [ ] `git push origin v"${NEW_VECTOR_VERSION}"` -- [ ] Wait for release workflow to complete +- [ ] Wait for release workflow to complete. - Discoverable via [release.yml](https://github.com/vectordotdev/vector/actions/workflows/release.yml) - [ ] Reset the `website` branch to the `HEAD` of the release branch to update https://vector.dev - [ ] `git switch website && git reset --hard origin/"${RELEASE_BRANCH}" && git push` - [ ] Confirm that the release changelog was published to https://vector.dev/releases/ - - The deployment is done by Amplify. You can see - the [deployment logs here](https://dd-corpsite.datadoghq.com/logs?query=service%3Awebsites-vector%20branch%3Awebsite&agg_m=count&agg_m_source=base&agg_t=count&cols=host%2Cservice&fromUser=true&messageDisplay=inline&refresh_mode=sliding&storage=hot&stream_sort=time%2Casc&viz=stream). -- [ ] Release Linux packages. See [`vector-release` usage](https://github.com/DataDog/vector-release#usage). - - Note: the pipeline inputs are the version number `v"${NEW_VECTOR_VERSION}"` and a personal GitHub token. - - [ ] Manually trigger the `trigger-package-release-pipeline-prod-stable` job. -- [ ] Release updated Helm chart. See [releasing Helm chart](https://github.com/vectordotdev/helm-charts#releasing). -- [ ] Once Helm chart is released, updated Vector manifests - - Run `cargo vdev build manifests` and open a PR with changes -- [ ] Add docker images to [https://github.com/DataDog/images](https://github.com/DataDog/images/tree/master/vector) to have them available internally. ([Example PR](https://github.com/DataDog/images/pull/7104)) + - Refer to the internal releasing doc to monitor the deployment. +- [ ] Release Linux packages. Refer to the internal releasing doc. +- [ ] Release updated Helm chart. See [releasing Helm chart](https://github.com/vectordotdev/helm-charts/blob/develop/RELEASING.md). +- [ ] Release Homebrew. Refer to the internal releasing doc. +- [ ] Create internal Docker images. Refer to the internal releasing doc. +- [ ] Update the latest [release tag](https://github.com/vectordotdev/vector/release) description with the release announcement. - [ ] Create a new PR with title starting as `chore(releasing):` - - [ ] Cherry-pick any release commits from the release branch that are not on `master`, to `master` + - [ ] Cherry-pick any release commits from the release branch that are not on `master`, to `master`. + - [ ] Run `cargo vdev build manifests` and commit changes. - [ ] Bump the release number in the `Cargo.toml` on master to the next minor release. - - [ ] Also, update `Cargo.lock` with: `cargo update -p vector` + - [ ] Also, update `Cargo.lock` with: `cargo update -p vector`. - [ ] If there is a VRL version update, revert it and make it track the git `main` branch and then run `cargo update -p vrl`. -- [ ] Kick-off post-mortems for any regressions resolved by the release diff --git a/.github/actions/install-vdev/action.yml b/.github/actions/install-vdev/action.yml index 0c6875a153b7c..7b35d99d5a253 100644 --- a/.github/actions/install-vdev/action.yml +++ b/.github/actions/install-vdev/action.yml @@ -5,12 +5,19 @@ branding: icon: tool color: purple +inputs: + skip-cache: + description: "Skip cache lookup and force compilation" + required: false + default: 'false' + runs: using: "composite" steps: - name: Cache vdev binary id: cache-vdev - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + if: ${{ inputs.skip-cache != 'true' }} + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 with: path: ~/.cargo/bin/vdev key: ${{ runner.os }}-vdev-${{ hashFiles('vdev/**', 'Cargo.toml', 'Cargo.lock') }} @@ -23,3 +30,15 @@ runs: run: | echo "Building vdev from source (cache miss)" cargo install --path vdev --locked --force + + - name: Save vdev to cache + if: ${{ inputs.skip-cache == 'true' }} + uses: actions/cache/save@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 + with: + path: ~/.cargo/bin/vdev + key: ${{ runner.os }}-vdev-${{ hashFiles('vdev/**', 'Cargo.toml', 'Cargo.lock') }} + + - name: Set VDEV environment variable + shell: bash + run: | + echo "VDEV=$(which vdev)" >> "$GITHUB_ENV" diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 6a22e716ef990..d8c5e8c02ba41 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -9,7 +9,7 @@ inputs: cargo-cache: required: false default: true - description: "Cache Cargo registry, index, git DB, and ~/.cargo/bin. Enabled automatically if any Rust tools are requested." + description: "Cache Cargo registry, index, and git DB. Enabled automatically if any Rust tools are requested." mold: required: false @@ -24,6 +24,10 @@ inputs: required: false default: false description: "Install cue." + libsasl2: + required: false + default: false + description: "Install libsasl2." # Required to fully build Vector # prepare.sh - rust rust: # rustup module @@ -60,7 +64,7 @@ inputs: description: "Install wasm-pack for Rust to WASM bridge." vdev: required: false - default: false + default: true description: "Install vdev CLI tool (cached by vdev/ directory changes)." # prepare.sh - npm @@ -76,13 +80,24 @@ inputs: runs: using: "composite" steps: + - name: Check vdev cache status + id: check-vdev-cache + if: ${{ inputs.vdev == 'true' }} + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 + with: + path: ~/.cargo/bin/vdev + key: ${{ runner.os }}-vdev-${{ hashFiles('vdev/**', 'Cargo.toml', 'Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-vdev- + lookup-only: true + - id: needs name: Compute if rust is needed shell: bash run: | rust="false" - # If any rust-related inputs are true, enable cache regardless + # If any rust-related inputs are true (excluding vdev), enable rust for val in "${{ inputs.mold }}" \ "${{ inputs.rust }}" \ "${{ inputs.cargo-deb }}" \ @@ -91,15 +106,22 @@ runs: "${{ inputs.cargo-deny }}" \ "${{ inputs.cargo-msrv }}" \ "${{ inputs.dd-rust-license-tool }}" \ - "${{ inputs.wasm-pack }}" \ - "${{ inputs.vdev }}"; do + "${{ inputs.wasm-pack }}"; do if [[ "$val" == "true" ]]; then rust="true" break fi done + # If vdev needs compilation (cache miss), enable rust + vdev_needs_compile="false" + if [[ "${{ inputs.vdev }}" == "true" && "${{ steps.check-vdev-cache.outputs.cache-hit }}" != "true" ]]; then + rust="true" + vdev_needs_compile="true" + fi + echo "NEEDS_RUST=$rust" >> "$GITHUB_ENV" + echo "VDEV_NEEDS_COMPILE=$vdev_needs_compile" >> "$GITHUB_ENV" if [[ "$rust" == "true" ]]; then echo "RUST_BACKTRACE=full" >> "$GITHUB_ENV" @@ -124,12 +146,11 @@ runs: EOF shell: bash - - name: Cache Cargo registry + index - if: ${{ inputs.cargo-cache == 'true' || env.NEEDS_RUST == 'true' }} - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + - name: Cache Cargo registry, index, and git DB + if: ${{ inputs.cargo-cache == 'true' || env.NEEDS_RUST == 'true' || env.VDEV_NEEDS_COMPILE == 'true' }} + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 with: path: | - ~/.cargo/bin/ ~/.cargo/registry/index/ ~/.cargo/registry/cache/ ~/.cargo/git/db/ @@ -138,7 +159,7 @@ runs: ${{ runner.os }}-cargo- - name: Install mold - if: ${{ inputs.mold == 'true' }} + if: ${{ runner.os == 'Linux' && env.DISABLE_MOLD != 'true' && (inputs.mold == 'true' || env.VDEV_NEEDS_COMPILE == 'true') }} shell: bash run: | echo "Installing mold" @@ -189,7 +210,11 @@ runs: shell: bash run: | echo "Installing protoc" - sudo bash ./scripts/environment/install-protoc.sh + if [[ "${{ runner.os }}" == "macOS" ]]; then + sudo bash ./scripts/environment/install-protoc.sh /usr/local/bin + else + sudo bash ./scripts/environment/install-protoc.sh + fi - name: Install cue if: ${{ inputs.cue == 'true' }} @@ -206,9 +231,34 @@ runs: sudo cp "${TEMP}/cue" /usr/bin/cue rm -rf "$TEMP" + - name: Install libsasl2 + if: ${{ inputs.libsasl2 == 'true' }} + shell: bash + run: sudo apt-get update && sudo apt-get install -y libsasl2-dev + - name: Install vdev if: ${{ inputs.vdev == 'true' }} uses: ./.github/actions/install-vdev + with: + skip-cache: ${{ env.VDEV_NEEDS_COMPILE == 'true' }} + + - name: Cache prepare.sh binaries + id: cache-prepare-binaries + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 + with: + path: | + ~/.cargo/bin/cargo-deb + ~/.cargo/bin/cross + ~/.cargo/bin/cargo-nextest + ~/.cargo/bin/cargo-deny + ~/.cargo/bin/cargo-msrv + ~/.cargo/bin/dd-rust-license-tool + ~/.cargo/bin/wasm-pack + /usr/local/bin/markdownlint + /usr/local/bin/datadog-ci + key: ${{ runner.os }}-prepare-binaries-${{ hashFiles('scripts/environment/*') }} + restore-keys: | + ${{ runner.os }}-prepare-binaries- - name: Run prepare.sh shell: bash @@ -226,4 +276,6 @@ runs: [[ "${{ inputs.datadog-ci }}" == "true" ]] && mods+=("datadog-ci") csm=$(IFS=,; echo "${mods[*]}") - [[ "$csm" != "" ]] && ./scripts/environment/prepare.sh --modules="${csm}" + if [[ "$csm" != "" ]]; then + ./scripts/environment/prepare.sh --modules="${csm}" + fi diff --git a/.github/actions/spelling/allow.txt b/.github/actions/spelling/allow.txt index 85ca0f9f2981f..9b14c6dbd7086 100644 --- a/.github/actions/spelling/allow.txt +++ b/.github/actions/spelling/allow.txt @@ -355,6 +355,7 @@ netcat netdata Netflix netlify +netlink Neue neuronull Nextbook diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt index 3f07a7f991a9a..1d68a6978570b 100644 --- a/.github/actions/spelling/expect.txt +++ b/.github/actions/spelling/expect.txt @@ -430,6 +430,7 @@ pgo PIDs PII plainify +plds ple podspec Ponge diff --git a/.github/audit.yml b/.github/audit.yml deleted file mode 100644 index 2544a1bad0530..0000000000000 --- a/.github/audit.yml +++ /dev/null @@ -1,18 +0,0 @@ -name: Security audit - -on: - schedule: - - cron: '0 * * * *' - push: - branches: - - master - -jobs: - security_audit: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - # TODO: replace this action - abandoned since 2020 - - uses: actions-rs/audit-check@35b7b53b1e25b55642157ac01b4adceb5b9ebef3 # v1.2.0 - with: - token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/dependabot.yml b/.github/dependabot.yml index a46fd34919f1f..7a362b2e66f4f 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -41,6 +41,9 @@ updates: patterns: - "futures" - "futures-util" + graphql: + patterns: + - "async-graphql*" metrics: patterns: - "metrics" @@ -49,9 +52,6 @@ updates: netlink: patterns: - "netlink-*" - graphql: - patterns: - - "async-graphql*" phf: patterns: - "phf*" @@ -59,6 +59,14 @@ updates: patterns: - "prost" - "prost-*" + serde: + patterns: + - "serde" + - "serde_*" + tokio: + patterns: + - "tokio" + - "tokio-*" tonic: patterns: - "tonic" @@ -67,6 +75,10 @@ updates: patterns: - "tower" - "tower-*" + tracing: + patterns: + - "tracing" + - "tracing-*" wasm-bindgen: patterns: - "wasm-bindgen-*" @@ -74,7 +86,22 @@ updates: patterns: - "zstd*" - package-ecosystem: "docker" - directory: "/distribution/docker/" + directory: "/distribution/docker/alpine" + schedule: + interval: "monthly" + time: "04:00" # UTC + labels: + - "domain: releasing" + - "no-changelog" + commit-message: + prefix: "chore(deps)" + open-pull-requests-limit: 100 + groups: + docker-images: + patterns: + - "*" + - package-ecosystem: "docker" + directory: "/distribution/docker/debian" schedule: interval: "monthly" time: "04:00" # UTC @@ -84,6 +111,40 @@ updates: commit-message: prefix: "chore(deps)" open-pull-requests-limit: 100 + groups: + docker-images: + patterns: + - "*" + - package-ecosystem: "docker" + directory: "/distribution/docker/distroless-static" + schedule: + interval: "monthly" + time: "04:00" # UTC + labels: + - "domain: releasing" + - "no-changelog" + commit-message: + prefix: "chore(deps)" + open-pull-requests-limit: 100 + groups: + docker-images: + patterns: + - "*" + - package-ecosystem: "docker" + directory: "/distribution/docker/distroless-libc" + schedule: + interval: "monthly" + time: "04:00" # UTC + labels: + - "domain: releasing" + - "no-changelog" + commit-message: + prefix: "chore(deps)" + open-pull-requests-limit: 100 + groups: + docker-images: + patterns: + - "*" - package-ecosystem: "github-actions" directory: "/" schedule: diff --git a/.github/workflows/build-test-runner.yml b/.github/workflows/build-test-runner.yml index d7dce608294f6..82e247c6f6f96 100644 --- a/.github/workflows/build-test-runner.yml +++ b/.github/workflows/build-test-runner.yml @@ -24,7 +24,7 @@ jobs: build: runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ inputs.checkout_ref || inputs.commit_sha }} @@ -33,7 +33,7 @@ jobs: vdev: true - name: Login to GitHub Container Registry - uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: registry: ghcr.io username: ${{ github.actor }} diff --git a/.github/workflows/changelog.yaml b/.github/workflows/changelog.yaml index a3534947a1c2e..e1d6beffdbf84 100644 --- a/.github/workflows/changelog.yaml +++ b/.github/workflows/changelog.yaml @@ -16,6 +16,9 @@ on: merge_group: types: [checks_requested] +permissions: + contents: read + jobs: validate-changelog: permissions: @@ -42,7 +45,7 @@ jobs: exit 0 # Checkout changelog script and changelog.d/ from master - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 if: env.SHOULD_RUN == 'true' with: ref: master @@ -52,7 +55,7 @@ jobs: sparse-checkout-cone-mode: false # Checkout PR's changelog.d/ into tmp/ - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 if: env.SHOULD_RUN == 'true' with: repository: ${{ github.event.pull_request.head.repo.full_name }} diff --git a/.github/workflows/changes.yml b/.github/workflows/changes.yml index d916a66307274..49e9fd19ebe6d 100644 --- a/.github/workflows/changes.yml +++ b/.github/workflows/changes.yml @@ -32,6 +32,8 @@ on: value: ${{ jobs.source.outputs.source }} dependencies: value: ${{ jobs.source.outputs.dependencies }} + deny: + value: ${{ jobs.source.outputs.deny }} internal_events: value: ${{ jobs.source.outputs.internal_events }} cue: @@ -90,6 +92,8 @@ on: value: ${{ jobs.int_tests.outputs.dnstap }} docker-logs: value: ${{ jobs.int_tests.outputs.docker-logs }} + doris: + value: ${{ jobs.int_tests.outputs.doris }} elasticsearch: value: ${{ jobs.int_tests.outputs.elasticsearch }} eventstoredb: @@ -141,6 +145,8 @@ on: value: ${{ jobs.e2e_tests.outputs.datadog-metrics }} e2e-opentelemetry-logs: value: ${{ jobs.e2e_tests.outputs.opentelemetry-logs }} + e2e-opentelemetry-metrics: + value: ${{ jobs.e2e_tests.outputs.opentelemetry-metrics }} int-tests-any: value: ${{ jobs.int_tests.outputs.any }} e2e-tests-any: @@ -158,6 +164,7 @@ jobs: outputs: source: ${{ steps.filter.outputs.source }} dependencies: ${{ steps.filter.outputs.dependencies }} + deny: ${{ steps.filter.outputs.deny }} internal_events: ${{ steps.filter.outputs.internal_events }} cue: ${{ steps.filter.outputs.cue }} component_docs: ${{ steps.filter.outputs.component_docs }} @@ -175,7 +182,7 @@ jobs: unit_mac-yml: ${{ steps.filter.outputs.unit_mac-yml }} unit_windows-yml: ${{ steps.filter.outputs.unit_windows-yml }} steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 id: filter @@ -207,6 +214,10 @@ jobs: - 'scripts/cross/**' - "vdev/**" - ".github/workflows/changes.yml" + deny: + - '**/Cargo.toml' + - 'Cargo.lock' + - ".github/workflows/deny.yml" cue: - 'website/cue/**' - "vdev/**" @@ -290,6 +301,7 @@ jobs: datadog-traces: ${{ steps.filter.outputs.datadog-traces }} dnstap: ${{ steps.filter.outputs.dnstap }} docker-logs: ${{ steps.filter.outputs.docker-logs }} + doris: ${{ steps.filter.outputs.doris }} elasticsearch: ${{ steps.filter.outputs.elasticsearch }} eventstoredb: ${{ steps.filter.outputs.eventstoredb }} fluent: ${{ steps.filter.outputs.fluent }} @@ -314,15 +326,17 @@ jobs: webhdfs: ${{ steps.filter.outputs.webhdfs }} any: ${{ steps.detect-changes.outputs.any }} steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: ./.github/actions/setup with: vdev: true + mold: false + cargo-cache: false # creates a yaml file that contains the filters for each integration, # extracted from the output of the `vdev int ci-paths` command, which - # sources the paths from the scripts/integration/.../test.yaml files + # sources the paths from the tests/integration/.../config/test.yaml files - name: Create filter rules for integrations run: vdev int ci-paths > int_test_filters.yaml @@ -353,6 +367,7 @@ jobs: "datadog-traces": ${{ steps.filter.outputs.datadog-traces }}, "dnstap": ${{ steps.filter.outputs.dnstap }}, "docker-logs": ${{ steps.filter.outputs.docker-logs }}, + "doris": ${{ steps.filter.outputs.doris }}, "elasticsearch": ${{ steps.filter.outputs.elasticsearch }}, "eventstoredb": ${{ steps.filter.outputs.eventstoredb }}, "fluent": ${{ steps.filter.outputs.fluent }}, @@ -386,7 +401,7 @@ jobs: echo "any=$any_changed" >> $GITHUB_OUTPUT - name: Upload JSON artifact - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: int_tests_changes path: int_tests_changes.json @@ -400,17 +415,20 @@ jobs: datadog-logs: ${{ steps.filter.outputs.datadog-logs }} datadog-metrics: ${{ steps.filter.outputs.datadog-metrics }} opentelemetry-logs: ${{ steps.filter.outputs.opentelemetry-logs }} + opentelemetry-metrics: ${{ steps.filter.outputs.opentelemetry-metrics }} any: ${{ steps.detect-changes.outputs.any }} steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: ./.github/actions/setup with: vdev: true + mold: false + cargo-cache: false # creates a yaml file that contains the filters for each test, # extracted from the output of the `vdev int ci-paths` command, which - # sources the paths from the scripts/e2e/.../test.yaml files + # sources the paths from the tests/e2e/.../test.yaml files - name: Create filter rules for e2e tests run: vdev e2e ci-paths > int_test_filters.yaml @@ -430,7 +448,8 @@ jobs: { "datadog-logs": ${{ steps.filter.outputs.datadog-logs }}, "datadog-metrics": ${{ steps.filter.outputs.datadog-metrics }}, - "opentelemetry-logs": ${{ steps.filter.outputs.opentelemetry-logs }} + "opentelemetry-logs": ${{ steps.filter.outputs.opentelemetry-logs }}, + "opentelemetry-metrics": ${{ steps.filter.outputs.opentelemetry-metrics }} } EOF ) @@ -442,7 +461,7 @@ jobs: echo "any=$any_changed" >> $GITHUB_OUTPUT - name: Upload JSON artifact - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: e2e_tests_changes path: e2e_tests_changes.json diff --git a/.github/workflows/ci-integration-review.yml b/.github/workflows/ci-integration-review.yml index 014f97b9547e3..d634a98391c13 100644 --- a/.github/workflows/ci-integration-review.yml +++ b/.github/workflows/ci-integration-review.yml @@ -95,6 +95,7 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 90 strategy: + fail-fast: false matrix: service: [ "amqp", "appsignal", "aws", "axiom", "azure", "clickhouse", "databend", "datadog-agent", @@ -104,7 +105,7 @@ jobs: "redis", "splunk", "webhdfs" ] steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: submodules: "recursive" ref: ${{ github.event.review.commit_id }} @@ -133,8 +134,14 @@ jobs: - build-test-runner runs-on: ubuntu-24.04-8core timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + service: [ + "datadog-logs", "datadog-metrics", "opentelemetry-logs", "opentelemetry-metrics" + ] steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: submodules: "recursive" ref: ${{ github.event.review.commit_id }} @@ -147,31 +154,16 @@ jobs: - run: bash scripts/environment/prepare.sh --modules=datadog-ci - - name: e2e-datadog-logs - if: ${{ startsWith(github.event.review.body, '/ci-run-e2e-datadog-logs') - || startsWith(github.event.review.body, '/ci-run-e2e-all') - || startsWith(github.event.review.body, '/ci-run-all') }} - uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 - with: - timeout_minutes: 35 - max_attempts: 3 - command: bash scripts/run-integration-test.sh e2e datadog-logs - - - name: datadog-e2e-metrics - if: ${{ startsWith(github.event.review.body, '/ci-run-e2e-datadog-metrics') - || startsWith(github.event.review.body, '/ci-run-e2e-all') - || startsWith(github.event.review.body, '/ci-run-all') }} + - name: E2E Tests - ${{ matrix.service }} + if: ${{ startsWith(github.event.review.body, '/ci-run-e2e-all') + || startsWith(github.event.review.body, '/ci-run-all') + || startsWith(github.event.review.body, format('/ci-run-e2e-{0}', matrix.service)) }} uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 with: timeout_minutes: 35 max_attempts: 3 - command: bash scripts/run-integration-test.sh e2e datadog-metrics + command: bash scripts/run-integration-test.sh e2e ${{ matrix.service }} - - name: e2e-opentelemetry-logs - if: ${{ startsWith(github.event.review.body, '/ci-run-e2e-opentelemetry-logs') - || startsWith(github.event.review.body, '/ci-run-e2e-all') - || startsWith(github.event.review.body, '/ci-run-all') }} - run: bash scripts/run-integration-test.sh e2e opentelemetry-logs update-pr-status: name: Signal result to PR diff --git a/.github/workflows/ci-review-trigger.yml b/.github/workflows/ci-review-trigger.yml index d85ffc9e5d7fa..d74974e42d3fb 100644 --- a/.github/workflows/ci-review-trigger.yml +++ b/.github/workflows/ci-review-trigger.yml @@ -29,7 +29,7 @@ on: types: [submitted] permissions: - statuses: write + contents: read env: DD_ENV: "ci" @@ -85,9 +85,10 @@ jobs: run: exit 1 cli: - needs: validate if: startsWith(github.event.review.body, '/ci-run-all') || contains(github.event.review.body, '/ci-run-cli') uses: ./.github/workflows/cli.yml + with: + ref: ${{ github.event.review.commit_id }} secrets: inherit make-test-behavior: @@ -95,6 +96,7 @@ jobs: if: startsWith(github.event.review.body, '/ci-run-all') || contains(github.event.review.body, '/ci-run-test-behavior') uses: ./.github/workflows/test-make-command.yml with: + ref: ${{ github.event.review.commit_id }} command: test-behavior job_name: make test-behavior secrets: inherit @@ -104,6 +106,7 @@ jobs: if: startsWith(github.event.review.body, '/ci-run-all') || contains(github.event.review.body, '/ci-run-check-examples') uses: ./.github/workflows/test-make-command.yml with: + ref: ${{ github.event.review.commit_id }} command: check-examples job_name: make check-examples secrets: inherit @@ -113,6 +116,7 @@ jobs: if: startsWith(github.event.review.body, '/ci-run-all') || contains(github.event.review.body, '/ci-run-test-docs') uses: ./.github/workflows/test-make-command.yml with: + ref: ${{ github.event.review.commit_id }} command: test-docs job_name: make test-docs secrets: inherit @@ -121,40 +125,54 @@ jobs: needs: validate if: startsWith(github.event.review.body, '/ci-run-all') || contains(github.event.review.body, '/ci-run-deny') uses: ./.github/workflows/deny.yml + with: + ref: ${{ github.event.review.commit_id }} secrets: inherit component-features: needs: validate if: startsWith(github.event.review.body, '/ci-run-all') || contains(github.event.review.body, '/ci-run-component-features') uses: ./.github/workflows/component_features.yml + with: + ref: ${{ github.event.review.commit_id }} secrets: inherit cross: needs: validate if: startsWith(github.event.review.body, '/ci-run-all') || contains(github.event.review.body, '/ci-run-cross') uses: ./.github/workflows/cross.yml + with: + ref: ${{ github.event.review.commit_id }} secrets: inherit unit-mac: needs: validate if: startsWith(github.event.review.body, '/ci-run-all') || contains(github.event.review.body, '/ci-run-unit-mac') uses: ./.github/workflows/unit_mac.yml + with: + ref: ${{ github.event.review.commit_id }} secrets: inherit unit-windows: needs: validate if: startsWith(github.event.review.body, '/ci-run-all') || contains(github.event.review.body, '/ci-run-unit-windows') uses: ./.github/workflows/unit_windows.yml + with: + ref: ${{ github.event.review.commit_id }} secrets: inherit environment: needs: validate if: startsWith(github.event.review.body, '/ci-run-all') || contains(github.event.review.body, '/ci-run-environment') uses: ./.github/workflows/environment.yml + with: + ref: ${{ github.event.review.commit_id }} secrets: inherit k8s: needs: validate if: startsWith(github.event.review.body, '/ci-run-all') || contains(github.event.review.body, '/ci-run-k8s') uses: ./.github/workflows/k8s_e2e.yml + with: + ref: ${{ github.event.review.commit_id }} secrets: inherit diff --git a/.github/workflows/cla.yml b/.github/workflows/cla.yml new file mode 100644 index 0000000000000..c2693a2d33a18 --- /dev/null +++ b/.github/workflows/cla.yml @@ -0,0 +1,51 @@ +name: "CLA Assistant" +on: + issue_comment: + types: [created] + pull_request_target: + types: [opened, closed, synchronize] + merge_group: + types: [checks_requested] + +permissions: + contents: read + pull-requests: write + id-token: write # Needed to federate tokens. + actions: write + +jobs: + CLAAssistant: + runs-on: ubuntu-latest + steps: + - name: CLA already verified on PR + if: github.event_name == 'merge_group' + run: echo "CLA verification not needed for merge queue - already checked on PR" + + - uses: DataDog/dd-octo-sts-action@acaa02eee7e3bb0839e4272dacb37b8f3b58ba80 # v1.0.3 + if: github.event_name != 'merge_group' + id: octo-sts + with: + scope: DataDog/cla-signatures + policy: self.write-signatures-vector + + - name: "CLA Assistant" + if: github.event_name != 'merge_group' && ((github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target') + uses: contributor-assistant/github-action@ca4a40a7d1004f18d9960b404b97e5f30a505a08 # v2.6.1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PERSONAL_ACCESS_TOKEN: ${{ steps.octo-sts.outputs.token }} + with: + path-to-signatures: 'cla.json' + path-to-document: 'https://gist.github.com/bits-bot/55bdc97a4fdad52d97feb4d6c3d1d618' # e.g. a CLA or a DCO document + branch: 'vector' + remote-repository-name: cla-signatures + remote-organization-name: DataDog + allowlist: step-security-bot + + # the followings are the optional inputs - If the optional inputs are not given, then default values will be taken + #create-file-commit-message: 'For example: Creating file for storing CLA Signatures' + #signed-commit-message: 'For example: $contributorName has signed the CLA in $owner/$repo#$pullRequestNo' + #custom-notsigned-prcomment: 'pull request comment with Introductory message to ask new contributors to sign' + #custom-pr-sign-comment: 'The signature to be committed in order to sign the CLA' + #custom-allsigned-prcomment: 'pull request comment when all contributors has signed, defaults to **CLA Assistant Lite bot** All Contributors have signed the CLA.' + #lock-pullrequest-aftermerge: false - if you don't want this bot to automatically lock the pull request after merging (default - true) diff --git a/.github/workflows/cleanup-ghcr-images.yml b/.github/workflows/cleanup-ghcr-images.yml index a2ab8ae35ede6..709190f9af108 100644 --- a/.github/workflows/cleanup-ghcr-images.yml +++ b/.github/workflows/cleanup-ghcr-images.yml @@ -13,6 +13,9 @@ on: - cron: '0 2 * * 0' workflow_dispatch: +permissions: + contents: read + jobs: cleanup: runs-on: ubuntu-latest diff --git a/.github/workflows/cli.yml b/.github/workflows/cli.yml index 80bb2d3002e0d..9c6728fb24dd4 100644 --- a/.github/workflows/cli.yml +++ b/.github/workflows/cli.yml @@ -2,61 +2,30 @@ name: CLI - Linux on: workflow_call: + inputs: + ref: + description: 'Git ref to checkout' + required: false + type: string permissions: - statuses: write + contents: read jobs: test-cli: runs-on: ubuntu-24.04 timeout-minutes: 30 - env: - CARGO_INCREMENTAL: 0 steps: - - name: (PR review) Set latest commit status as pending - if: ${{ github.event_name == 'pull_request_review' }} - uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - sha: ${{ github.event.review.commit_id }} - token: ${{ secrets.GITHUB_TOKEN }} - context: CLI - Linux - status: pending - - - name: (PR review) Checkout review SHA - if: ${{ github.event_name == 'pull_request_review' }} - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: - ref: ${{ github.event.review.commit_id }} - - - name: Checkout branch - if: ${{ github.event_name != 'pull_request_review' }} - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - - name: Cache Cargo registry + index - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + ref: ${{ inputs.ref }} + - uses: ./.github/actions/setup with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-cargo- - - - run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh - - run: bash scripts/environment/prepare.sh --modules=cargo-nextest,datadog-ci - - run: echo "::add-matcher::.github/matchers/rust.json" + rust: true + cargo-nextest: true + protoc: true + datadog-ci: true - run: make test-cli - name: Upload test results run: scripts/upload-test-results.sh if: always() - - - name: (PR review) Set latest commit status as ${{ job.status }} - uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 - if: always() && github.event_name == 'pull_request_review' - with: - sha: ${{ github.event.review.commit_id }} - token: ${{ secrets.GITHUB_TOKEN }} - context: CLI - Linux - status: ${{ job.status }} diff --git a/.github/workflows/compilation-timings.yml b/.github/workflows/compilation-timings.yml index 2b1c95d3a685e..dec323e8185fc 100644 --- a/.github/workflows/compilation-timings.yml +++ b/.github/workflows/compilation-timings.yml @@ -16,7 +16,7 @@ jobs: runs-on: ubuntu-24.04-8core steps: - uses: colpal/actions-clean@36e6ca1abd35efe61cb60f912bd7837f67887c8a # v1.1.1 - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh - run: bash scripts/environment/prepare.sh --modules=rustup - run: cargo clean @@ -32,7 +32,7 @@ jobs: PROFILE: debug steps: - uses: colpal/actions-clean@36e6ca1abd35efe61cb60f912bd7837f67887c8a # v1.1.1 - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh - run: bash scripts/environment/prepare.sh --modules=rustup - run: cargo clean @@ -43,7 +43,7 @@ jobs: runs-on: ubuntu-24.04-8core steps: - uses: colpal/actions-clean@36e6ca1abd35efe61cb60f912bd7837f67887c8a # v1.1.1 - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh - run: bash scripts/environment/prepare.sh --modules=rustup - run: cargo clean @@ -54,7 +54,7 @@ jobs: runs-on: ubuntu-24.04-8core steps: - uses: colpal/actions-clean@36e6ca1abd35efe61cb60f912bd7837f67887c8a # v1.1.1 - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh - run: bash scripts/environment/prepare.sh --modules=rustup - run: cargo clean @@ -67,7 +67,7 @@ jobs: runs-on: ubuntu-24.04-8core steps: - uses: colpal/actions-clean@36e6ca1abd35efe61cb60f912bd7837f67887c8a # v1.1.1 - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh - run: bash scripts/environment/prepare.sh --modules=rustup - run: cargo clean diff --git a/.github/workflows/component_features.yml b/.github/workflows/component_features.yml index 62d101d9bac4e..78abccaa055db 100644 --- a/.github/workflows/component_features.yml +++ b/.github/workflows/component_features.yml @@ -11,49 +11,39 @@ name: Component Features - Linux on: workflow_call: + inputs: + ref: + description: 'Git ref to checkout' + required: false + type: string workflow_dispatch: + inputs: + ref: + description: 'Git ref to checkout' + required: false + type: string schedule: - cron: '0 10 * * 1' # 10:00 UTC Monday (6 AM EST NYC) permissions: - statuses: write + contents: read jobs: check-component-features: # use free tier on schedule and 8 core to expedite results on demand invocation runs-on: ${{ github.event_name == 'schedule' && 'ubuntu-24.04' || 'ubuntu-24.04-8core' }} timeout-minutes: 180 # Usually takes 2h but this prevents it from hanging for an indefinite time - if: github.event_name == 'pull_request_review' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' steps: - - name: (PR review) Set latest commit status as pending - if: github.event_name == 'pull_request_review' - uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 + - name: Checkout branch + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - sha: ${{ github.event.review.commit_id }} - token: ${{ secrets.GITHUB_TOKEN }} - context: Component Features - Linux - status: pending + ref: ${{ inputs.ref }} - - name: (PR review) Checkout PR branch - if: github.event_name == 'pull_request_review' - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: ./.github/actions/setup with: - ref: ${{ github.event.review.commit_id }} - - - name: Checkout branch - if: github.event_name != 'pull_request_review' - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + rust: true + cargo-nextest: true + protoc: true + libsasl2: true - - run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh - - run: bash scripts/environment/prepare.sh --modules=rustup - - run: echo "::add-matcher::.github/matchers/rust.json" - run: make check-component-features - - - name: (PR review) Set latest commit status as ${{ job.status }} - if: always() && github.event_name == 'pull_request_review' - uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 - with: - sha: ${{ github.event.review.commit_id }} - token: ${{ secrets.GITHUB_TOKEN }} - context: Component Features - Linux - status: ${{ job.status }} diff --git a/.github/workflows/cross.yml b/.github/workflows/cross.yml index ce5797ca367f1..64da74d3f4877 100644 --- a/.github/workflows/cross.yml +++ b/.github/workflows/cross.yml @@ -2,9 +2,14 @@ name: Cross on: workflow_call: + inputs: + ref: + description: 'Git ref to checkout' + required: false + type: string permissions: - statuses: write + contents: read jobs: cross-linux: @@ -25,26 +30,12 @@ jobs: - arm-unknown-linux-gnueabi - arm-unknown-linux-musleabi steps: - - name: (PR review) Set latest commit status as pending - if: ${{ github.event_name == 'pull_request_review' }} - uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 - with: - sha: ${{ github.event.review.commit_id }} - token: ${{ secrets.GITHUB_TOKEN }} - context: Cross - status: pending - - - name: (PR review) Checkout PR branch - if: ${{ github.event_name == 'pull_request_review' }} - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: - ref: ${{ github.event.review.commit_id }} - - name: Checkout branch - if: ${{ github.event_name != 'pull_request_review' }} - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ inputs.ref }} - - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + - uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 name: Cache Cargo registry + index with: path: | @@ -64,31 +55,7 @@ jobs: # aarch64 and musl in particular are notoriously hard to link. # While it may be tempting to slot a `check` in here for quickness, please don't. - run: make cross-build-${{ matrix.target }} - - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: "vector-debug-${{ matrix.target }}" path: "./target/${{ matrix.target }}/debug/vector" - - - name: (PR review) Set latest commit status as failed - uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 - if: failure() && github.event_name == 'pull_request_review' - with: - sha: ${{ steps.comment-branch.outputs.head_sha }} - token: ${{ secrets.GITHUB_TOKEN }} - context: Cross - status: 'failure' - - update-pr-status: - name: (PR review) Signal result to PR - runs-on: ubuntu-24.04 - timeout-minutes: 5 - needs: cross-linux - if: needs.cross-linux.result == 'success' && github.event_name == 'pull_request_review' - steps: - - name: (PR review) Submit PR result as success - uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 - with: - sha: ${{ github.event.review.commit_id }} - token: ${{ secrets.GITHUB_TOKEN }} - context: Cross - status: 'success' diff --git a/.github/workflows/deny.yml b/.github/workflows/deny.yml index 35a66c2583444..2a9696cc3f76d 100644 --- a/.github/workflows/deny.yml +++ b/.github/workflows/deny.yml @@ -6,68 +6,60 @@ # - scheduled UTC midnight # - on PR review (see comment-trigger.yml) # - on demand from github actions UI +# - on pull requests when Cargo.toml or Cargo.lock files change name: Deny - Linux on: workflow_call: + inputs: + ref: + description: 'Git ref to checkout' + required: false + type: string + workflow_dispatch: + inputs: + ref: + description: 'Git ref to checkout' + required: false + type: string + + pull_request: schedule: # Same schedule as nightly.yml - cron: "0 5 * * 2-6" # Runs at 5:00 AM UTC, Tuesday through Saturday +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + permissions: - statuses: write + contents: read jobs: + changes: + if: ${{ github.event_name == 'pull_request' }} + uses: ./.github/workflows/changes.yml + secrets: inherit + test-deny: runs-on: ubuntu-24.04 timeout-minutes: 30 + if: ${{ always() && (github.event_name != 'pull_request' || needs.changes.outputs.deny == 'true') }} + needs: [changes] env: CARGO_INCREMENTAL: 0 steps: - - name: (PR review) Set latest commit status as pending - if: ${{ github.event_name == 'pull_request_review' }} - uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 - with: - sha: ${{ github.event.review.commit_id }} - token: ${{ secrets.GITHUB_TOKEN }} - context: Deny - Linux - status: pending - - - name: (PR review) Checkout PR branch - if: ${{ github.event_name == 'pull_request_review' }} - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: - ref: ${{ github.event.review.commit_id }} - - name: Checkout branch - if: ${{ github.event_name != 'pull_request_review' }} - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ inputs.ref }} - - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 - name: Cache Cargo registry + index + - uses: ./.github/actions/setup with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-cargo- + mold: false + cargo-deny: true - - run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh - - run: bash scripts/environment/prepare.sh --modules=cargo-deny - - run: echo "::add-matcher::.github/matchers/rust.json" - name: Check cargo deny advisories/licenses run: make check-deny - - - name: (PR review) Set latest commit status as ${{ job.status }} - uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 - if: always() && github.event_name == 'pull_request_review' - with: - sha: ${{ github.event.review.commit_id }} - token: ${{ secrets.GITHUB_TOKEN }} - context: Deny - Linux - status: ${{ job.status }} diff --git a/.github/workflows/environment.yml b/.github/workflows/environment.yml index 3f1ac09b8e78e..129e4c6612d9b 100644 --- a/.github/workflows/environment.yml +++ b/.github/workflows/environment.yml @@ -2,12 +2,23 @@ name: Environment Suite on: workflow_call: + inputs: + ref: + description: 'Git ref to checkout' + required: false + type: string + workflow_dispatch: inputs: push: description: "Push image when manually triggered" type: boolean default: false + ref: + description: 'Git ref to checkout' + required: false + type: string + schedule: - cron: '0 6 * * 1' # Every Monday at 06:00 UTC @@ -17,45 +28,31 @@ env: SHOULD_PUBLISH: ${{ (github.event_name != 'workflow_dispatch' && github.ref == 'refs/heads/master') || (github.event_name == 'workflow_dispatch' && github.event.inputs.push == 'true') }} permissions: - statuses: write + contents: read jobs: publish-new-environment: runs-on: ubuntu-24.04 timeout-minutes: 30 steps: - - name: (PR review) Set latest commit status as pending - if: ${{ github.event_name == 'pull_request_review' }} - uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 - with: - sha: ${{ github.event.review.commit_id }} - token: ${{ secrets.GITHUB_TOKEN }} - context: Environment Suite - status: pending - - - name: (PR review) Checkout PR branch - if: ${{ github.event_name == 'pull_request_review' }} - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: - ref: ${{ github.event.review.commit_id }} - - name: Checkout branch - if: ${{ github.event_name != 'pull_request_review' }} - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ inputs.ref }} - name: Set up QEMU - uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0 + uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3.7.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 - name: Login to DockerHub - uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 if: env.SHOULD_PUBLISH == 'true' with: username: ${{ secrets.CI_DOCKER_USERNAME }} password: ${{ secrets.CI_DOCKER_PASSWORD }} - name: Extract metadata (tags, labels) for Docker id: meta - uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5.8.0 + uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5.10.0 with: images: timberio/vector-dev flavor: | @@ -99,12 +96,3 @@ jobs: push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - - - name: (PR review) Set latest commit status as ${{ job.status }} - uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 - if: always() && github.event_name == 'pull_request_review' - with: - sha: ${{ github.event.review.commit_id }} - token: ${{ secrets.GITHUB_TOKEN }} - context: Environment Suite - status: ${{ job.status }} diff --git a/.github/workflows/gardener_open_issue.yml b/.github/workflows/gardener_open_issue.yml index c8ba082d3ed63..87389577949ac 100644 --- a/.github/workflows/gardener_open_issue.yml +++ b/.github/workflows/gardener_open_issue.yml @@ -6,6 +6,9 @@ on: types: - opened +permissions: + contents: read + jobs: add-to-project: name: Add issue to Gardener project board diff --git a/.github/workflows/gardener_remove_waiting_author.yml b/.github/workflows/gardener_remove_waiting_author.yml index ce80b9c0d07fd..b431395b743ef 100644 --- a/.github/workflows/gardener_remove_waiting_author.yml +++ b/.github/workflows/gardener_remove_waiting_author.yml @@ -16,7 +16,7 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 5 steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0 # v1.3.0 with: labels: "meta: awaiting author" diff --git a/.github/workflows/install-sh.yml b/.github/workflows/install-sh.yml index 172204439191b..69777ead18d83 100644 --- a/.github/workflows/install-sh.yml +++ b/.github/workflows/install-sh.yml @@ -2,49 +2,31 @@ name: Update install.sh Suite on: workflow_call: + inputs: + ref: + description: 'Git ref to checkout' + required: false + type: string workflow_dispatch: + inputs: + ref: + description: 'Git ref to checkout' + required: false + type: string permissions: - statuses: write + contents: read jobs: test-install: runs-on: ubuntu-24.04 timeout-minutes: 5 steps: - - name: (PR comment) Get PR branch - if: ${{ github.event_name == 'issue_comment' }} - uses: xt0rted/pull-request-comment-branch@e8b8daa837e8ea7331c0003c9c316a64c6d8b0b1 # v3.0.0 - id: comment-branch - - - name: (PR comment) Set latest commit status as pending - if: ${{ github.event_name == 'issue_comment' }} - uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 - with: - sha: ${{ steps.comment-branch.outputs.head_sha }} - token: ${{ secrets.GITHUB_TOKEN }} - context: Update install.sh Suite - status: pending - - - name: (PR comment) Checkout PR branch - if: ${{ github.event_name == 'issue_comment' }} - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: - ref: ${{ steps.comment-branch.outputs.head_ref }} - - name: Checkout branch - if: ${{ github.event_name != 'issue_comment' }} - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ inputs.ref }} - run: sudo apt-get install --yes bc - run: bash distribution/install.sh -- -y - run: ~/.vector/bin/vector --version - - - name: (PR comment) Set latest commit status as ${{ job.status }} - if: github.event_name == 'issue_comment' - uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 - with: - sha: ${{ steps.comment-branch.outputs.head_sha }} - token: ${{ secrets.GITHUB_TOKEN }} - context: Update install.sh Suite - status: ${{ job.status }} diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 257a5ec386831..838c922e63fae 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -39,13 +39,13 @@ jobs: - name: (PR comment) Checkout PR branch if: ${{ github.event_name == 'issue_comment' }} - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ steps.comment-branch.outputs.head_ref }} - name: Checkout branch if: ${{ github.event_name != 'issue_comment' }} - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - run: bash scripts/environment/prepare.sh --modules=rustup,datadog-ci diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 2dc938aab9962..94e8a9466c723 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -46,15 +46,15 @@ jobs: build-test-runner: needs: changes - if: | - ${{ - github.event_name == 'workflow_dispatch' || - (github.event_name == 'merge_group' && - (needs.changes.outputs.dependencies == 'true' || - needs.changes.outputs.integration-yml == 'true' || - needs.changes.outputs.int-tests-any == 'true' || - needs.changes.outputs.e2e-tests-any == 'true')) - }} + if: ${{ + github.event_name == 'workflow_dispatch' || + (github.event_name == 'merge_group' && + (needs.changes.outputs.dependencies == 'true' || + needs.changes.outputs.integration-yml == 'true' || + needs.changes.outputs.int-tests-any == 'true' || + needs.changes.outputs.e2e-tests-any == 'true')) + }} + uses: ./.github/workflows/build-test-runner.yml with: commit_sha: ${{ github.sha }} @@ -65,26 +65,26 @@ jobs: - changes - build-test-runner - if: ${{ !failure() && !cancelled() && (github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch') }} + if: ${{ !failure() && !cancelled() && needs.build-test-runner.result == 'success' && (github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch') }} strategy: matrix: # TODO: Add "splunk" back once https://github.com/vectordotdev/vector/issues/23474 is fixed. # If you modify this list, please also update the `int_tests` job in changes.yml. service: [ "amqp", "appsignal", "axiom", "aws", "azure", "clickhouse", "databend", "datadog-agent", - "datadog-logs", "datadog-metrics", "datadog-traces", "dnstap", "docker-logs", "elasticsearch", + "datadog-logs", "datadog-metrics", "datadog-traces", "dnstap", "docker-logs", "doris", "elasticsearch", "eventstoredb", "fluent", "gcp", "greptimedb", "http-client", "influxdb", "kafka", "logstash", "loki", "mqtt", "mongodb", "nats", "nginx", "opentelemetry", "postgres", "prometheus", "pulsar", "redis", "webhdfs" ] timeout-minutes: 90 steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: submodules: "recursive" - name: Download JSON artifact from changes.yml - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 if: github.event_name == 'merge_group' with: name: int_tests_changes @@ -133,19 +133,19 @@ jobs: needs: - changes - build-test-runner - if: ${{ !failure() && !cancelled() && (github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch') }} + if: ${{ !failure() && !cancelled() && needs.build-test-runner.result == 'success' && (github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch') }} strategy: matrix: - service: [ "datadog-logs", "datadog-metrics", "opentelemetry-logs" ] + service: [ "datadog-logs", "datadog-metrics", "opentelemetry-logs", "opentelemetry-metrics" ] timeout-minutes: 90 steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: submodules: "recursive" - name: Download JSON artifact from changes.yml - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 if: github.event_name == 'merge_group' with: name: e2e_tests_changes diff --git a/.github/workflows/k8s_e2e.yml b/.github/workflows/k8s_e2e.yml index f3b0959513980..fc36a0ed89a14 100644 --- a/.github/workflows/k8s_e2e.yml +++ b/.github/workflows/k8s_e2e.yml @@ -15,11 +15,21 @@ name: K8S E2E Suite permissions: - statuses: write + contents: read on: workflow_dispatch: + inputs: + ref: + description: 'Git ref to checkout' + required: false + type: string workflow_call: + inputs: + ref: + description: 'Git ref to checkout' + required: false + type: string pull_request: merge_group: types: [checks_requested] @@ -43,7 +53,6 @@ env: RUST_BACKTRACE: full VECTOR_LOG: vector=debug VERBOSE: true - DISABLE_MOLD: true CI: true PROFILE: debug @@ -66,57 +75,27 @@ jobs: CARGO_PROFILE_RELEASE_OPT_LEVEL: 0 CARGO_PROFILE_RELEASE_CODEGEN_UNITS: 256 CARGO_INCREMENTAL: 0 + DISABLE_MOLD: true steps: - - name: (PR review) Set latest commit status as pending - if: ${{ github.event_name == 'pull_request_review' }} - uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 - with: - sha: ${{ github.event.review.commit_id }} - token: ${{ secrets.GITHUB_TOKEN }} - status: pending - - - name: (PR review) Checkout PR branch - if: ${{ github.event_name == 'pull_request_review' }} - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: - ref: ${{ github.event.review.commit_id }} - - name: Checkout branch - if: ${{ github.event_name != 'pull_request_review' }} - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - path: | - ~/.cargo/registry - ~/.cargo/git - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - + ref: ${{ inputs.ref }} - run: sudo -E bash scripts/ci-free-disk-space.sh - - run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh - - run: bash scripts/environment/prepare.sh --modules=rustup,cross - - run: rustup target add x86_64-unknown-linux-gnu - - run: echo "::add-matcher::.github/matchers/rust.json" - # TODO: Use the setup action in this workflow. - - name: Install vdev - uses: ./.github/actions/install-vdev + - uses: ./.github/actions/setup + with: + rust: true + cross: true + mold: false - run: VECTOR_VERSION="$(vdev version)" make package-deb-x86_64-unknown-linux-gnu - - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: e2e-test-deb-package path: target/artifacts/* - - name: (PR review) Set latest commit status as 'failure' - uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 - if: failure() && github.event_name == 'pull_request_review' - with: - sha: ${{ github.event.review.commit_id }} - token: ${{ secrets.GITHUB_TOKEN }} - status: 'failure' - # GitHub Actions don't support `matrix` at the job-level `if:` condition. # We apply this workaround - compute `matrix` in a preceding job, and assign # it's value dynamically at the actual test job. @@ -191,22 +170,12 @@ jobs: matrix: ${{ fromJson(needs.compute-k8s-test-plan.outputs.matrix) }} fail-fast: false steps: - - name: (PR review) Get PR branch - if: ${{ github.event_name == 'pull_request_review' }} - uses: xt0rted/pull-request-comment-branch@e8b8daa837e8ea7331c0003c9c316a64c6d8b0b1 # v3.0.0 - id: comment-branch - - - name: (PR review) Checkout PR branch - if: ${{ github.event_name == 'pull_request_review' }} - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: - ref: ${{ steps.comment-branch.outputs.head_ref }} - - name: Checkout branch - if: ${{ github.event_name != 'pull_request_review' }} - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ inputs.ref }} - - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: e2e-test-deb-package path: target/artifacts @@ -230,14 +199,6 @@ jobs: max_attempts: 3 command: make test-e2e-kubernetes - - name: (PR review) Set latest commit status as failure - uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 - if: failure() && github.event_name == 'pull_request_review' - with: - sha: ${{ github.event.review.commit_id }} - token: ${{ secrets.GITHUB_TOKEN }} - status: 'failure' - final-result: name: K8s E2E Suite runs-on: ubuntu-24.04 @@ -251,19 +212,6 @@ jobs: env: FAILED: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }} steps: - - name: (PR review) Get PR branch - if: github.event_name == 'pull_request_review' && env.FAILED != 'true' - uses: xt0rted/pull-request-comment-branch@e8b8daa837e8ea7331c0003c9c316a64c6d8b0b1 # v3.0.0 - id: comment-branch - - - name: (PR review) Submit PR result as success - if: github.event_name == 'pull_request_review' && env.FAILED != 'true' - uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 - with: - sha: ${{ github.event.review.commit_id }} - token: ${{ secrets.GITHUB_TOKEN }} - status: 'success' - - name: Check all jobs status run: | if [[ "${{ env.FAILED }}" == "true" ]]; then diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 7fc80a2846cfb..e94a4bb384ec6 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -2,6 +2,9 @@ name: "Pull Request Labeler" on: pull_request_target: +permissions: + contents: read + jobs: label: runs-on: ubuntu-24.04 diff --git a/.github/workflows/master_merge_queue.yml b/.github/workflows/master_merge_queue.yml index b73039c209cb4..da951506d1344 100644 --- a/.github/workflows/master_merge_queue.yml +++ b/.github/workflows/master_merge_queue.yml @@ -20,7 +20,7 @@ on: types: [checks_requested] permissions: - statuses: write + contents: read concurrency: # `github.ref` is unique for MQ runs and PRs @@ -112,6 +112,8 @@ jobs: secrets: inherit master-merge-queue-check: + permissions: + statuses: write name: Master Merge Queue Suite # Always run this so that pull_request triggers are marked as success. if: always() diff --git a/.github/workflows/msrv.yml b/.github/workflows/msrv.yml index 16def3ba54bd9..5bc98ced18f2d 100644 --- a/.github/workflows/msrv.yml +++ b/.github/workflows/msrv.yml @@ -14,14 +14,21 @@ env: CI: true PROFILE: debug +permissions: + contents: read + jobs: check-msrv: runs-on: ubuntu-24.04 timeout-minutes: 45 steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ inputs.checkout_ref }} - - run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh - - run: bash scripts/environment/prepare.sh --modules=cargo-msrv + - uses: ./.github/actions/setup + with: + rust: true + protoc: true + libsasl2: true + cargo-msrv: true - run: cargo msrv verify diff --git a/.github/workflows/preview_site_trigger.yml b/.github/workflows/preview_site_trigger.yml index 0716a769a685b..b1e1838b5c49f 100644 --- a/.github/workflows/preview_site_trigger.yml +++ b/.github/workflows/preview_site_trigger.yml @@ -12,8 +12,9 @@ jobs: # Validate branch name - name: Validate branch name and set output id: validate + env: + BRANCH: ${{ github.head_ref }} run: | - BRANCH="${{ github.head_ref }}" if [[ ! "$BRANCH" =~ ^[a-zA-Z0-9_-]+$ ]]; then echo "valid=false" >> $GITHUB_OUTPUT else @@ -45,7 +46,7 @@ jobs: # Upload the artifact using latest version (only if branch is valid) - name: Upload PR information artifact if: steps.validate.outputs.valid == 'true' - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: pr path: pr/ diff --git a/.github/workflows/protobuf.yml b/.github/workflows/protobuf.yml index b261f18b43631..06e8411a81541 100644 --- a/.github/workflows/protobuf.yml +++ b/.github/workflows/protobuf.yml @@ -22,7 +22,7 @@ jobs: timeout-minutes: 5 steps: # Run `git checkout` - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 # Install the `buf` CLI - uses: bufbuild/buf-setup-action@a47c93e0b1648d5651a065437926377d060baa99 # v1.50.0 with: diff --git a/.github/workflows/publish-homebrew.yml b/.github/workflows/publish-homebrew.yml index 49a8269a7090f..117fe087c8441 100644 --- a/.github/workflows/publish-homebrew.yml +++ b/.github/workflows/publish-homebrew.yml @@ -17,6 +17,9 @@ on: required: true type: string +permissions: + contents: read + jobs: publish-homebrew: runs-on: ubuntu-24.04 @@ -25,7 +28,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.HOMEBREW_PAT }} steps: - name: Checkout Vector - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ inputs.git_ref || github.ref_name }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 280b081cec1f8..3e33c58119644 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -37,7 +37,7 @@ jobs: vector_release_channel: ${{ steps.generate-publish-metadata.outputs.vector_release_channel }} steps: - name: Checkout Vector - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ inputs.git_ref }} - name: Generate publish metadata @@ -55,7 +55,7 @@ jobs: CHANNEL: ${{ needs.generate-publish-metadata.outputs.vector_release_channel }} steps: - name: Checkout Vector - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ inputs.git_ref }} - name: Bootstrap runner environment (Ubuntu-specific) @@ -65,7 +65,7 @@ jobs: - name: Build Vector run: make package-x86_64-unknown-linux-musl-all - name: Stage package artifacts for publish - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-unknown-linux-musl path: target/artifacts/vector* @@ -81,7 +81,7 @@ jobs: CHANNEL: ${{ needs.generate-publish-metadata.outputs.vector_release_channel }} steps: - name: Checkout Vector - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ inputs.git_ref }} - name: Bootstrap runner environment (Ubuntu-specific) @@ -91,7 +91,7 @@ jobs: - name: Build Vector run: make package-x86_64-unknown-linux-gnu-all - name: Stage package artifacts for publish - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-unknown-linux-gnu path: target/artifacts/vector* @@ -107,7 +107,7 @@ jobs: CHANNEL: ${{ needs.generate-publish-metadata.outputs.vector_release_channel }} steps: - name: Checkout Vector - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ inputs.git_ref }} - name: Bootstrap runner environment (Ubuntu-specific) @@ -119,7 +119,7 @@ jobs: DOCKER_PRIVILEGED: "true" run: make package-aarch64-unknown-linux-musl-all - name: Stage package artifacts for publish - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: vector-${{ env.VECTOR_VERSION }}-aarch64-unknown-linux-musl path: target/artifacts/vector* @@ -135,7 +135,7 @@ jobs: CHANNEL: ${{ needs.generate-publish-metadata.outputs.vector_release_channel }} steps: - name: Checkout Vector - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ inputs.git_ref }} - name: Bootstrap runner environment (Ubuntu-specific) @@ -147,7 +147,7 @@ jobs: DOCKER_PRIVILEGED: "true" run: make package-aarch64-unknown-linux-gnu-all - name: Stage package artifacts for publish - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: vector-${{ env.VECTOR_VERSION }}-aarch64-unknown-linux-gnu path: target/artifacts/vector* @@ -163,7 +163,7 @@ jobs: CHANNEL: ${{ needs.generate-publish-metadata.outputs.vector_release_channel }} steps: - name: Checkout Vector - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ inputs.git_ref }} - name: Bootstrap runner environment (Ubuntu-specific) @@ -175,7 +175,7 @@ jobs: DOCKER_PRIVILEGED: "true" run: make package-armv7-unknown-linux-gnueabihf-all - name: Stage package artifacts for publish - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: vector-${{ env.VECTOR_VERSION }}-armv7-unknown-linux-gnueabihf path: target/artifacts/vector* @@ -191,7 +191,7 @@ jobs: CHANNEL: ${{ needs.generate-publish-metadata.outputs.vector_release_channel }} steps: - name: Checkout Vector - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ inputs.git_ref }} - name: Bootstrap runner environment (Ubuntu-specific) @@ -203,7 +203,7 @@ jobs: DOCKER_PRIVILEGED: "true" run: make package-armv7-unknown-linux-musleabihf - name: Stage package artifacts for publish - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: vector-${{ env.VECTOR_VERSION }}-armv7-unknown-linux-musleabihf path: target/artifacts/vector* @@ -219,7 +219,7 @@ jobs: CHANNEL: ${{ needs.generate-publish-metadata.outputs.vector_release_channel }} steps: - name: Checkout Vector - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ inputs.git_ref }} - name: Bootstrap runner environment (Ubuntu-specific) @@ -231,7 +231,7 @@ jobs: DOCKER_PRIVILEGED: "true" run: make package-arm-unknown-linux-gnueabi-all - name: Stage package artifacts for publish - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: vector-${{ env.VECTOR_VERSION }}-arm-unknown-linux-gnueabi path: target/artifacts/vector* @@ -247,7 +247,7 @@ jobs: CHANNEL: ${{ needs.generate-publish-metadata.outputs.vector_release_channel }} steps: - name: Checkout Vector - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ inputs.git_ref }} - name: Bootstrap runner environment (Ubuntu-specific) @@ -259,7 +259,7 @@ jobs: DOCKER_PRIVILEGED: "true" run: make package-arm-unknown-linux-musleabi - name: Stage package artifacts for publish - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: vector-${{ env.VECTOR_VERSION }}-arm-unknown-linux-musleabi path: target/artifacts/vector* @@ -290,7 +290,7 @@ jobs: exit 1 fi - name: Checkout Vector - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ inputs.git_ref }} - name: Bootstrap runner environment (macOS-specific) @@ -305,7 +305,7 @@ jobs: export PATH="$HOME/.cargo/bin:$PATH" make package - name: Stage package artifacts for publish - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: vector-${{ env.VECTOR_VERSION }}-${{ matrix.architecture }}-apple-darwin path: target/artifacts/vector* @@ -324,7 +324,7 @@ jobs: RELEASE_BUILDER: "true" steps: - name: Checkout Vector - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ inputs.git_ref }} - name: Bootstrap runner environment (Windows-specific) @@ -353,7 +353,7 @@ jobs: export PATH="/c/wix:$PATH" ./scripts/package-msi.sh - name: Stage package artifacts for publish - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-pc-windows-msvc path: target/artifacts/vector* @@ -394,11 +394,11 @@ jobs: - name: Fix Git safe directories issue when in containers (actions/checkout#760) run: git config --global --add safe.directory /__w/vector/vector - name: Checkout Vector - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ inputs.git_ref }} - name: Download staged package artifacts (x86_64-unknown-linux-gnu) - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-unknown-linux-gnu path: target/artifacts @@ -445,11 +445,11 @@ jobs: - name: Fix Git safe directories issue when in containers (actions/checkout#760) run: git config --global --add safe.directory /__w/vector/vector - name: Checkout Vector - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ inputs.git_ref }} - name: Download staged package artifacts (x86_64-unknown-linux-gnu) - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-unknown-linux-gnu path: target/artifacts @@ -474,11 +474,11 @@ jobs: runner: macos-14 steps: - name: Checkout Vector - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ inputs.git_ref }} - name: Download staged package artifacts (${{ matrix.target }}) - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: vector-${{ env.VECTOR_VERSION }}-${{ matrix.target }} path: target/artifacts @@ -508,67 +508,67 @@ jobs: CHANNEL: ${{ needs.generate-publish-metadata.outputs.vector_release_channel }} steps: - name: Checkout Vector - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ inputs.git_ref }} - name: Login to DockerHub - uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: username: ${{ secrets.CI_DOCKER_USERNAME }} password: ${{ secrets.CI_DOCKER_PASSWORD }} - name: Login to GitHub Container Registry - uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Set up QEMU - uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0 + uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3.7.0 with: platforms: all - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 with: version: latest install: true - name: Download staged package artifacts (aarch64-unknown-linux-gnu) - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: vector-${{ env.VECTOR_VERSION }}-aarch64-unknown-linux-gnu path: target/artifacts - name: Download staged package artifacts (aarch64-unknown-linux-musl) - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: vector-${{ env.VECTOR_VERSION }}-aarch64-unknown-linux-musl path: target/artifacts - name: Download staged package artifacts (x86_64-unknown-linux-gnu) - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-unknown-linux-gnu path: target/artifacts - name: Download staged package artifacts (x86_64-unknown-linux-musl) - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-unknown-linux-musl path: target/artifacts - name: Download staged package artifacts (armv7-unknown-linux-gnueabihf) - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: vector-${{ env.VECTOR_VERSION }}-armv7-unknown-linux-gnueabihf path: target/artifacts - name: Download staged package artifacts (armv7-unknown-linux-musleabihf) - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: vector-${{ env.VECTOR_VERSION }}-armv7-unknown-linux-musleabihf path: target/artifacts - name: Download staged package artifacts (arm-unknown-linux-gnueabi) - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: vector-${{ env.VECTOR_VERSION }}-arm-unknown-linux-gnueabi path: target/artifacts - name: Download staged package artifacts (arm-unknown-linux-musleabi) - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: vector-${{ env.VECTOR_VERSION }}-arm-unknown-linux-musleabi path: target/artifacts @@ -607,56 +607,56 @@ jobs: CHANNEL: ${{ needs.generate-publish-metadata.outputs.vector_release_channel }} steps: - name: Checkout Vector - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ inputs.git_ref }} - name: Download staged package artifacts (aarch64-unknown-linux-gnu) - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: vector-${{ env.VECTOR_VERSION }}-aarch64-unknown-linux-gnu path: target/artifacts - name: Download staged package artifacts (aarch64-unknown-linux-musl) - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: vector-${{ env.VECTOR_VERSION }}-aarch64-unknown-linux-musl path: target/artifacts - name: Download staged package artifacts (x86_64-unknown-linux-gnu) - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-unknown-linux-gnu path: target/artifacts - name: Download staged package artifacts (x86_64-unknown-linux-musl) - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-unknown-linux-musl path: target/artifacts - name: Download staged package artifacts (arm64-apple-darwin) - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: vector-${{ env.VECTOR_VERSION }}-arm64-apple-darwin path: target/artifacts - name: Download staged package artifacts (x86_64-pc-windows-msvc) - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-pc-windows-msvc path: target/artifacts - name: Download staged package artifacts (armv7-unknown-linux-gnueabihf) - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: vector-${{ env.VECTOR_VERSION }}-armv7-unknown-linux-gnueabihf path: target/artifacts - name: Download staged package artifacts (armv7-unknown-linux-musleabihf) - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: vector-${{ env.VECTOR_VERSION }}-armv7-unknown-linux-musleabihf path: target/artifacts - name: Download staged package artifacts (arm-unknown-linux-gnueabi) - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: vector-${{ env.VECTOR_VERSION }}-arm-unknown-linux-gnueabi path: target/artifacts - name: Download staged package artifacts (arm-unknown-linux-musleabi) - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: vector-${{ env.VECTOR_VERSION }}-arm-unknown-linux-musleabi path: target/artifacts @@ -692,61 +692,61 @@ jobs: VECTOR_VERSION: ${{ needs.generate-publish-metadata.outputs.vector_version }} steps: - name: Checkout Vector - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ inputs.git_ref }} - name: Download staged package artifacts (aarch64-unknown-linux-gnu) - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: vector-${{ env.VECTOR_VERSION }}-aarch64-unknown-linux-gnu path: target/artifacts - name: Download staged package artifacts (aarch64-unknown-linux-musl) - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: vector-${{ env.VECTOR_VERSION }}-aarch64-unknown-linux-musl path: target/artifacts - name: Download staged package artifacts (x86_64-unknown-linux-gnu) - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-unknown-linux-gnu path: target/artifacts - name: Download staged package artifacts (x86_64-unknown-linux-musl) - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-unknown-linux-musl path: target/artifacts - name: Download staged package artifacts (arm64-apple-darwin) - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: vector-${{ env.VECTOR_VERSION }}-arm64-apple-darwin path: target/artifacts - name: Download staged package artifacts (x86_64-pc-windows-msvc) - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-pc-windows-msvc path: target/artifacts - name: Download staged package artifacts (armv7-unknown-linux-gnueabihf) - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: vector-${{ env.VECTOR_VERSION }}-armv7-unknown-linux-gnueabihf path: target/artifacts - name: Download staged package artifacts (armv7-unknown-linux-musleabihf) - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: vector-${{ env.VECTOR_VERSION }}-armv7-unknown-linux-musleabihf path: target/artifacts - name: Download artifact checksums - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: vector-${{ env.VECTOR_VERSION }}-SHA256SUMS path: target/artifacts - name: Download staged package artifacts (arm-unknown-linux-gnueabi) - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: vector-${{ env.VECTOR_VERSION }}-arm-unknown-linux-gnueabi path: target/artifacts - name: Download staged package artifacts (arm-unknown-linux-musleabi) - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: vector-${{ env.VECTOR_VERSION }}-arm-unknown-linux-musleabi path: target/artifacts @@ -775,75 +775,63 @@ jobs: VECTOR_VERSION: ${{ needs.generate-publish-metadata.outputs.vector_version }} steps: - name: Checkout Vector - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ inputs.git_ref }} - name: Download staged package artifacts (aarch64-unknown-linux-gnu) - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: vector-${{ env.VECTOR_VERSION }}-aarch64-unknown-linux-gnu path: target/artifacts - name: Download staged package artifacts (aarch64-unknown-linux-musl) - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: vector-${{ env.VECTOR_VERSION }}-aarch64-unknown-linux-musl path: target/artifacts - name: Download staged package artifacts (x86_64-unknown-linux-gnu) - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-unknown-linux-gnu path: target/artifacts - name: Download staged package artifacts (x86_64-unknown-linux-musl) - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-unknown-linux-musl path: target/artifacts - name: Download staged package artifacts (arm64-apple-darwin) - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: vector-${{ env.VECTOR_VERSION }}-arm64-apple-darwin path: target/artifacts - name: Download staged package artifacts (x86_64-pc-windows-msvc) - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-pc-windows-msvc path: target/artifacts - name: Download staged package artifacts (armv7-unknown-linux-gnueabihf) - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: vector-${{ env.VECTOR_VERSION }}-armv7-unknown-linux-gnueabihf path: target/artifacts - name: Download staged package artifacts (armv7-unknown-linux-musleabihf) - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: vector-${{ env.VECTOR_VERSION }}-armv7-unknown-linux-musleabihf path: target/artifacts - name: Download staged package artifacts (arm-unknown-linux-gnueabi) - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: vector-${{ env.VECTOR_VERSION }}-arm-unknown-linux-gnueabi path: target/artifacts - name: Download staged package artifacts (arm-unknown-linux-musleabi) - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: vector-${{ env.VECTOR_VERSION }}-arm-unknown-linux-musleabi path: target/artifacts - name: Generate SHA256 checksums for artifacts run: make sha256sum - name: Stage checksum for publish - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: vector-${{ env.VECTOR_VERSION }}-SHA256SUMS path: target/artifacts/vector-${{ env.VECTOR_VERSION }}-SHA256SUMS - - publish-homebrew: - name: Publish to Homebrew - # We only publish versioned releases to Homebrew. - if: ${{ inputs.channel == 'release' }} - uses: ./.github/workflows/publish-homebrew.yml - needs: - - generate-publish-metadata - - publish-s3 - with: - git_ref: ${{ inputs.git_ref }} - vector_version: ${{ needs.generate-publish-metadata.outputs.vector_version }} diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 29e8fe8177398..0c462b27de27a 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -51,7 +51,7 @@ jobs: smp-version: ${{ steps.experimental-meta.outputs.SMP_CRATE_VERSION }} steps: - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 # need to pull repository history to find merge bases @@ -105,7 +105,7 @@ jobs: - name: Set SMP version id: experimental-meta run: | - export SMP_CRATE_VERSION="0.24.1" + export SMP_CRATE_VERSION="0.25.1" echo "smp crate version: ${SMP_CRATE_VERSION}" echo "SMP_CRATE_VERSION=${SMP_CRATE_VERSION}" >> $GITHUB_OUTPUT @@ -117,7 +117,7 @@ jobs: outputs: source_changed: ${{ steps.filter.outputs.SOURCE_CHANGED }} steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Collect file changes id: changes @@ -192,16 +192,16 @@ jobs: steps: - uses: colpal/actions-clean@36e6ca1abd35efe61cb60f912bd7837f67887c8a # v1.1.1 - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ needs.resolve-inputs.outputs.baseline-sha }} path: baseline-vector - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 - name: Build 'vector' target image uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 @@ -216,7 +216,7 @@ jobs: vector:${{ needs.resolve-inputs.outputs.baseline-tag }} - name: Upload image as artifact - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: baseline-image path: "${{ runner.temp }}/baseline-image.tar" @@ -231,16 +231,16 @@ jobs: steps: - uses: colpal/actions-clean@36e6ca1abd35efe61cb60f912bd7837f67887c8a # v1.1.1 - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ needs.resolve-inputs.outputs.comparison-sha }} path: comparison-vector - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 - name: Build 'vector' target image uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 @@ -255,7 +255,7 @@ jobs: vector:${{ needs.resolve-inputs.outputs.comparison-tag }} - name: Upload image as artifact - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: comparison-image path: "${{ runner.temp }}/comparison-image.tar" @@ -269,7 +269,7 @@ jobs: - resolve-inputs steps: - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@a03048d87541d1d9fcf2ecf528a4a65ba9bd7838 # v5.0.0 + uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708 # v5.1.1 with: aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} @@ -294,7 +294,7 @@ jobs: - build-baseline steps: - name: 'Download baseline image' - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: baseline-image @@ -303,7 +303,7 @@ jobs: docker load --input baseline-image.tar - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@a03048d87541d1d9fcf2ecf528a4a65ba9bd7838 # v5.0.0 + uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708 # v5.1.1 with: aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} @@ -314,7 +314,7 @@ jobs: uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2.0.1 - name: Docker Login to ECR - uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: registry: ${{ steps.login-ecr.outputs.registry }} @@ -334,7 +334,7 @@ jobs: - build-comparison steps: - name: 'Download comparison image' - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: comparison-image @@ -343,7 +343,7 @@ jobs: docker load --input comparison-image.tar - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@a03048d87541d1d9fcf2ecf528a4a65ba9bd7838 # v5.0.0 + uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708 # v5.1.1 with: aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} @@ -354,7 +354,7 @@ jobs: uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2.0.1 - name: Docker Login to ECR - uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: registry: ${{ steps.login-ecr.outputs.registry }} @@ -373,12 +373,12 @@ jobs: - upload-baseline-image-to-ecr - upload-comparison-image-to-ecr steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ needs.resolve-inputs.outputs.comparison-sha }} - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@a03048d87541d1d9fcf2ecf528a4a65ba9bd7838 # v5.0.0 + uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708 # v5.1.1 with: aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} @@ -408,7 +408,7 @@ jobs: --submission-metadata ${{ runner.temp }}/submission-metadata \ --replicas ${{ env.SMP_REPLICAS }} - - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: vector-submission-metadata path: ${{ runner.temp }}/submission-metadata @@ -448,10 +448,10 @@ jobs: - should-run-gate - resolve-inputs steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@a03048d87541d1d9fcf2ecf528a4a65ba9bd7838 # v5.0.0 + uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708 # v5.1.1 with: aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} @@ -462,7 +462,7 @@ jobs: aws s3 cp s3://smp-cli-releases/v${{ needs.resolve-inputs.outputs.smp-version }}/x86_64-unknown-linux-musl/smp ${{ runner.temp }}/bin/smp - name: Download submission metadata - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: vector-submission-metadata path: ${{ runner.temp }}/ @@ -485,12 +485,12 @@ jobs: - submit-job - resolve-inputs steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ needs.resolve-inputs.outputs.comparison-sha }} - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@a03048d87541d1d9fcf2ecf528a4a65ba9bd7838 # v5.0.0 + uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708 # v5.1.1 with: aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} @@ -501,7 +501,7 @@ jobs: aws s3 cp s3://smp-cli-releases/v${{ needs.resolve-inputs.outputs.smp-version }}/x86_64-unknown-linux-musl/smp ${{ runner.temp }}/bin/smp - name: Download submission metadata - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: vector-submission-metadata path: ${{ runner.temp }}/ @@ -523,7 +523,7 @@ jobs: path: ${{ runner.temp }}/outputs/report.md - name: Upload regression report to artifacts - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: capture-artifacts path: ${{ runner.temp }}/outputs/* @@ -547,7 +547,7 @@ jobs: steps: - name: Download capture-artifacts continue-on-error: true - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: capture-artifacts diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index d1c4dda39357b..a6482fb83b075 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -32,7 +32,7 @@ jobs: steps: - name: "Checkout code" - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false @@ -59,7 +59,7 @@ jobs: # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF # format to the repository Actions tab. - name: "Upload artifact" - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: SARIF file path: results.sarif @@ -68,6 +68,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard (optional). # Commenting out will disable upload of results to your repo's Code Scanning dashboard - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@64d10c13136e1c5bce3e5fbde8d4906eeaafc885 # v3.30.6 + uses: github/codeql-action/upload-sarif@b20883b0cd1f46c72ae0ba6d1090936928f9fa30 # v4.32.0 with: sarif_file: results.sarif diff --git a/.github/workflows/semantic.yml b/.github/workflows/semantic.yml index 1b8ae3c0021c8..cd5c6f9330328 100644 --- a/.github/workflows/semantic.yml +++ b/.github/workflows/semantic.yml @@ -9,8 +9,14 @@ on: pull_request: types: [opened, edited, synchronize] +permissions: + contents: read + jobs: main: + permissions: + pull-requests: read # for amannn/action-semantic-pull-request to analyze PRs + statuses: write # for amannn/action-semantic-pull-request to mark status of analyzed PR name: Check Semantic PR runs-on: ubuntu-24.04 steps: @@ -159,6 +165,7 @@ jobs: kubernetes_logs source logstash source mongodb_metrics source + mqtt source nats source new source nginx_metrics source @@ -177,6 +184,7 @@ jobs: vector source websocket source + aggregate transform aws_ec2_metadata transform dedupe transform exclusive_route transform @@ -213,6 +221,7 @@ jobs: datadog_events sink datadog_logs sink datadog_metrics sink + doris sink elasticsearch sink file sink gcp_chronicle sink diff --git a/.github/workflows/spelling.yml b/.github/workflows/spelling.yml index 810fb03867dd3..425111667a5e8 100644 --- a/.github/workflows/spelling.yml +++ b/.github/workflows/spelling.yml @@ -67,6 +67,9 @@ on: - 'reopened' - 'synchronize' +permissions: + contents: read + jobs: spelling: name: Check Spelling diff --git a/.github/workflows/test-make-command.yml b/.github/workflows/test-make-command.yml index 555bda0fcba19..3f2d483edb0cd 100644 --- a/.github/workflows/test-make-command.yml +++ b/.github/workflows/test-make-command.yml @@ -11,9 +11,14 @@ on: description: 'Display name for the job/status check' required: true type: string + ref: + description: 'Git ref to checkout' + required: false + type: string + permissions: - statuses: write + contents: read jobs: run-make-command: @@ -22,47 +27,14 @@ jobs: env: CARGO_INCREMENTAL: 0 steps: - - name: (PR review) Set latest commit status as pending - if: ${{ github.event_name == 'pull_request_review' }} - uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 - with: - sha: ${{ github.event.review.commit_id }} - token: ${{ secrets.GITHUB_TOKEN }} - context: ${{ inputs.job_name }} - status: pending - - - name: (PR review) Checkout review SHA - if: ${{ github.event_name == 'pull_request_review' }} - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: - ref: ${{ github.event.review.commit_id }} - - name: Checkout branch - if: ${{ github.event_name != 'pull_request_review' }} - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 - name: Cache Cargo registry + index + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-cargo- + ref: ${{ inputs.ref }} - - run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh - - run: bash scripts/environment/prepare.sh --modules=rustup - - run: echo "::add-matcher::.github/matchers/rust.json" - - run: make ${{ inputs.command }} - - - name: (PR review) Set latest commit status as ${{ job.status }} - uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 - if: always() && github.event_name == 'pull_request_review' + - uses: ./.github/actions/setup with: - sha: ${{ github.event.review.commit_id }} - token: ${{ secrets.GITHUB_TOKEN }} - context: ${{ inputs.job_name }} - status: ${{ job.status }} + rust: true + protoc: true + libsasl2: true + - run: make ${{ inputs.command }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a18c3a913160f..114321ae28242 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -30,7 +30,7 @@ jobs: if: ${{ needs.changes.outputs.source == 'true' || needs.changes.outputs.test-yml == 'true' }} needs: changes steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: ./.github/actions/setup with: rust: true @@ -42,12 +42,12 @@ jobs: if: ${{ needs.changes.outputs.source == 'true' || needs.changes.outputs.test-yml == 'true' }} needs: changes steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: ./.github/actions/setup with: rust: true protoc: true - - run: sudo apt-get update && sudo apt-get install -y libsasl2-dev + libsasl2: true - run: make check-clippy test: @@ -56,14 +56,14 @@ jobs: if: ${{ needs.changes.outputs.source == 'true' || needs.changes.outputs.test-yml == 'true' }} needs: changes steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: ./.github/actions/setup with: rust: true cargo-nextest: true datadog-ci: true protoc: true - - run: sudo apt-get update && sudo apt-get install -y libsasl2-dev + libsasl2: true - name: Unit Test run: make test env: @@ -83,7 +83,12 @@ jobs: if: ${{ needs.changes.outputs.scripts == 'true' || needs.changes.outputs.test-yml == 'true' }} needs: changes steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: ./.github/actions/setup + with: + cargo-cache: false + mold: false + vdev: true - run: make check-scripts check-events: @@ -92,10 +97,12 @@ jobs: if: ${{ needs.changes.outputs.source == 'true' || needs.changes.outputs.test-yml == 'true' }} needs: changes steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: ./.github/actions/setup with: - rust: true + cargo-cache: false + mold: false + vdev: true - run: make check-events check-licenses: @@ -104,11 +111,12 @@ jobs: if: ${{ needs.changes.outputs.dependencies == 'true' || needs.changes.outputs.test-yml == 'true' }} needs: changes steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: ./.github/actions/setup with: cargo-cache: false mold: false + vdev: true dd-rust-license-tool: true - run: make check-licenses @@ -118,7 +126,7 @@ jobs: if: ${{ needs.changes.outputs.cue == 'true' || needs.changes.outputs.test-yml == 'true' }} needs: changes steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: ./.github/actions/setup with: rust: true @@ -131,7 +139,7 @@ jobs: if: ${{ needs.changes.outputs.markdown == 'true' || needs.changes.outputs.test-yml == 'true' }} needs: changes steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: ./.github/actions/setup with: rust: true @@ -144,13 +152,13 @@ jobs: if: ${{ needs.changes.outputs.source == 'true' || needs.changes.outputs.component_docs == 'true' || needs.changes.outputs.test-yml == 'true' }} needs: changes steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - run: sudo apt-get update && sudo apt-get install -y libsasl2-dev + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: ./.github/actions/setup with: rust: true protoc: true cue: true + libsasl2: true - run: make check-component-docs check-rust-docs: @@ -159,11 +167,11 @@ jobs: if: ${{ needs.changes.outputs.source == 'true' || needs.changes.outputs.test-yml == 'true' }} needs: changes steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: ./.github/actions/setup with: rust: true - - run: cd rust-doc && make docs + - run: cd rust-doc && make ci-docs-build test-vrl: name: VRL - Linux @@ -171,7 +179,7 @@ jobs: if: ${{ needs.changes.outputs.source == 'true' || needs.changes.outputs.cue == 'true' || needs.changes.outputs.test-yml == 'true' }} needs: changes steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: ./.github/actions/setup with: rust: true @@ -185,7 +193,7 @@ jobs: if: ${{ needs.changes.outputs.source == 'true' || needs.changes.outputs.dependencies == 'true' || needs.changes.outputs.test-yml == 'true' }} needs: changes steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: ./.github/actions/setup with: rust: true diff --git a/.github/workflows/unit_mac.yml b/.github/workflows/unit_mac.yml index 8a59c864a5c85..b177b018d13ef 100644 --- a/.github/workflows/unit_mac.yml +++ b/.github/workflows/unit_mac.yml @@ -2,9 +2,15 @@ name: Unit - Mac on: workflow_call: + inputs: + ref: + description: 'Git ref to checkout' + required: false + type: string + permissions: - statuses: write + contents: read jobs: unit-mac: @@ -13,40 +19,16 @@ jobs: env: CARGO_INCREMENTAL: 0 steps: - - name: (PR review) Set latest commit status as pending - if: ${{ github.event_name == 'pull_request_review' }} - uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 - with: - sha: ${{ github.event.review.commit_id }} - token: ${{ secrets.GITHUB_TOKEN }} - context: Unit - Mac - status: pending - - - name: (PR review) Checkout PR branch - if: ${{ github.event_name == 'pull_request_review' }} - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: - ref: ${{ github.event.review.commit_id }} - - name: Checkout branch - if: ${{ github.event_name != 'pull_request_review' }} - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 - name: Cache Cargo registry + index + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-cargo- + ref: ${{ inputs.ref }} - - run: bash scripts/environment/bootstrap-macos.sh - - run: bash scripts/environment/prepare.sh --modules=cargo-nextest - - run: echo "::add-matcher::.github/matchers/rust.json" + - uses: ./.github/actions/setup + with: + rust: true + cargo-nextest: true + protoc: true # Some tests e.g. `reader_exits_cleanly_when_writer_done_and_in_flight_acks` are flaky. - name: Run tests @@ -57,12 +39,3 @@ jobs: command: make test - run: make test-behavior - - - name: (PR review) Set latest commit status as ${{ job.status }} - uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 - if: always() && github.event_name == 'pull_request_review' - with: - sha: ${{ github.event.review.commit_id }} - token: ${{ secrets.GITHUB_TOKEN }} - context: Unit - Mac - status: ${{ job.status }} diff --git a/.github/workflows/unit_windows.yml b/.github/workflows/unit_windows.yml index 41a2f24b776f8..831eec042f869 100644 --- a/.github/workflows/unit_windows.yml +++ b/.github/workflows/unit_windows.yml @@ -2,45 +2,28 @@ name: Unit - Windows on: workflow_call: + inputs: + ref: + description: 'Git ref to checkout' + required: false + type: string + permissions: - statuses: write + contents: read jobs: test-windows: runs-on: windows-2025-8core timeout-minutes: 60 steps: - - name: (PR review) Set latest commit status as pending - if: ${{ github.event_name == 'pull_request_review' }} - uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 - with: - sha: ${{ github.event.review.commit_id }} - token: ${{ secrets.GITHUB_TOKEN }} - context: Unit - Windows - status: pending - - - name: (PR review) Checkout PR branch - if: ${{ github.event_name == 'pull_request_review' }} - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: - ref: ${{ github.event.review.commit_id }} - - name: Checkout branch - if: ${{ github.event_name != 'pull_request_review' }} - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ inputs.ref }} - - uses: actions/setup-python@v6 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.10" - run: .\scripts\environment\bootstrap-windows-2025.ps1 - run: make test - - - name: (PR review) Set latest commit status as ${{ job.status }} - uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 - if: always() && github.event_name == 'pull_request_review' - with: - sha: ${{ github.event.review.commit_id }} - token: ${{ secrets.GITHUB_TOKEN }} - context: Unit - Windows - status: ${{ job.status }} diff --git a/.github/workflows/vdev_publish.yml b/.github/workflows/vdev_publish.yml index 36590315228e3..155db01ecbbed 100644 --- a/.github/workflows/vdev_publish.yml +++ b/.github/workflows/vdev_publish.yml @@ -19,7 +19,7 @@ jobs: steps: - name: Checkout Vector - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Bootstrap runner environment (Ubuntu) if: startsWith(matrix.os, 'ubuntu') @@ -56,7 +56,7 @@ jobs: echo "ASSET=${OUTDIR}.tgz" >> "$GITHUB_ENV" - name: Upload asset to release - uses: softprops/action-gh-release@v2 + uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # v2.5.0 with: files: ${{ env.ASSET }} env: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 80a261ea0dc1b..93868085dc574 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -338,7 +338,7 @@ Vector requires all contributors to sign the Contributor License Agreement (CLA). This gives Vector the right to use your contribution as well as ensuring that you own your contributions and can use them for other purposes. -The full text of the CLA can be found at [https://cla.datadoghq.com/vectordotdev/vector](https://cla.datadoghq.com/vectordotdev/vector). +The full text of the CLA can be found [here](https://gist.github.com/bits-bot/55bdc97a4fdad52d97feb4d6c3d1d618). ### Granted rights and copyright assignment diff --git a/Cargo.lock b/Cargo.lock index 61e4ffce98b44..1f073f6c4eda4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12,21 +12,6 @@ dependencies = [ "regex", ] -[[package]] -name = "RustyXML" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b5ace29ee3216de37c0546865ad08edef58b0f9e76838ed8959a84a990e58c5" - -[[package]] -name = "addr2line" -version = "0.24.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" -dependencies = [ - "gimli", -] - [[package]] name = "adler2" version = "2.0.0" @@ -94,6 +79,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ "cfg-if", + "const-random", "getrandom 0.2.15", "once_cell", "serde", @@ -286,6 +272,30 @@ dependencies = [ "uuid", ] +[[package]] +name = "apache-avro" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a033b4ced7c585199fb78ef50fca7fe2f444369ec48080c5fd072efa1a03cc7" +dependencies = [ + "bigdecimal", + "bon", + "digest", + "log", + "miniz_oxide", + "num-bigint", + "quad-rand", + "rand 0.9.2", + "regex-lite", + "serde", + "serde_bytes", + "serde_json", + "strum 0.27.2", + "strum_macros 0.27.2", + "thiserror 2.0.17", + "uuid", +] + [[package]] name = "approx" version = "0.5.1" @@ -338,6 +348,175 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" +[[package]] +name = "arrow" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e833808ff2d94ed40d9379848a950d995043c7fb3e81a30b383f4c6033821cc" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-ipc", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", +] + +[[package]] +name = "arrow-arith" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad08897b81588f60ba983e3ca39bda2b179bdd84dced378e7df81a5313802ef8" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "num", +] + +[[package]] +name = "arrow-array" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8548ca7c070d8db9ce7aa43f37393e4bfcf3f2d3681df278490772fd1673d08d" +dependencies = [ + "ahash 0.8.11", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "hashbrown 0.16.0", + "num", +] + +[[package]] +name = "arrow-buffer" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e003216336f70446457e280807a73899dd822feaf02087d31febca1363e2fccc" +dependencies = [ + "bytes 1.10.1", + "half", + "num", +] + +[[package]] +name = "arrow-cast" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "919418a0681298d3a77d1a315f625916cb5678ad0d74b9c60108eb15fd083023" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "atoi", + "base64 0.22.1", + "chrono", + "half", + "lexical-core", + "num", + "ryu", +] + +[[package]] +name = "arrow-data" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5c64fff1d142f833d78897a772f2e5b55b36cb3e6320376f0961ab0db7bd6d0" +dependencies = [ + "arrow-buffer", + "arrow-schema", + "half", + "num", +] + +[[package]] +name = "arrow-ipc" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d3594dcddccc7f20fd069bc8e9828ce37220372680ff638c5e00dea427d88f5" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "flatbuffers", +] + +[[package]] +name = "arrow-ord" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c8f82583eb4f8d84d4ee55fd1cb306720cddead7596edce95b50ee418edf66f" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", +] + +[[package]] +name = "arrow-row" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d07ba24522229d9085031df6b94605e0f4b26e099fb7cdeec37abd941a73753" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "half", +] + +[[package]] +name = "arrow-schema" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3aa9e59c611ebc291c28582077ef25c97f1975383f1479b12f3b9ffee2ffabe" + +[[package]] +name = "arrow-select" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c41dbbd1e97bfcaee4fcb30e29105fb2c75e4d82ae4de70b792a5d3f66b2e7a" +dependencies = [ + "ahash 0.8.11", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "num", +] + +[[package]] +name = "arrow-string" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53f5183c150fbc619eede22b861ea7c0eebed8eaac0333eaa7f6da5205fd504d" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "memchr", + "num", + "regex", + "regex-syntax", +] + [[package]] name = "ascii" version = "0.9.3" @@ -497,7 +676,7 @@ dependencies = [ "futures-timer", "futures-util", "http 1.3.1", - "indexmap 2.11.0", + "indexmap 2.12.0", "mime", "multer", "num-traits", @@ -546,7 +725,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34ecdaff7c9cffa3614a9f9999bf9ee4c3078fe3ce4d6a6e161736b56febf2de" dependencies = [ "bytes 1.10.1", - "indexmap 2.11.0", + "indexmap 2.12.0", "serde", "serde_json", ] @@ -847,7 +1026,7 @@ dependencies = [ "bytes 1.10.1", "fastrand 2.3.0", "http 0.2.9", - "http-body 0.4.5", + "http-body 0.4.6", "percent-encoding", "pin-project-lite", "tracing 0.1.41", @@ -875,7 +1054,7 @@ dependencies = [ "fastrand 2.3.0", "flate2", "http 0.2.9", - "http-body 0.4.5", + "http-body 0.4.6", "once_cell", "regex-lite", "tracing 0.1.41", @@ -1022,7 +1201,7 @@ dependencies = [ "hmac", "http 0.2.9", "http 1.3.1", - "http-body 0.4.5", + "http-body 0.4.6", "lru 0.12.5", "once_cell", "percent-encoding", @@ -1217,7 +1396,7 @@ dependencies = [ "crc64fast-nvme", "hex", "http 0.2.9", - "http-body 0.4.5", + "http-body 0.4.6", "md-5", "pin-project-lite", "sha1", @@ -1237,7 +1416,7 @@ dependencies = [ "flate2", "futures-util", "http 0.2.9", - "http-body 0.4.5", + "http-body 0.4.6", "pin-project-lite", "tracing 0.1.41", ] @@ -1267,7 +1446,7 @@ dependencies = [ "futures-core", "http 0.2.9", "http 1.3.1", - "http-body 0.4.5", + "http-body 0.4.6", "percent-encoding", "pin-project-lite", "pin-utils", @@ -1286,8 +1465,8 @@ dependencies = [ "h2 0.3.26", "h2 0.4.12", "http 0.2.9", - "http-body 0.4.5", - "hyper 0.14.28", + "http-body 0.4.6", + "hyper 0.14.32", "hyper-rustls 0.24.2", "pin-project-lite", "rustls 0.21.12", @@ -1339,7 +1518,7 @@ dependencies = [ "fastrand 2.3.0", "http 0.2.9", "http 1.3.1", - "http-body 0.4.5", + "http-body 0.4.6", "http-body 1.0.0", "pin-project-lite", "pin-utils", @@ -1376,7 +1555,7 @@ dependencies = [ "futures-core", "http 0.2.9", "http 1.3.1", - "http-body 0.4.5", + "http-body 0.4.6", "http-body 1.0.0", "http-body-util", "itoa", @@ -1409,7 +1588,7 @@ dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", "aws-smithy-types", - "rustc_version 0.4.1", + "rustc_version", "tracing 0.1.41", ] @@ -1425,8 +1604,8 @@ dependencies = [ "bytes 1.10.1", "futures-util", "http 0.2.9", - "http-body 0.4.5", - "hyper 0.14.28", + "http-body 0.4.6", + "hyper 0.14.32", "itoa", "matchit", "memchr", @@ -1479,7 +1658,7 @@ dependencies = [ "bytes 1.10.1", "futures-util", "http 0.2.9", - "http-body 0.4.5", + "http-body 0.4.6", "mime", "rustversion", "tower-layer", @@ -1508,46 +1687,18 @@ dependencies = [ [[package]] name = "azure_core" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b552ad43a45a746461ec3d3a51dfb6466b4759209414b439c165eb6a6b7729e" -dependencies = [ - "async-trait", - "base64 0.22.1", - "bytes 1.10.1", - "dyn-clone", - "futures 0.3.31", - "getrandom 0.2.15", - "http-types", - "once_cell", - "openssl", - "paste", - "pin-project", - "quick-xml 0.31.0", - "rand 0.8.5", - "reqwest 0.12.9", - "rustc_version 0.4.1", - "serde", - "serde_json", - "time", - "tracing 0.1.41", - "url", - "uuid", -] - -[[package]] -name = "azure_core" -version = "0.25.0" +version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82c33c072c9d87777262f35abfe2a64b609437076551d4dac8373e60f0e3fde9" +checksum = "f35444aeeb91e29ca82d04dbb8ad904570f837c82093454dc1989c64a33554a7" dependencies = [ "async-lock 3.4.0", "async-trait", + "azure_core_macros", "bytes 1.10.1", "futures 0.3.31", "openssl", "pin-project", - "rustc_version 0.4.1", + "rustc_version", "serde", "serde_json", "tracing 0.1.41", @@ -1556,79 +1707,32 @@ dependencies = [ ] [[package]] -name = "azure_identity" -version = "0.25.0" +name = "azure_core_macros" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb64e97087965481c94f1703c57e678df09df73e2cdaee8952558f9c6c7d100" +checksum = "94f7cc1bbae04cfe11de9e39e2c6dc755947901e0f4e76180ab542b6deb5e15e" dependencies = [ - "async-lock 3.4.0", - "async-trait", - "azure_core 0.25.0", - "futures 0.3.31", - "pin-project", - "serde", - "time", + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", "tracing 0.1.41", - "typespec_client_core", - "url", ] [[package]] -name = "azure_storage" -version = "0.21.0" +name = "azure_storage_blob" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59f838159f4d29cb400a14d9d757578ba495ae64feb07a7516bf9e4415127126" +checksum = "c38e589153b04af727a736f435d07b48b1a2b17df9353e5d439698fe8b718412" dependencies = [ - "RustyXML", - "async-lock 3.4.0", "async-trait", - "azure_core 0.21.0", - "bytes 1.10.1", - "serde", - "serde_derive", - "time", - "tracing 0.1.41", - "url", - "uuid", -] - -[[package]] -name = "azure_storage_blobs" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97e83c3636ae86d9a6a7962b2112e3b19eb3903915c50ce06ff54ff0a2e6a7e4" -dependencies = [ - "RustyXML", - "azure_core 0.21.0", - "azure_storage", - "azure_svc_blobstorage", - "bytes 1.10.1", - "futures 0.3.31", + "azure_core", "serde", - "serde_derive", "serde_json", - "time", - "tracing 0.1.41", + "typespec_client_core", "url", "uuid", ] -[[package]] -name = "azure_svc_blobstorage" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e6c6f20c5611b885ba94c7bae5e02849a267381aecb8aee577e8c35ff4064c6" -dependencies = [ - "azure_core 0.21.0", - "bytes 1.10.1", - "futures 0.3.31", - "log", - "once_cell", - "serde", - "serde_json", - "time", -] - [[package]] name = "backoff" version = "0.4.0" @@ -1651,21 +1755,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "backtrace" -version = "0.3.75" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6806a6321ec58106fea15becdad98371e28d92ccbc7c8f1b3b6dd724fe8f1002" -dependencies = [ - "addr2line", - "cfg-if", - "libc", - "miniz_oxide", - "object", - "rustc-demangle", - "windows-targets 0.52.6", -] - [[package]] name = "base16" version = "0.2.1" @@ -1721,6 +1810,38 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" +[[package]] +name = "bigdecimal" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "560f42649de9fa436b73517378a147ec21f6c997a546581df4b4b31677828934" +dependencies = [ + "autocfg", + "libm", + "num-bigint", + "num-integer", + "num-traits", + "serde", +] + +[[package]] +name = "bindgen" +version = "0.72.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +dependencies = [ + "bitflags 2.10.0", + "cexpr", + "clang-sys", + "itertools 0.13.0", + "proc-macro2 1.0.101", + "quote 1.0.40", + "regex", + "rustc-hash", + "shlex", + "syn 2.0.106", +] + [[package]] name = "bit-set" version = "0.8.0" @@ -1744,11 +1865,11 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.9.0" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" +checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" dependencies = [ - "serde", + "serde_core", ] [[package]] @@ -1848,7 +1969,7 @@ dependencies = [ "serde_json", "serde_repr", "serde_urlencoded", - "thiserror 2.0.3", + "thiserror 2.0.17", "tokio", "tokio-util", "tower-service", @@ -1866,14 +1987,62 @@ dependencies = [ "serde", "serde_json", "serde_repr", - "serde_with 3.14.0", + "serde_with", +] + +[[package]] +name = "bon" +version = "3.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebeb9aaf9329dff6ceb65c689ca3db33dbf15f324909c60e4e5eef5701ce31b1" +dependencies = [ + "bon-macros", + "rustversion", +] + +[[package]] +name = "bon-macros" +version = "3.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77e9d642a7e3a318e37c2c9427b5a6a48aa1ad55dcd986f3034ab2239045a645" +dependencies = [ + "darling 0.21.3", + "ident_case", + "prettyplease 0.2.15", + "proc-macro2 1.0.101", + "quote 1.0.40", + "rustversion", + "syn 2.0.106", ] [[package]] name = "borrow-or-share" -version = "0.2.2" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0b364ead1874514c8c2855ab558056ebfeb775653e7ae45ff72f28f8f3166c" + +[[package]] +name = "borsh" +version = "1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8646f98db542e39fc66e68a20b2144f6a732636df7c2354e74645faaa433ce" +dependencies = [ + "borsh-derive", + "cfg_aliases", +] + +[[package]] +name = "borsh-derive" +version = "1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eeab4423108c5d7c744f4d234de88d18d636100093ae04caf4825134b9c3a32" +checksum = "fdd1d3c0c2f5833f22386f252fe8ed005c7f59fdcddeef025c01b4c3b9fd9ac3" +dependencies = [ + "once_cell", + "proc-macro-crate 3.2.0", + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", +] [[package]] name = "brotli" @@ -1898,18 +2067,20 @@ dependencies = [ [[package]] name = "bson" -version = "2.8.0" +version = "2.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61570f4de0cc9c03b481c96057b3ae7c6ff7b5b35da8b0832c44f0131987a718" +checksum = "7969a9ba84b0ff843813e7249eed1678d9b6607ce5a3b8f0a47af3fcf7978e6e" dependencies = [ "ahash 0.8.11", - "base64 0.13.1", + "base64 0.22.1", "bitvec", + "getrandom 0.2.15", + "getrandom 0.3.4", "hex", - "indexmap 1.9.3", + "indexmap 2.12.0", "js-sys", "once_cell", - "rand 0.8.5", + "rand 0.9.2", "serde", "serde_bytes", "serde_json", @@ -1978,6 +2149,20 @@ name = "bytemuck" version = "1.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef657dfab802224e671f5818e9a4935f9b1957ed18e58292690cc39e7a4092a3" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff" +dependencies = [ + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", +] [[package]] name = "byteorder" @@ -2026,18 +2211,12 @@ version = "10.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c06acb4f71407ba205a07cb453211e0e6a67b21904e47f6ba1f9589e38f2e454" dependencies = [ - "semver 1.0.26", + "semver", "serde", "toml 0.8.23", "url", ] -[[package]] -name = "cassowary" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53" - [[package]] name = "cast" version = "0.3.0" @@ -2079,6 +2258,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom 7.1.3", +] + [[package]] name = "cfb-mode" version = "0.8.2" @@ -2146,7 +2334,7 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-link", + "windows-link 0.1.0", ] [[package]] @@ -2204,11 +2392,22 @@ dependencies = [ "zeroize", ] +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + [[package]] name = "clap" -version = "4.5.48" +version = "4.5.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2134bb3ea021b78629caa971416385309e0131b351b25e01dc16fb54e1b5fae" +checksum = "a75ca66430e33a14957acc24c5077b503e7d374151b2b4b3a10c83b4ceb4be0e" dependencies = [ "clap_builder", "clap_derive", @@ -2226,31 +2425,31 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.48" +version = "4.5.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2ba64afa3c0a6df7fa517765e31314e983f51dda798ffba27b988194fb65dc9" +checksum = "793207c7fa6300a0608d1080b858e5fdbe713cdc1c8db9fb17777d8a13e63df0" dependencies = [ "anstream", "anstyle", "clap_lex", - "strsim 0.11.1", + "strsim", "terminal_size", ] [[package]] name = "clap_complete" -version = "4.5.58" +version = "4.5.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75bf0b32ad2e152de789bb635ea4d3078f6b838ad7974143e99b99f45a04af4a" +checksum = "430b4dc2b5e3861848de79627b2bedc9f3342c7da5173a14eaa5d0f8dc18ae5d" dependencies = [ "clap", ] [[package]] name = "clap_derive" -version = "4.5.47" +version = "4.5.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbfd7eae0b0f1a6e63d4b13c9c478de77c2eb546fba158ad50b4203dc24b9f9c" +checksum = "a92793da1a46a5f2a02a6f4c46c6496b28c43638adea8306fcb0caa1634f24e5" dependencies = [ "heck 0.5.0", "proc-macro2 1.0.101", @@ -2286,9 +2485,9 @@ dependencies = [ [[package]] name = "cmake" -version = "0.1.50" +version = "0.1.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31c789563b815f77f4250caee12365734369f942439b7defd71e18a48197130" +checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0" dependencies = [ "cc", ] @@ -2297,17 +2496,21 @@ dependencies = [ name = "codecs" version = "0.1.0" dependencies = [ - "apache-avro", + "apache-avro 0.20.0", + "arrow", + "async-trait", "bytes 1.10.1", "chrono", "csv-core", "derivative", + "derive_more 2.0.1", "dyn-clone", "flate2", "futures 0.3.31", "indoc", "influxdb-line-protocol", "memchr", + "metrics", "opentelemetry-proto", "ordered-float 4.6.0", "prost 0.12.6", @@ -2315,23 +2518,30 @@ dependencies = [ "rand 0.9.2", "regex", "rstest", + "rust_decimal", "serde", + "serde-aux", + "serde_arrow", "serde_json", - "serde_with 3.14.0", + "serde_with", "similar-asserts", "smallvec", "snafu 0.8.9", + "strum 0.26.3", "syslog_loose 0.23.0", "tokio", "tokio-util", + "toml 0.9.8", "tracing 0.1.41", "tracing-test", "uuid", "vector-common", + "vector-common-macros", "vector-config", "vector-config-macros", "vector-core", "vector-lookup", + "vector-vrl-functions", "vrl", ] @@ -2404,9 +2614,9 @@ dependencies = [ [[package]] name = "compact_str" -version = "0.8.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6050c3a16ddab2e412160b31f2c871015704239bca62f72f6e5f0be631d3f644" +checksum = "3fdb1325a1cece981e8a296ab8f0f9b63ae357bd0784a9faaf548cc7b480707a" dependencies = [ "castaway", "cfg-if", @@ -2445,18 +2655,6 @@ dependencies = [ "crossbeam-utils", ] -[[package]] -name = "confy" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f29222b549d4e3ded127989d523da9e928918d0d0d7f7c1690b439d0d538bae9" -dependencies = [ - "directories", - "serde", - "thiserror 2.0.3", - "toml 0.8.23", -] - [[package]] name = "console" version = "0.15.7" @@ -2471,15 +2669,15 @@ dependencies = [ [[package]] name = "console" -version = "0.16.0" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e09ced7ebbccb63b4c65413d821f2e00ce54c5ca4514ddc6b3c892fdbcbc69d" +checksum = "b430743a6eb14e9764d4260d4c0d8123087d504eeb9c48f2b2a5e810dd369df4" dependencies = [ "encode_unicode 1.0.0", "libc", "once_cell", "unicode-width 0.2.0", - "windows-sys 0.60.2", + "windows-sys 0.61.0", ] [[package]] @@ -2527,6 +2725,32 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28c122c3980598d243d63d9a704629a2d748d101f278052ff068be5a4423ab6f" +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.15", + "once_cell", + "tiny-keccak", +] + +[[package]] +name = "const-str" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93e19f68b180ebff43d6d42005c4b5f046c65fcac28369ba8b3beaad633f9ec0" + [[package]] name = "convert_case" version = "0.4.0" @@ -2570,13 +2794,13 @@ checksum = "396de984970346b0d9e93d1415082923c679e5ae5c3ee3dcbd104f5610af126b" [[package]] name = "cookie_store" -version = "0.21.1" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eac901828f88a5241ee0600950ab981148a18f2f756900ffba1b125ca6a3ef9" +checksum = "3fc4bff745c9b4c7fb1e97b25d13153da2bc7796260141df62378998d070207f" dependencies = [ "cookie", "document-features", - "idna 1.0.3", + "idna", "log", "publicsuffix", "serde", @@ -2651,7 +2875,7 @@ version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" dependencies = [ - "rustc_version 0.4.1", + "rustc_version", ] [[package]] @@ -2761,30 +2985,15 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" -[[package]] -name = "crossterm" -version = "0.28.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" -dependencies = [ - "bitflags 2.9.0", - "crossterm_winapi", - "mio", - "parking_lot 0.12.4", - "rustix 0.38.40", - "signal-hook", - "signal-hook-mio", - "winapi", -] - [[package]] name = "crossterm" version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8b9f2e4c67f833b660cdb0a3523065869fb35570177239812ed4c905aeff87b" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.10.0", "crossterm_winapi", + "derive_more 2.0.1", "document-features", "futures-core", "mio", @@ -2880,9 +3089,9 @@ dependencies = [ [[package]] name = "curl-sys" -version = "0.4.74+curl-8.9.0" +version = "0.4.84+curl-8.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8af10b986114528fcdc4b63b6f5f021b7057618411046a4de2ba0f0149a097bf" +checksum = "abc4294dc41b882eaff37973c2ec3ae203d0091341ee68fbadd1d06e0c18a73b" dependencies = [ "cc", "libc", @@ -2890,7 +3099,7 @@ dependencies = [ "openssl-sys", "pkg-config", "vcpkg", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -2904,7 +3113,7 @@ dependencies = [ "curve25519-dalek-derive", "digest", "fiat-crypto", - "rustc_version 0.4.1", + "rustc_version", "subtle", "zeroize", ] @@ -2922,79 +3131,79 @@ dependencies = [ [[package]] name = "darling" -version = "0.13.4" +version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a01d95850c592940db9b8194bc39f4bc0e89dee5c4265e4b1807c34a9aba453c" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" dependencies = [ - "darling_core 0.13.4", - "darling_macro 0.13.4", + "darling_core 0.20.11", + "darling_macro 0.20.11", ] [[package]] name = "darling" -version = "0.20.11" +version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" dependencies = [ - "darling_core 0.20.11", - "darling_macro 0.20.11", + "darling_core 0.21.3", + "darling_macro 0.21.3", ] [[package]] name = "darling_core" -version = "0.13.4" +version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "859d65a907b6852c9361e3185c862aae7fafd2887876799fa55f5f99dc40d610" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" dependencies = [ "fnv", "ident_case", "proc-macro2 1.0.101", "quote 1.0.40", - "strsim 0.10.0", - "syn 1.0.109", + "strsim", + "syn 2.0.106", ] [[package]] name = "darling_core" -version = "0.20.11" +version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" dependencies = [ "fnv", "ident_case", "proc-macro2 1.0.101", "quote 1.0.40", - "strsim 0.11.1", + "strsim", "syn 2.0.106", ] [[package]] name = "darling_macro" -version = "0.13.4" +version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ - "darling_core 0.13.4", + "darling_core 0.20.11", "quote 1.0.40", - "syn 1.0.109", + "syn 2.0.106", ] [[package]] name = "darling_macro" -version = "0.20.11" +version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" dependencies = [ - "darling_core 0.20.11", + "darling_core 0.21.3", "quote 1.0.40", "syn 2.0.106", ] [[package]] name = "dary_heap" -version = "0.3.6" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7762d17f1241643615821a8455a0b2c3e803784b058693d990b11f2dce25a0ca" +checksum = "06d2e3287df1c007e74221c49ca10a95d557349e54b3a75dc2fb14712c751f04" [[package]] name = "dashmap" @@ -3033,8 +3242,8 @@ dependencies = [ "once_cell", "parking_lot 0.12.4", "percent-encoding", - "reqwest 0.12.9", - "semver 1.0.26", + "reqwest 0.12.28", + "semver", "serde", "serde_json", "tokio", @@ -3087,12 +3296,12 @@ dependencies = [ [[package]] name = "deranged" -version = "0.3.9" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f32d04922c60427da6f9fef14d042d9edddef64cb9d4ce0d64d0685fbeb1fd3" +checksum = "ececcb659e7ba858fb4f10388c250a7252eb0a27373f1a72b8748afdd248e587" dependencies = [ "powerfmt", - "serde", + "serde_core", ] [[package]] @@ -3106,6 +3315,28 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "derive-syn-parse" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d65d7ce8132b7c0e54497a4d9a55a1c2a0912a0d786cf894472ba818fba45762" +dependencies = [ + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", +] + +[[package]] +name = "derive-where" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef941ded77d15ca19b40374869ac6000af1c9f2a4c0f3d4c70926287e6364a8f" +dependencies = [ + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", +] + [[package]] name = "derive_arbitrary" version = "1.3.2" @@ -3157,10 +3388,32 @@ dependencies = [ "convert_case 0.4.0", "proc-macro2 1.0.101", "quote 1.0.40", - "rustc_version 0.4.1", + "rustc_version", "syn 1.0.109", ] +[[package]] +name = "derive_more" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "093242cf7570c207c83073cf82f79706fe7b8317e98620a47d5be7c3d8497678" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bda628edc44c4bb645fbe0f758797143e4e07926f7ebf4e9bdfbd3d2ce621df3" +dependencies = [ + "convert_case 0.7.1", + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", + "unicode-xid 0.2.4", +] + [[package]] name = "difflib" version = "0.4.0" @@ -3207,7 +3460,7 @@ dependencies = [ "libc", "option-ext", "redox_users 0.5.0", - "windows-sys 0.60.2", + "windows-sys 0.61.0", ] [[package]] @@ -3250,7 +3503,7 @@ version = "0.1.0" dependencies = [ "criterion", "data-encoding", - "hickory-proto", + "hickory-proto 0.25.2", "snafu 0.8.9", ] @@ -3264,13 +3517,16 @@ dependencies = [ "chrono", "chrono-tz", "dnsmsg-parser", - "hickory-proto", + "hickory-proto 0.25.2", "paste", "prost 0.12.6", "prost-build 0.12.6", "snafu 0.8.9", "tracing 0.1.41", - "vector-lib", + "vector-common", + "vector-config", + "vector-core", + "vector-lookup", "vrl", ] @@ -3353,12 +3609,6 @@ dependencies = [ "shared_child", ] -[[package]] -name = "dunce" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" - [[package]] name = "dyn-clone" version = "1.0.20" @@ -3485,22 +3735,12 @@ version = "0.1.0" dependencies = [ "arc-swap", "chrono", + "const-str", "dyn-clone", + "indoc", "vrl", ] -[[package]] -name = "enum-as-inner" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21cdad81446a7f7dc43f6a77409efeb9733d2fa65553efef6018ef257c959b73" -dependencies = [ - "heck 0.4.1", - "proc-macro2 1.0.101", - "quote 1.0.40", - "syn 1.0.109", -] - [[package]] name = "enum-as-inner" version = "0.6.0" @@ -3732,28 +3972,16 @@ dependencies = [ ] [[package]] -name = "fallible-iterator" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" - -[[package]] -name = "fancy-regex" -version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6215aee357f8c7c989ebb4b8466ca4d7dc93b3957039f2fc3ea2ade8ea5f279" -dependencies = [ - "bit-set", - "derivative", - "regex-automata 0.4.8", - "regex-syntax", -] - -[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + +[[package]] name = "fancy-regex" -version = "0.16.1" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf04c5ec15464ace8355a7b440a33aece288993475556d461154d7a62ad9947c" +checksum = "72cf461f865c862bb7dc573f643dd6a2b6842f7c30b07882b56bd148cc2761b8" dependencies = [ "bit-set", "regex-automata 0.4.8", @@ -3802,7 +4030,7 @@ dependencies = [ "futures 0.3.31", "futures-util", "glob", - "indexmap 2.11.0", + "indexmap 2.12.0", "libc", "quickcheck", "tempfile", @@ -3848,6 +4076,16 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" +[[package]] +name = "flatbuffers" +version = "25.9.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09b6620799e7340ebd9968d2e0708eb82cf1971e9a16821e2091b6d6e475eed5" +dependencies = [ + "bitflags 2.10.0", + "rustc_version", +] + [[package]] name = "flate2" version = "1.1.2" @@ -3876,9 +4114,9 @@ dependencies = [ [[package]] name = "fluent-uri" -version = "0.3.2" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1918b65d96df47d3591bed19c5cca17e3fa5d0707318e4b5ef2eae01764df7e5" +checksum = "bc74ac4d8359ae70623506d512209619e5cf8f347124910440dbc221714b328e" dependencies = [ "borrow-or-share", "ref-cast", @@ -3917,6 +4155,12 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f81ec6369c545a7d40e4589b5597581fa1c441fe1cce96dd1de43159910a36a2" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "foreign-types" version = "0.3.2" @@ -4126,17 +4370,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "getrandom" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" -dependencies = [ - "cfg-if", - "libc", - "wasi 0.9.0+wasi-snapshot-preview1", -] - [[package]] name = "getrandom" version = "0.2.15" @@ -4146,37 +4379,31 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi", "wasm-bindgen", ] [[package]] name = "getrandom" -version = "0.3.1" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", "js-sys", "libc", - "wasi 0.13.3+wasi-0.2.2", + "r-efi", + "wasip2", "wasm-bindgen", - "windows-targets 0.52.6", ] -[[package]] -name = "gimli" -version = "0.31.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" - [[package]] name = "git2" version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2deb07a133b1520dc1a5690e9bd08950108873d7ed5de38dcc74d3b5ebffa110" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.10.0", "libc", "libgit2-sys", "log", @@ -4225,7 +4452,7 @@ dependencies = [ "arc-swap", "futures 0.3.31", "log", - "reqwest 0.12.9", + "reqwest 0.12.28", "serde", "serde_derive", "serde_json", @@ -4246,7 +4473,7 @@ dependencies = [ "futures-sink", "futures-timer", "futures-util", - "getrandom 0.3.1", + "getrandom 0.3.4", "hashbrown 0.15.2", "nonzero_ext", "parking_lot 0.12.4", @@ -4384,7 +4611,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.9", - "indexmap 2.11.0", + "indexmap 2.12.0", "slab", "tokio", "tokio-util", @@ -4403,7 +4630,7 @@ dependencies = [ "futures-core", "futures-sink", "http 1.3.1", - "indexmap 2.11.0", + "indexmap 2.12.0", "slab", "tokio", "tokio-util", @@ -4416,8 +4643,10 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" dependencies = [ + "bytemuck", "cfg-if", "crunchy", + "num-traits", ] [[package]] @@ -4443,32 +4672,34 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.13.1" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ff8ae62cd3a9102e5637afc8452c55acf3844001bd5374e0b0bd7b6616c038" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" dependencies = [ "ahash 0.8.11", + "allocator-api2", ] [[package]] name = "hashbrown" -version = "0.14.5" +version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" dependencies = [ - "ahash 0.8.11", "allocator-api2", + "equivalent", + "foldhash 0.1.3", ] [[package]] name = "hashbrown" -version = "0.15.2" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" +checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d" dependencies = [ "allocator-api2", "equivalent", - "foldhash", + "foldhash 0.2.0", ] [[package]] @@ -4672,6 +4903,30 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hickory-proto" +version = "0.24.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92652067c9ce6f66ce53cc38d1169daa36e6e7eb7dd3b63b5103bd9d97117248" +dependencies = [ + "async-trait", + "cfg-if", + "data-encoding", + "enum-as-inner", + "futures-channel", + "futures-io", + "futures-util", + "idna", + "ipnet", + "once_cell", + "rand 0.8.5", + "thiserror 1.0.68", + "tinyvec", + "tokio", + "tracing 0.1.41", + "url", +] + [[package]] name = "hickory-proto" version = "0.25.2" @@ -4679,26 +4934,47 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8a6fe56c0038198998a6f217ca4e7ef3a5e51f46163bd6dd60b5c71ca6c6502" dependencies = [ "async-trait", - "bitflags 2.9.0", + "bitflags 2.10.0", "cfg-if", "data-encoding", - "enum-as-inner 0.6.0", + "enum-as-inner", "futures-channel", "futures-io", "futures-util", - "idna 1.0.3", + "idna", "ipnet", "once_cell", "rand 0.9.2", "ring", "rustls-pki-types", - "thiserror 2.0.3", + "thiserror 2.0.17", "time", "tinyvec", "tracing 0.1.41", "url", ] +[[package]] +name = "hickory-resolver" +version = "0.24.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbb117a1ca520e111743ab2f6688eddee69db4e0ea242545a604dce8a66fd22e" +dependencies = [ + "cfg-if", + "futures-util", + "hickory-proto 0.24.4", + "ipconfig", + "lru-cache", + "once_cell", + "parking_lot 0.12.4", + "rand 0.8.5", + "resolv-conf", + "smallvec", + "thiserror 1.0.68", + "tokio", + "tracing 0.1.41", +] + [[package]] name = "hkdf" version = "0.12.3" @@ -4772,9 +5048,9 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes 1.10.1", "http 0.2.9", @@ -4820,26 +5096,6 @@ dependencies = [ "serde", ] -[[package]] -name = "http-types" -version = "2.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e9b187a72d63adbfba487f48095306ac823049cb504ee195541e91c7775f5ad" -dependencies = [ - "anyhow", - "async-channel 1.9.0", - "base64 0.13.1", - "futures-lite 1.13.0", - "infer", - "pin-project-lite", - "rand 0.7.3", - "serde", - "serde_json", - "serde_qs", - "serde_urlencoded", - "url", -] - [[package]] name = "httparse" version = "1.10.1" @@ -4860,9 +5116,9 @@ checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" [[package]] name = "hyper" -version = "0.14.28" +version = "0.14.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" dependencies = [ "bytes 1.10.1", "futures-channel", @@ -4870,7 +5126,7 @@ dependencies = [ "futures-util", "h2 0.3.26", "http 0.2.9", - "http-body 0.4.5", + "http-body 0.4.6", "httparse", "httpdate", "itoa", @@ -4927,7 +5183,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6ee5d7a8f718585d1c3c61dfde28ef5b0bb14734b4db13f5ada856cdc6c612b" dependencies = [ "http 0.2.9", - "hyper 0.14.28", + "hyper 0.14.32", "linked_hash_set", "once_cell", "openssl", @@ -4967,7 +5223,7 @@ dependencies = [ "futures 0.3.31", "headers", "http 0.2.9", - "hyper 0.14.28", + "hyper 0.14.32", "openssl", "tokio", "tokio-openssl", @@ -4982,7 +5238,7 @@ checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", "http 0.2.9", - "hyper 0.14.28", + "hyper 0.14.32", "log", "rustls 0.21.12", "rustls-native-certs 0.6.3", @@ -5015,7 +5271,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" dependencies = [ - "hyper 0.14.28", + "hyper 0.14.32", "pin-project-lite", "tokio", "tokio-io-timeout", @@ -5041,7 +5297,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ "bytes 1.10.1", - "hyper 0.14.28", + "hyper 0.14.32", "native-tls", "tokio", "tokio-native-tls", @@ -5065,21 +5321,28 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.9" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41296eb09f183ac68eec06e03cdbea2e759633d4067b2f6552fc2e009bcad08b" +checksum = "3c6995591a8f1380fcb4ba966a252a4b29188d51d2b89e3a252f5305be65aea8" dependencies = [ + "base64 0.22.1", "bytes 1.10.1", "futures-channel", + "futures-core", "futures-util", "http 1.3.1", "http-body 1.0.0", "hyper 1.7.0", + "ipnet", + "libc", + "percent-encoding", "pin-project-lite", - "socket2 0.5.10", + "socket2 0.6.0", + "system-configuration 0.6.1", "tokio", "tower-service", "tracing 0.1.41", + "windows-registry", ] [[package]] @@ -5244,17 +5507,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" -[[package]] -name = "idna" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" -dependencies = [ - "matches", - "unicode-bidi", - "unicode-normalization", -] - [[package]] name = "idna" version = "1.0.3" @@ -5289,13 +5541,14 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.11.0" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2481980430f9f78649238835720ddccc57e52df14ffce1c6f37391d61b563e9" +checksum = "6717a8d2a5a929a1a2eb43a12812498ed141a0bcfb7e8f7844fbdbe4303bba9f" dependencies = [ "equivalent", - "hashbrown 0.15.2", + "hashbrown 0.16.0", "serde", + "serde_core", ] [[package]] @@ -5304,7 +5557,7 @@ version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70a646d946d06bedbbc4cac4c218acf4bbf2d87757a784857025f4d447e4e1cd" dependencies = [ - "console 0.16.0", + "console 0.16.1", "portable-atomic", "unicode-segmentation", "unicode-width 0.2.0", @@ -5318,12 +5571,6 @@ version = "2.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f4c7245a08504955605670dbf141fceab975f15ca21570696aebe9d2e71576bd" -[[package]] -name = "infer" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64e9829a50b42bb782c1df523f78d332fe371b10c661e78b7a3c34b0198e9fac" - [[package]] name = "influxdb-line-protocol" version = "2.0.0" @@ -5343,7 +5590,7 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f37dccff2791ab604f9babef0ba14fbe0be30bd368dc541e2b08d07c8aa908f3" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.10.0", "inotify-sys", "libc", ] @@ -5409,17 +5656,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "io-uring" -version = "0.7.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d93587f37623a1a17d94ef2bc9ada592f5465fe7732084ab7beefabe5c77c0c4" -dependencies = [ - "bitflags 2.9.0", - "cfg-if", - "libc", -] - [[package]] name = "iovec" version = "0.1.4" @@ -5465,6 +5701,16 @@ version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf370abdafd54d13e54a620e8c3e1145f28e46cc9d704bc6d94414559df41763" +[[package]] +name = "iri-string" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f867b9d1d896b67beb18518eda36fdb77a32ea590de864f1325b294a6d14397" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "is-terminal" version = "0.4.9" @@ -5614,27 +5860,28 @@ dependencies = [ [[package]] name = "jsonschema" -version = "0.32.1" +version = "0.38.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24690c68dfcdde5980d676b0f1820981841016b1f29eecb4c42ad48ab4118681" +checksum = "89f50532ce4a0ba3ae930212908d8ec50e7806065c059fe9c75da2ece6132294" dependencies = [ "ahash 0.8.11", - "base64 0.22.1", "bytecount", + "data-encoding", "email_address", - "fancy-regex 0.16.1", + "fancy-regex", "fraction", - "idna 1.0.3", + "getrandom 0.3.4", + "idna", "itoa", "num-cmp", "num-traits", - "once_cell", "percent-encoding", "referencing", "regex", "regex-syntax", "serde", "serde_json", + "unicode-general-category", "uuid-simd", ] @@ -5693,6 +5940,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "kasuari" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fe90c1150662e858c7d5f945089b7517b0a80d8bf7ba4b1b5ffc984e7230a5b" +dependencies = [ + "hashbrown 0.16.0", + "portable-atomic", + "thiserror 2.0.17", +] + [[package]] name = "keccak" version = "0.1.4" @@ -5885,6 +6143,63 @@ dependencies = [ "spin 0.5.2", ] +[[package]] +name = "lexical-core" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" +dependencies = [ + "lexical-parse-integer", + "lexical-util", +] + +[[package]] +name = "lexical-parse-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "lexical-util" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" + +[[package]] +name = "lexical-write-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361" +dependencies = [ + "lexical-util", + "lexical-write-integer", +] + +[[package]] +name = "lexical-write-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df" +dependencies = [ + "lexical-util", +] + [[package]] name = "libc" version = "0.2.175" @@ -5893,9 +6208,9 @@ checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543" [[package]] name = "libflate" -version = "2.0.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7d5654ae1795afc7ff76f4365c2c8791b0feb18e8996a96adad8ffd7c3b2bf" +checksum = "249fa21ba2b59e8cbd69e722f5b31e1b466db96c937ae3de23e8b99ead0d1383" dependencies = [ "adler32", "core2", @@ -5906,12 +6221,12 @@ dependencies = [ [[package]] name = "libflate_lz77" -version = "2.0.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be5f52fb8c451576ec6b79d3f4deb327398bc05bbdbd99021a6e77a4c855d524" +checksum = "a599cb10a9cd92b1300debcef28da8f70b935ec937f44fcd1b70a7c986a11c5c" dependencies = [ "core2", - "hashbrown 0.13.1", + "hashbrown 0.16.0", "rle-decode-fast", ] @@ -5929,6 +6244,16 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link 0.2.0", +] + [[package]] name = "libm" version = "0.2.8" @@ -5941,7 +6266,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.10.0", "libc", ] @@ -5990,6 +6315,15 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "line-clipping" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f4de44e98ddbf09375cbf4d17714d18f39195f4f4894e8524501726fd9a8a4a" +dependencies = [ + "bitflags 2.10.0", +] + [[package]] name = "linked-hash-map" version = "0.5.6" @@ -6064,9 +6398,9 @@ checksum = "3a69c0481fc2424cb55795de7da41add33372ea75a94f9b6588ab6a2826dfebc" [[package]] name = "log" -version = "0.4.27" +version = "0.4.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" +checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" [[package]] name = "loki-logproto" @@ -6089,9 +6423,12 @@ dependencies = [ [[package]] name = "lru" -version = "0.16.0" +version = "0.16.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86ea4e65087ff52f3862caff188d489f1fab49a0cb09e01b2e3f1a617b10aaed" +checksum = "a1dc47f592c06f33f8e3aea9591776ec7c9f9e4124778ff8a3c3b87159f7e593" +dependencies = [ + "hashbrown 0.16.0", +] [[package]] name = "lru-cache" @@ -6164,6 +6501,54 @@ dependencies = [ "libc", ] +[[package]] +name = "macro_magic" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc33f9f0351468d26fbc53d9ce00a096c8522ecb42f19b50f34f2c422f76d21d" +dependencies = [ + "macro_magic_core", + "macro_magic_macros", + "quote 1.0.40", + "syn 2.0.106", +] + +[[package]] +name = "macro_magic_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1687dc887e42f352865a393acae7cf79d98fab6351cde1f58e9e057da89bf150" +dependencies = [ + "const-random", + "derive-syn-parse", + "macro_magic_core_macros", + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", +] + +[[package]] +name = "macro_magic_core_macros" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b02abfe41815b5bd98dbd4260173db2c116dda171dc0fe7838cb206333b83308" +dependencies = [ + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", +] + +[[package]] +name = "macro_magic_macros" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ea28ee64b88876bf45277ed9a5817c1817df061a74f2b988971a12570e5869" +dependencies = [ + "macro_magic_core", + "quote 1.0.40", + "syn 2.0.106", +] + [[package]] name = "malloc_buf" version = "0.0.6" @@ -6173,6 +6558,21 @@ dependencies = [ "libc", ] +[[package]] +name = "marrow" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea734fcb7619dfcc47a396f7bf0c72571ccc8c18ae7236ae028d485b27424b74" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "bytemuck", + "half", + "serde", +] + [[package]] name = "match_cfg" version = "0.1.0" @@ -6188,12 +6588,6 @@ dependencies = [ "regex-automata 0.4.8", ] -[[package]] -name = "matches" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" - [[package]] name = "matchit" version = "0.7.3" @@ -6212,16 +6606,16 @@ dependencies = [ [[package]] name = "maxminddb" -version = "0.26.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a197e44322788858682406c74b0b59bf8d9b4954fe1f224d9a25147f1880bba" +checksum = "7ef0551fc3e7345a6c854c1026b0ddada1e443e51f4fb4cdcf86cc1a71d4b337" dependencies = [ "ipnetwork", "log", "memchr", "serde", "simdutf8", - "thiserror 2.0.3", + "thiserror 2.0.17", ] [[package]] @@ -6292,7 +6686,7 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1ada651cd6bdffe01e5f35067df53491f1fe853d2b154008ca2bd30b3d3fcf6" dependencies = [ - "indexmap 2.11.0", + "indexmap 2.12.0", "itoa", "lockfree-object-pool", "metrics", @@ -6313,7 +6707,7 @@ dependencies = [ "crossbeam-epoch", "crossbeam-utils", "hashbrown 0.15.2", - "indexmap 2.11.0", + "indexmap 2.12.0", "metrics", "ordered-float 4.6.0", "quanta", @@ -6345,9 +6739,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.8.8" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3be647b768db090acb35d5ec5db2b0e1f1de11133ca123b9eacf5137868f892a" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" dependencies = [ "adler2", ] @@ -6361,7 +6755,7 @@ dependencies = [ "hermit-abi 0.3.9", "libc", "log", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi", "windows-sys 0.52.0", ] @@ -6431,7 +6825,7 @@ dependencies = [ "once_cell", "parking_lot 0.12.4", "quanta", - "rustc_version 0.4.1", + "rustc_version", "smallvec", "tagptr", "thiserror 1.0.68", @@ -6439,51 +6833,84 @@ dependencies = [ "uuid", ] +[[package]] +name = "mongocrypt" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22426d6318d19c5c0773f783f85375265d6a8f0fa76a733da8dc4355516ec63d" +dependencies = [ + "bson", + "mongocrypt-sys", + "once_cell", + "serde", +] + +[[package]] +name = "mongocrypt-sys" +version = "0.1.4+1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dda42df21d035f88030aad8e877492fac814680e1d7336a57b2a091b989ae388" + [[package]] name = "mongodb" -version = "2.8.2" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef206acb1b72389b49bc9985efe7eb1f8a9bb18e5680d262fac26c07f44025f1" +checksum = "622f272c59e54a3c85f5902c6b8e7b1653a6b6681f45e4c42d6581301119a4b8" dependencies = [ "async-trait", "base64 0.13.1", "bitflags 1.3.2", "bson", "chrono", - "derivative", - "derive_more", + "derive-where", + "derive_more 0.99.17", "futures-core", "futures-executor", "futures-io", "futures-util", "hex", + "hickory-proto 0.24.4", + "hickory-resolver", "hmac", - "lazy_static", + "macro_magic", "md-5", + "mongocrypt", + "mongodb-internal-macros", + "once_cell", "pbkdf2", "percent-encoding", "rand 0.8.5", "rustc_version_runtime", - "rustls 0.21.12", - "rustls-pemfile 1.0.4", + "rustls 0.23.23", + "rustversion", "serde", "serde_bytes", - "serde_with 1.14.0", - "sha-1", + "serde_with", + "sha1", "sha2", - "socket2 0.4.10", + "socket2 0.5.10", "stringprep", - "strsim 0.10.0", + "strsim", "take_mut", "thiserror 1.0.68", "tokio", - "tokio-rustls 0.24.1", + "tokio-rustls 0.26.2", "tokio-util", - "trust-dns-proto", - "trust-dns-resolver", - "typed-builder 0.10.0", + "typed-builder 0.20.1", "uuid", - "webpki-roots 0.25.2", + "webpki-roots 0.26.1", +] + +[[package]] +name = "mongodb-internal-macros" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63981427a0f26b89632fd2574280e069d09fb2912a3138da15de0174d11dd077" +dependencies = [ + "macro_magic", + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", ] [[package]] @@ -6548,7 +6975,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17ebbe97acce52d06aebed4cd4a87c0941f4b2519b59b82b4feb5bd0ce003dfd" dependencies = [ - "indexmap 2.11.0", + "indexmap 2.12.0", "itertools 0.13.0", "ndarray", "noisy_float", @@ -6669,7 +7096,7 @@ version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.10.0", "cfg-if", "cfg_aliases", "libc", @@ -6755,7 +7182,7 @@ version = "8.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d3d07927151ff8575b7087f245456e549fea62edf0ec4e565a5ee50c8402bc3" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.10.0", "fsevent-sys", "inotify", "kqueue", @@ -6830,6 +7257,7 @@ checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" dependencies = [ "num-integer", "num-traits", + "serde", ] [[package]] @@ -6994,12 +7422,6 @@ dependencies = [ "libc", ] -[[package]] -name = "number_prefix" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" - [[package]] name = "oauth2" version = "4.4.2" @@ -7035,7 +7457,7 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c10c2894a6fed806ade6027bcd50662746363a9589d3ec9d9bef30a4e4bc166" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.10.0", ] [[package]] @@ -7048,15 +7470,6 @@ dependencies = [ "objc2-core-foundation", ] -[[package]] -name = "object" -version = "0.36.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" -dependencies = [ - "memchr", -] - [[package]] name = "octseq" version = "0.5.2" @@ -7093,7 +7506,7 @@ version = "6.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "336b9c63443aceef14bea841b899035ae3abe89b7c486aaf4c5bd8aafedac3f0" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.10.0", "libc", "once_cell", "onig_sys", @@ -7140,7 +7553,7 @@ dependencies = [ "md-5", "percent-encoding", "quick-xml 0.37.4", - "reqwest 0.12.9", + "reqwest 0.12.28", "serde", "serde_json", "tokio", @@ -7172,7 +7585,7 @@ dependencies = [ "serde_json", "serde_path_to_error", "serde_plain", - "serde_with 3.14.0", + "serde_with", "sha2", "subtle", "thiserror 1.0.68", @@ -7185,7 +7598,7 @@ version = "0.10.73" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8505734d46c8ab1e19a1dce3aef597ad87dcb4c37e7188231769bd6bd51cebf8" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.10.0", "cfg-if", "foreign-types", "libc", @@ -7498,25 +7911,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" dependencies = [ "fixedbitset", - "indexmap 2.11.0", + "indexmap 2.12.0", ] [[package]] name = "phf" -version = "0.11.2" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" +checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" dependencies = [ - "phf_shared 0.11.2", + "phf_shared 0.12.1", ] [[package]] name = "phf" -version = "0.12.1" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" +checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" dependencies = [ - "phf_shared 0.12.1", + "phf_shared 0.13.1", + "serde", ] [[package]] @@ -7530,18 +7944,18 @@ dependencies = [ [[package]] name = "phf_shared" -version = "0.11.2" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" +checksum = "06005508882fb681fd97892ecff4b7fd0fee13ef1aa569f8695dae7ab9099981" dependencies = [ - "siphasher 0.3.11", + "siphasher 1.0.1", ] [[package]] name = "phf_shared" -version = "0.12.1" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06005508882fb681fd97892ecff4b7fd0fee13ef1aa569f8695dae7ab9099981" +checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" dependencies = [ "siphasher 1.0.1", ] @@ -7624,9 +8038,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.27" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" [[package]] name = "platforms" @@ -7719,13 +8133,6 @@ dependencies = [ "portable-atomic", ] -[[package]] -name = "portpicker" -version = "1.0.0" -dependencies = [ - "rand 0.9.2", -] - [[package]] name = "postgres-openssl" version = "0.5.1" @@ -7740,9 +8147,9 @@ dependencies = [ [[package]] name = "postgres-protocol" -version = "0.6.8" +version = "0.6.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76ff0abab4a9b844b93ef7b81f1efc0a366062aaef2cd702c76256b5dc075c54" +checksum = "3ee9dd5fe15055d2b6806f4736aa0c9637217074e224bbec46d4041b91bb9491" dependencies = [ "base64 0.22.1", "byteorder", @@ -7758,9 +8165,9 @@ dependencies = [ [[package]] name = "postgres-types" -version = "0.2.9" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613283563cd90e1dfc3518d548caee47e0e725455ed619881f5cf21f36de4b48" +checksum = "54b858f82211e84682fecd373f68e1ceae642d8d751a1ebd13f33de6257b3e20" dependencies = [ "bytes 1.10.1", "chrono", @@ -7941,7 +8348,7 @@ dependencies = [ name = "prometheus-parser" version = "0.1.0" dependencies = [ - "indexmap 2.11.0", + "indexmap 2.12.0", "nom 8.0.0", "prost 0.12.6", "prost-build 0.12.6", @@ -7958,7 +8365,7 @@ checksum = "2bb0be07becd10686a0bb407298fb425360a5c44a663774406340c59a22de4ce" dependencies = [ "bit-set", "bit-vec", - "bitflags 2.9.0", + "bitflags 2.10.0", "lazy_static", "num-traits", "rand 0.9.2", @@ -8195,7 +8602,7 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f42ea446cab60335f76979ec15e12619a2165b5ae2c12166bef27d283a9fadf" dependencies = [ - "idna 1.0.3", + "idna", "psl-types", ] @@ -8238,9 +8645,9 @@ dependencies = [ [[package]] name = "quad-rand" -version = "0.2.1" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "658fa1faf7a4cc5f057c9ee5ef560f717ad9d8dc66d975267f709624d6e1ab88" +checksum = "5a651516ddc9168ebd67b24afd085a718be02f8858fe406591b013d101ce2f40" [[package]] name = "quanta" @@ -8252,7 +8659,7 @@ dependencies = [ "libc", "once_cell", "raw-cpuid", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi", "web-sys", "winapi", ] @@ -8270,19 +8677,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ed1a693391a16317257103ad06a88c6529ac640846021da7c435a06fffdacd7" dependencies = [ "chrono", - "indexmap 2.11.0", + "indexmap 2.12.0", "newtype-uuid", "quick-xml 0.37.4", "strip-ansi-escapes", - "thiserror 2.0.3", + "thiserror 2.0.17", "uuid", ] [[package]] name = "quick-xml" -version = "0.31.0" +version = "0.37.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33" +checksum = "a4ce8c88de324ff838700f36fb6ab86c96df0e3c4ab6ef3a9b2044465cce1369" dependencies = [ "memchr", "serde", @@ -8290,9 +8697,9 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.37.4" +version = "0.38.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4ce8c88de324ff838700f36fb6ab86c96df0e3c4ab6ef3a9b2044465cce1369" +checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c" dependencies = [ "memchr", "serde", @@ -8333,7 +8740,7 @@ dependencies = [ "rustc-hash", "rustls 0.23.23", "socket2 0.5.10", - "thiserror 2.0.3", + "thiserror 2.0.17", "tokio", "tracing 0.1.41", ] @@ -8352,7 +8759,7 @@ dependencies = [ "rustls 0.23.23", "rustls-pki-types", "slab", - "thiserror 2.0.3", + "thiserror 2.0.17", "tinyvec", "tracing 0.1.41", "web-time", @@ -8396,6 +8803,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79ec282e887b434b68c18fe5c121d38e72a5cf35119b59e54ec5b992ea9c8eb0" +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + [[package]] name = "radium" version = "0.7.0" @@ -8412,19 +8825,6 @@ dependencies = [ "nibble_vec", ] -[[package]] -name = "rand" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" -dependencies = [ - "getrandom 0.1.16", - "libc", - "rand_chacha 0.2.2", - "rand_core 0.5.1", - "rand_hc", -] - [[package]] name = "rand" version = "0.8.5" @@ -8446,16 +8846,6 @@ dependencies = [ "rand_core 0.9.0", ] -[[package]] -name = "rand_chacha" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" -dependencies = [ - "ppv-lite86", - "rand_core 0.5.1", -] - [[package]] name = "rand_chacha" version = "0.3.1" @@ -8476,15 +8866,6 @@ dependencies = [ "rand_core 0.9.0", ] -[[package]] -name = "rand_core" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" -dependencies = [ - "getrandom 0.1.16", -] - [[package]] name = "rand_core" version = "0.6.4" @@ -8500,7 +8881,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b08f3c9802962f7e1b25113931d94f43ed9725bebc59db9d0c3e9a23b67e15ff" dependencies = [ - "getrandom 0.3.1", + "getrandom 0.3.4", "zerocopy 0.8.16", ] @@ -8514,15 +8895,6 @@ dependencies = [ "rand 0.9.2", ] -[[package]] -name = "rand_hc" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" -dependencies = [ - "rand_core 0.5.1", -] - [[package]] name = "rand_xorshift" version = "0.4.0" @@ -8534,32 +8906,74 @@ dependencies = [ [[package]] name = "ratatui" -version = "0.29.0" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1ce67fb8ba4446454d1c8dbaeda0557ff5e94d39d5e5ed7f10a65eb4c8266bc" +dependencies = [ + "instability", + "ratatui-core", + "ratatui-crossterm", + "ratatui-widgets", +] + +[[package]] +name = "ratatui-core" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eabd94c2f37801c20583fc49dd5cd6b0ba68c716787c2dd6ed18571e1e63117b" +checksum = "5ef8dea09a92caaf73bff7adb70b76162e5937524058a7e5bff37869cbbec293" dependencies = [ - "bitflags 2.9.0", - "cassowary", + "bitflags 2.10.0", "compact_str", - "crossterm 0.28.1", + "hashbrown 0.16.0", "indoc", - "instability", - "itertools 0.13.0", - "lru 0.12.5", - "paste", - "strum 0.26.3", + "itertools 0.14.0", + "kasuari", + "lru 0.16.3", + "strum 0.27.2", + "thiserror 2.0.17", "unicode-segmentation", "unicode-truncate", "unicode-width 0.2.0", ] +[[package]] +name = "ratatui-crossterm" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "577c9b9f652b4c121fb25c6a391dd06406d3b092ba68827e6d2f09550edc54b3" +dependencies = [ + "cfg-if", + "crossterm", + "instability", + "ratatui-core", +] + +[[package]] +name = "ratatui-widgets" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7dbfa023cd4e604c2553483820c5fe8aa9d71a42eea5aa77c6e7f35756612db" +dependencies = [ + "bitflags 2.10.0", + "hashbrown 0.16.0", + "indoc", + "instability", + "itertools 0.14.0", + "line-clipping", + "ratatui-core", + "strum 0.27.2", + "time", + "unicode-segmentation", + "unicode-width 0.2.0", +] + [[package]] name = "raw-cpuid" version = "11.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d86a7c4638d42c44551f4791a20e687dbb4c3de1f33c43dd71e355cd429def1" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.10.0", ] [[package]] @@ -8596,9 +9010,9 @@ dependencies = [ [[package]] name = "rdkafka" -version = "0.37.0" +version = "0.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14b52c81ac3cac39c9639b95c20452076e74b8d9a71bc6fc4d83407af2ea6fff" +checksum = "5f1856d72dbbbea0d2a5b2eaf6af7fb3847ef2746e883b11781446a51dbc85c0" dependencies = [ "futures-channel", "futures-util", @@ -8614,9 +9028,9 @@ dependencies = [ [[package]] name = "rdkafka-sys" -version = "4.8.0+2.3.0" +version = "4.9.0+2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ced38182dc436b3d9df0c77976f37a67134df26b050df1f0006688e46fc4c8be" +checksum = "5230dca48bc354d718269f3e4353280e188b610f7af7e2fcf54b7a79d5802872" dependencies = [ "cmake", "curl-sys", @@ -8691,7 +9105,7 @@ version = "0.5.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "928fca9cf2aa042393a8325b9ead81d2f0df4cb12e1e24cef072922ccd99c5af" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.10.0", ] [[package]] @@ -8713,7 +9127,7 @@ checksum = "dd6f9d3d47bdd2ad6945c5015a226ec6155d0bcdfd8f7cd29f86b71f8de99d2b" dependencies = [ "getrandom 0.2.15", "libredox", - "thiserror 2.0.3", + "thiserror 2.0.17", ] [[package]] @@ -8738,13 +9152,14 @@ dependencies = [ [[package]] name = "referencing" -version = "0.32.1" +version = "0.38.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a3d769362109497b240e66462606bc28af68116436c8669bac17069533b908e" +checksum = "15a8af0c6bb8eaf8b07cb06fc31ff30ca6fe19fb99afa476c276d8b24f365b0b" dependencies = [ "ahash 0.8.11", - "fluent-uri 0.3.2", - "once_cell", + "fluent-uri 0.4.1", + "getrandom 0.3.4", + "hashbrown 0.16.0", "parking_lot 0.12.4", "percent-encoding", "serde_json", @@ -8786,7 +9201,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c11639076bf147be211b90e47790db89f4c22b6c8a9ca6e960833869da67166" dependencies = [ "aho-corasick", - "indexmap 2.11.0", + "indexmap 2.12.0", "itertools 0.13.0", "nohash", "regex", @@ -8795,9 +9210,9 @@ dependencies = [ [[package]] name = "regex-lite" -version = "0.1.5" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30b661b2f27137bdbc16f00eda72866a92bb28af1753ffbd56744fb6e2e9cd8e" +checksum = "8d942b98df5e658f56f20d592c7f868833fe38115e65c33003d8cd224b0155da" [[package]] name = "regex-syntax" @@ -8833,8 +9248,8 @@ dependencies = [ "futures-util", "h2 0.3.26", "http 0.2.9", - "http-body 0.4.5", - "hyper 0.14.28", + "http-body 0.4.6", + "hyper 0.14.32", "hyper-rustls 0.24.2", "hyper-tls 0.5.0", "ipnet", @@ -8851,7 +9266,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "sync_wrapper 0.1.2", - "system-configuration", + "system-configuration 0.5.1", "tokio", "tokio-native-tls", "tokio-rustls 0.24.1", @@ -8866,15 +9281,15 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.12.9" +version = "0.12.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a77c62af46e79de0a562e1a9849205ffcb7fc1238876e9bd743357570e04046f" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ - "async-compression", "base64 0.22.1", "bytes 1.10.1", "cookie", "cookie_store", + "encoding_rs", "futures-channel", "futures-core", "futures-util", @@ -8886,19 +9301,16 @@ dependencies = [ "hyper-rustls 0.27.5", "hyper-tls 0.6.0", "hyper-util", - "ipnet", "js-sys", "log", "mime", "mime_guess", "native-tls", - "once_cell", "percent-encoding", "pin-project-lite", "quinn", "rustls 0.23.23", "rustls-native-certs 0.8.1", - "rustls-pemfile 2.1.0", "rustls-pki-types", "serde", "serde_json", @@ -8908,14 +9320,15 @@ dependencies = [ "tokio-native-tls", "tokio-rustls 0.26.2", "tokio-util", + "tower 0.5.2", + "tower-http 0.6.8", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots 0.26.1", - "windows-registry", + "webpki-roots 1.0.4", ] [[package]] @@ -8927,7 +9340,7 @@ dependencies = [ "anyhow", "async-trait", "http 1.3.1", - "reqwest 0.12.9", + "reqwest 0.12.28", "serde", "thiserror 1.0.68", "tower-service", @@ -8946,7 +9359,7 @@ dependencies = [ "http 1.3.1", "hyper 1.7.0", "parking_lot 0.11.2", - "reqwest 0.12.9", + "reqwest 0.12.28", "reqwest-middleware", "retry-policies", "thiserror 1.0.68", @@ -8956,13 +9369,9 @@ dependencies = [ [[package]] name = "resolv-conf" -version = "0.7.0" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52e44394d2086d010551b14b53b1f24e31647570cd1deb0379e2c21b329aba00" -dependencies = [ - "hostname 0.3.1", - "quick-error", -] +checksum = "6b3789b30bd25ba102de4beabd95d21ac45b69b1be7d14522bab988c526d6799" [[package]] name = "retry-policies" @@ -8999,9 +9408,9 @@ dependencies = [ [[package]] name = "rkyv" -version = "0.7.45" +version = "0.7.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9008cd6385b9e161d8229e1f6549dd23c3d022f132a2ea37ac3a10ac4935779b" +checksum = "2297bf9c81a3f0dc96bc9521370b88f054168c29826a75e89c55ff196e7ed6a1" dependencies = [ "bitvec", "bytecheck", @@ -9017,9 +9426,9 @@ dependencies = [ [[package]] name = "rkyv_derive" -version = "0.7.45" +version = "0.7.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "503d1d27590a2b0a3a4ca4c94755aa2875657196ecbf401a42eff41d7de532c0" +checksum = "84d7b42d4b8d06048d3ac8db0eb31bcb942cbeb709f0b5f2b2ebde398d3038f5" dependencies = [ "proc-macro2 1.0.101", "quote 1.0.40", @@ -9078,9 +9487,12 @@ dependencies = [ [[package]] name = "roxmltree" -version = "0.20.0" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c20b6793b5c2fa6553b250154b78d6d0db37e72700ae35fad9387a46f487c97" +checksum = "f1964b10c76125c36f8afe190065a4bf9a87bf324842c05701330bba9f1cacbb" +dependencies = [ + "memchr", +] [[package]] name = "rsa" @@ -9126,7 +9538,7 @@ dependencies = [ "quote 1.0.40", "regex", "relative-path", - "rustc_version 0.4.1", + "rustc_version", "syn 2.0.106", "unicode-ident", ] @@ -9151,34 +9563,25 @@ dependencies = [ [[package]] name = "rust_decimal" -version = "1.33.1" +version = "1.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06676aec5ccb8fc1da723cc8c0f9a46549f21ebb8753d3915c6c41db1e7f1dc4" +checksum = "35affe401787a9bd846712274d97654355d21b2a2c092a3139aabe31e9022282" dependencies = [ "arrayvec", + "borsh", + "bytes 1.10.1", "num-traits", + "rand 0.8.5", + "rkyv", + "serde", + "serde_json", ] -[[package]] -name = "rustc-demangle" -version = "0.1.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace" - [[package]] name = "rustc-hash" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" - -[[package]] -name = "rustc_version" -version = "0.2.3" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" -dependencies = [ - "semver 0.9.0", -] +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" [[package]] name = "rustc_version" @@ -9186,17 +9589,17 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" dependencies = [ - "semver 1.0.26", + "semver", ] [[package]] name = "rustc_version_runtime" -version = "0.2.1" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d31b7153270ebf48bf91c65ae5b0c00e749c4cfad505f66530ac74950249582f" +checksum = "2dd18cd2bae1820af0b6ad5e54f4a51d0f3fcc53b05f845675074efcc7af071d" dependencies = [ - "rustc_version 0.2.3", - "semver 0.9.0", + "rustc_version", + "semver", ] [[package]] @@ -9219,7 +9622,7 @@ version = "0.38.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "99e4ea3e1cdc4b559b8e5650f9c8e5998e3e5c1343b4eaf034565f32318d63c0" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.10.0", "errno", "libc", "linux-raw-sys 0.4.14", @@ -9232,7 +9635,7 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dade4812df5c384711475be5fcd8c162555352945401aed22a35bffeab61f657" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.10.0", "errno", "libc", "linux-raw-sys 0.9.2", @@ -9271,6 +9674,7 @@ version = "0.23.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47796c98c480fce5406ef69d1c76378375492c3b0a0de587be0c1d9feb12f395" dependencies = [ + "log", "once_cell", "ring", "rustls-pki-types", @@ -9367,9 +9771,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.14" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" [[package]] name = "rusty-fork" @@ -9385,11 +9789,11 @@ dependencies = [ [[package]] name = "rustyline" -version = "16.0.0" +version = "17.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62fd9ca5ebc709e8535e8ef7c658eb51457987e48c98ead2be482172accc408d" +checksum = "e902948a25149d50edc1a8e0141aad50f54e22ba83ff988cf8f7c9ef07f50564" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.10.0", "cfg-if", "clipboard-win", "libc", @@ -9399,7 +9803,7 @@ dependencies = [ "unicode-segmentation", "unicode-width 0.2.0", "utf8parse", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -9558,7 +9962,7 @@ version = "3.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3297343eaf830f66ede390ea39da1d462b6b0c1b000f420d0a83f898bbbe6ef" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.10.0", "core-foundation 0.10.1", "core-foundation-sys", "libc", @@ -9575,15 +9979,6 @@ dependencies = [ "libc", ] -[[package]] -name = "semver" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" -dependencies = [ - "semver-parser", -] - [[package]] name = "semver" version = "1.0.26" @@ -9594,18 +9989,25 @@ dependencies = [ ] [[package]] -name = "semver-parser" -version = "0.7.0" +name = "serde" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] [[package]] -name = "serde" -version = "1.0.219" +name = "serde-aux" +version = "4.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +checksum = "207f67b28fe90fb596503a9bf0bf1ea5e831e21307658e177c5dfcdfc3ab8a0a" dependencies = [ - "serde_derive", + "chrono", + "serde", + "serde-value", + "serde_json", ] [[package]] @@ -9614,7 +10016,7 @@ version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fc44799282f511a5d403d72a4ff028dc2c87f7fe6830abe3c33bb2fa6dfccec" dependencies = [ - "toml 0.9.5", + "toml 0.9.8", ] [[package]] @@ -9627,20 +10029,45 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_arrow" +version = "0.13.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "038967a6dda16f5c6ca5b6e1afec9cd2361d39f0db681ca338ac5f0ccece6469" +dependencies = [ + "arrow-array", + "arrow-schema", + "bytemuck", + "chrono", + "half", + "marrow", + "serde", +] + [[package]] name = "serde_bytes" -version = "0.11.17" +version = "0.11.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8437fd221bde2d4ca316d61b90e337e9e702b3820b87d63caa9ba6c02bd06d96" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" dependencies = [ "serde", + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.219" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2 1.0.101", "quote 1.0.40", @@ -9660,15 +10087,16 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.143" +version = "1.0.145" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d401abef1d108fbd9cbaebc3e46611f4b1021f714a0597a71f41ee463f5f4a5a" +checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" dependencies = [ - "indexmap 2.11.0", + "indexmap 2.12.0", "itoa", "memchr", "ryu", "serde", + "serde_core", ] [[package]] @@ -9699,17 +10127,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_qs" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7715380eec75f029a4ef7de39a9200e0a63823176b759d055b613f5a87df6a6" -dependencies = [ - "percent-encoding", - "serde", - "thiserror 1.0.68", -] - [[package]] name = "serde_repr" version = "0.1.17" @@ -9732,11 +10149,11 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "1.0.0" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40734c41988f7306bb04f0ecf60ec0f3f1caa34290e4e8ea471dcd3346483b83" +checksum = "e24345aa0fe688594e73770a5f6d1b216508b4f93484c0026d521acd30134392" dependencies = [ - "serde", + "serde_core", ] [[package]] @@ -9751,16 +10168,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_with" -version = "1.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "678b5a069e50bf00ecd22d0cd8ddf7c236f68581b03db652061ed5eb13a312ff" -dependencies = [ - "serde", - "serde_with_macros 1.5.2", -] - [[package]] name = "serde_with" version = "3.14.0" @@ -9771,26 +10178,14 @@ dependencies = [ "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.11.0", + "indexmap 2.12.0", "schemars 0.9.0", "schemars 1.0.3", - "serde", - "serde_derive", - "serde_json", - "serde_with_macros 3.14.0", - "time", -] - -[[package]] -name = "serde_with_macros" -version = "1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e182d6ec6f05393cc0e5ed1bf81ad6db3a8feedf8ee515ecdd369809bcce8082" -dependencies = [ - "darling 0.13.4", - "proc-macro2 1.0.101", - "quote 1.0.40", - "syn 1.0.109", + "serde", + "serde_derive", + "serde_json", + "serde_with_macros", + "time", ] [[package]] @@ -9811,7 +10206,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.11.0", + "indexmap 2.12.0", "itoa", "ryu", "serde", @@ -10211,20 +10606,22 @@ dependencies = [ "futures-util", "hashbrown 0.15.2", "hashlink", - "indexmap 2.11.0", + "indexmap 2.12.0", "log", "memchr", "once_cell", "percent-encoding", + "rustls 0.23.23", "serde", "serde_json", "sha2", "smallvec", - "thiserror 2.0.3", + "thiserror 2.0.17", "tokio", "tokio-stream", "tracing 0.1.41", "url", + "webpki-roots 0.26.1", ] [[package]] @@ -10273,7 +10670,7 @@ checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526" dependencies = [ "atoi", "base64 0.22.1", - "bitflags 2.9.0", + "bitflags 2.10.0", "byteorder", "bytes 1.10.1", "chrono", @@ -10303,7 +10700,7 @@ dependencies = [ "smallvec", "sqlx-core", "stringprep", - "thiserror 2.0.3", + "thiserror 2.0.17", "tracing 0.1.41", "whoami", ] @@ -10316,7 +10713,7 @@ checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" dependencies = [ "atoi", "base64 0.22.1", - "bitflags 2.9.0", + "bitflags 2.10.0", "byteorder", "chrono", "crc", @@ -10341,7 +10738,7 @@ dependencies = [ "smallvec", "sqlx-core", "stringprep", - "thiserror 2.0.3", + "thiserror 2.0.17", "tracing 0.1.41", "whoami", ] @@ -10366,7 +10763,7 @@ dependencies = [ "serde", "serde_urlencoded", "sqlx-core", - "thiserror 2.0.3", + "thiserror 2.0.17", "tracing 0.1.41", "url", ] @@ -10433,12 +10830,6 @@ dependencies = [ "vte", ] -[[package]] -name = "strsim" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" - [[package]] name = "strsim" version = "0.11.1" @@ -10460,6 +10851,15 @@ dependencies = [ "strum_macros 0.26.4", ] +[[package]] +name = "strum" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" +dependencies = [ + "strum_macros 0.27.2", +] + [[package]] name = "strum_macros" version = "0.25.3" @@ -10486,6 +10886,18 @@ dependencies = [ "syn 2.0.106", ] +[[package]] +name = "strum_macros" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +dependencies = [ + "heck 0.5.0", + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", +] + [[package]] name = "subtle" version = "2.5.0" @@ -10625,7 +11037,18 @@ checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" dependencies = [ "bitflags 1.3.2", "core-foundation 0.9.3", - "system-configuration-sys", + "system-configuration-sys 0.5.0", +] + +[[package]] +name = "system-configuration" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" +dependencies = [ + "bitflags 2.10.0", + "core-foundation 0.9.3", + "system-configuration-sys 0.6.0", ] [[package]] @@ -10638,6 +11061,16 @@ dependencies = [ "libc", ] +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tagptr" version = "0.2.0" @@ -10680,10 +11113,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16" dependencies = [ "fastrand 2.3.0", - "getrandom 0.3.1", + "getrandom 0.3.4", "once_cell", "rustix 1.0.1", - "windows-sys 0.60.2", + "windows-sys 0.61.0", ] [[package]] @@ -10755,11 +11188,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.3" +version = "2.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c006c85c7651b3cf2ada4584faa36773bd07bac24acfb39f3c431b36d7e667aa" +checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" dependencies = [ - "thiserror-impl 2.0.3", + "thiserror-impl 2.0.17", ] [[package]] @@ -10775,9 +11208,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.3" +version = "2.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f077553d607adc1caf65430528a576c757a71ed73944b66ebb58ef2bbd243568" +checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" dependencies = [ "proc-macro2 1.0.101", "quote 1.0.40", @@ -10815,9 +11248,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.36" +version = "0.3.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" +checksum = "91e7d9e3bb61134e77bde20dd4825b97c010155709965fedf0f49bb138e52a9d" dependencies = [ "deranged", "itoa", @@ -10833,20 +11266,29 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.2" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" +checksum = "40868e7c1d2f0b8d73e4a8c7f0ff63af4f6d19be117e90bd73eb1d62cf831c6b" [[package]] name = "time-macros" -version = "0.2.18" +version = "0.2.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" +checksum = "30cfb0125f12d9c277f35663a0a33f8c30190f4e4574868a330595412d34ebf3" dependencies = [ "num-conv", "time-core", ] +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + [[package]] name = "tinystr" version = "0.7.6" @@ -10884,23 +11326,20 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.47.1" +version = "1.49.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89e49afdadebb872d3145a5638b59eb0691ea23e46ca484037cfab3b76b95038" +checksum = "72a2903cd7736441aac9df9d7688bd0ce48edccaadf181c3b90be801e81d3d86" dependencies = [ - "backtrace", "bytes 1.10.1", - "io-uring", "libc", "mio", "parking_lot 0.12.4", "pin-project-lite", "signal-hook-registry", - "slab", "socket2 0.6.0", "tokio-macros", "tracing 0.1.41", - "windows-sys 0.59.0", + "windows-sys 0.61.0", ] [[package]] @@ -10926,9 +11365,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" +checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" dependencies = [ "proc-macro2 1.0.101", "quote 1.0.40", @@ -10958,9 +11397,9 @@ dependencies = [ [[package]] name = "tokio-postgres" -version = "0.7.13" +version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c95d533c83082bb6490e0189acaa0bbeef9084e60471b696ca6988cd0541fb0" +checksum = "2b40d66d9b2cfe04b628173409368e58247e8eddbbd3b0e6c6ba1d09f20f6c9e" dependencies = [ "async-trait", "byteorder", @@ -10971,12 +11410,12 @@ dependencies = [ "log", "parking_lot 0.12.4", "percent-encoding", - "phf 0.11.2", + "phf 0.13.1", "pin-project-lite", "postgres-protocol", "postgres-types", "rand 0.9.2", - "socket2 0.5.10", + "socket2 0.6.0", "tokio", "tokio-util", "whoami", @@ -11026,9 +11465,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" dependencies = [ "futures-core", "pin-project-lite", @@ -11038,12 +11477,10 @@ dependencies = [ [[package]] name = "tokio-test" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2468baabc3311435b55dd935f702f42cd1b8abb7e754fb7dfb16bd36aa88f9f7" +checksum = "3f6d24790a10a7af737693a3e8f1d03faef7e6ca0cc99aae5066f533766de545" dependencies = [ - "async-stream", - "bytes 1.10.1", "futures-core", "tokio", "tokio-stream", @@ -11123,17 +11560,17 @@ dependencies = [ [[package]] name = "toml" -version = "0.9.5" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75129e1dc5000bfbaa9fee9d1b21f974f9fbad9daec557a521ee6e080825f6e8" +checksum = "f0dc8b1fb61449e27716ec0e1bdf0f6b8f3e8f6b05391e8497b8b6d7804ea6d8" dependencies = [ - "indexmap 2.11.0", - "serde", - "serde_spanned 1.0.0", - "toml_datetime 0.7.0", + "indexmap 2.12.0", + "serde_core", + "serde_spanned 1.0.3", + "toml_datetime 0.7.3", "toml_parser", "toml_writer", - "winnow 0.7.10", + "winnow 0.7.13", ] [[package]] @@ -11147,11 +11584,11 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.7.0" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bade1c3e902f58d73d3f294cd7f20391c1cb2fbcb643b73566bc773971df91e3" +checksum = "f2cdb639ebbc97961c51720f858597f7f24c4fc295327923af55b74c3c724533" dependencies = [ - "serde", + "serde_core", ] [[package]] @@ -11160,7 +11597,7 @@ version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap 2.11.0", + "indexmap 2.12.0", "toml_datetime 0.6.11", "winnow 0.5.18", ] @@ -11171,21 +11608,21 @@ version = "0.22.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ - "indexmap 2.11.0", + "indexmap 2.12.0", "serde", "serde_spanned 0.6.9", "toml_datetime 0.6.11", "toml_write", - "winnow 0.7.10", + "winnow 0.7.13", ] [[package]] name = "toml_parser" -version = "1.0.2" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b551886f449aa90d4fe2bdaa9f4a2577ad2dde302c61ecf262d80b116db95c10" +checksum = "c0cbe268d35bdb4bb5a56a2de88d0ad0eb70af5384a99d648cd4b3d04039800e" dependencies = [ - "winnow 0.7.10", + "winnow 0.7.13", ] [[package]] @@ -11196,9 +11633,9 @@ checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" [[package]] name = "toml_writer" -version = "1.0.2" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcc842091f2def52017664b53082ecbbeb5c7731092bad69d2c63050401dfd64" +checksum = "df8b2b54733674ad286d16267dcfc7a71ed5c776e4ac7aa3c3e2561f7c637bf2" [[package]] name = "tonic" @@ -11214,8 +11651,8 @@ dependencies = [ "flate2", "h2 0.3.26", "http 0.2.9", - "http-body 0.4.5", - "hyper 0.14.28", + "http-body 0.4.6", + "hyper 0.14.32", "hyper-timeout 0.4.1", "percent-encoding", "pin-project", @@ -11317,7 +11754,7 @@ checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" dependencies = [ "futures-core", "futures-util", - "indexmap 2.11.0", + "indexmap 2.12.0", "pin-project-lite", "slab", "sync_wrapper 1.0.1", @@ -11335,12 +11772,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61c5bb1d698276a2443e5ecfabc1008bf15a36c12e6a7176e7bf089ea9131140" dependencies = [ "async-compression", - "bitflags 2.9.0", + "bitflags 2.10.0", "bytes 1.10.1", "futures-core", "futures-util", "http 0.2.9", - "http-body 0.4.5", + "http-body 0.4.6", "http-range-header", "pin-project-lite", "tokio", @@ -11357,7 +11794,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5" dependencies = [ "base64 0.21.7", - "bitflags 2.9.0", + "bitflags 2.10.0", "bytes 1.10.1", "http 1.3.1", "http-body 1.0.0", @@ -11369,6 +11806,29 @@ dependencies = [ "tracing 0.1.41", ] +[[package]] +name = "tower-http" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" +dependencies = [ + "async-compression", + "bitflags 2.10.0", + "bytes 1.10.1", + "futures-core", + "futures-util", + "http 1.3.1", + "http-body 1.0.0", + "http-body-util", + "iri-string", + "pin-project-lite", + "tokio", + "tokio-util", + "tower 0.5.2", + "tower-layer", + "tower-service", +] + [[package]] name = "tower-layer" version = "0.3.3" @@ -11571,51 +12031,6 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b2cb4fbb9995eeb36ac86fadf24031ccd58f99d6b4b2d7b911db70bddb80d90" -[[package]] -name = "trust-dns-proto" -version = "0.21.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c31f240f59877c3d4bb3b3ea0ec5a6a0cff07323580ff8c7a605cd7d08b255d" -dependencies = [ - "async-trait", - "cfg-if", - "data-encoding", - "enum-as-inner 0.4.0", - "futures-channel", - "futures-io", - "futures-util", - "idna 0.2.3", - "ipnet", - "lazy_static", - "log", - "rand 0.8.5", - "smallvec", - "thiserror 1.0.68", - "tinyvec", - "tokio", - "url", -] - -[[package]] -name = "trust-dns-resolver" -version = "0.21.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4ba72c2ea84515690c9fcef4c6c660bb9df3036ed1051686de84605b74fd558" -dependencies = [ - "cfg-if", - "futures-util", - "ipconfig", - "lazy_static", - "log", - "lru-cache", - "parking_lot 0.12.4", - "resolv-conf", - "smallvec", - "thiserror 1.0.68", - "tokio", - "trust-dns-proto", -] - [[package]] name = "try-lock" version = "0.2.4" @@ -11678,22 +12093,20 @@ checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" [[package]] name = "typed-builder" -version = "0.10.0" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89851716b67b937e393b3daa8423e67ddfc4bbbf1654bcf05488e95e0828db0c" +checksum = "34085c17941e36627a879208083e25d357243812c30e7d7387c3b954f30ade16" dependencies = [ - "proc-macro2 1.0.101", - "quote 1.0.40", - "syn 1.0.109", + "typed-builder-macro 0.16.2", ] [[package]] name = "typed-builder" -version = "0.16.2" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34085c17941e36627a879208083e25d357243812c30e7d7387c3b954f30ade16" +checksum = "cd9d30e3a08026c78f246b173243cf07b3696d274debd26680773b6773c2afc7" dependencies = [ - "typed-builder-macro", + "typed-builder-macro 0.20.1", ] [[package]] @@ -11707,6 +12120,17 @@ dependencies = [ "syn 2.0.106", ] +[[package]] +name = "typed-builder-macro" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c36781cc0e46a83726d9879608e4cf6c2505237e263a8eb8c24502989cfdb28" +dependencies = [ + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", +] + [[package]] name = "typenum" version = "1.17.0" @@ -11715,11 +12139,14 @@ checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "typespec" -version = "0.5.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04c7a952f1f34257f945fc727b20defe7a3c01c05ddd42925977626cfa6e62ab" +checksum = "44f91ea93fdd5fd4985fcc0a197ed8e8da18705912bef63c9b9b3148d6f35510" dependencies = [ "base64 0.22.1", + "bytes 1.10.1", + "futures 0.3.31", + "quick-xml 0.38.4", "serde", "serde_json", "url", @@ -11727,19 +12154,18 @@ dependencies = [ [[package]] name = "typespec_client_core" -version = "0.4.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5879ce67ba9e525fe088c882ede1337c32c3f80e83e72d9fd3cc6c8e05bcb3d7" +checksum = "1a0f6f7345c3389663551a64fc4dca78fa9689ece758c5ca76e82d6da69349dc" dependencies = [ "async-trait", "base64 0.22.1", - "bytes 1.10.1", "dyn-clone", "futures 0.3.31", - "getrandom 0.2.15", + "getrandom 0.3.4", "pin-project", - "rand 0.8.5", - "reqwest 0.12.9", + "rand 0.9.2", + "reqwest 0.12.28", "serde", "serde_json", "time", @@ -11753,13 +12179,13 @@ dependencies = [ [[package]] name = "typespec_macros" -version = "0.4.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cbccdbe531c8d553812a609bdb70c0d1002ad91333498e18df42c98744b15cc" +checksum = "6ecee5b05c459ea4cd97df7685db58699c32e070465f88dc806d7c98a5088edc" dependencies = [ "proc-macro2 1.0.101", "quote 1.0.40", - "rustc_version 0.4.1", + "rustc_version", "syn 2.0.106", ] @@ -11825,6 +12251,12 @@ version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" +[[package]] +name = "unicode-general-category" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b993bddc193ae5bd0d623b49ec06ac3e9312875fdae725a975c51db1cc1677f" + [[package]] name = "unicode-ident" version = "1.0.13" @@ -11848,12 +12280,13 @@ checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" [[package]] name = "unicode-truncate" -version = "1.0.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a5fbabedabe362c618c714dbefda9927b5afc8e2a8102f47f081089a9019226" +checksum = "8fbf03860ff438702f3910ca5f28f8dac63c1c11e7efb5012b8b175493606330" dependencies = [ - "itertools 0.12.1", - "unicode-width 0.1.13", + "itertools 0.13.0", + "unicode-segmentation", + "unicode-width 0.2.0", ] [[package]] @@ -11882,9 +12315,9 @@ checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" [[package]] name = "unit-prefix" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "323402cff2dd658f39ca17c789b502021b3f18707c91cdf22e3838e1b4023817" +checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3" [[package]] name = "universal-hash" @@ -11935,7 +12368,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" dependencies = [ "form_urlencoded", - "idna 1.0.3", + "idna", "percent-encoding", "serde", ] @@ -11982,7 +12415,7 @@ version = "1.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2" dependencies = [ - "getrandom 0.3.1", + "getrandom 0.3.4", "js-sys", "rand 0.9.2", "serde", @@ -11996,7 +12429,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23b082222b4f6619906941c17eb2297fff4c2fb96cb60164170522942a200bd8" dependencies = [ "outref", - "uuid", "vsimd", ] @@ -12022,13 +12454,11 @@ dependencies = [ "clap", "clap-verbosity-flag", "clap_complete", - "confy", "directories", - "dunce", "git2", "glob", "hex", - "indexmap 2.11.0", + "indexmap 2.12.0", "indicatif", "indoc", "itertools 0.14.0", @@ -12037,23 +12467,26 @@ dependencies = [ "paste", "regex", "reqwest 0.11.26", - "semver 1.0.26", + "semver", "serde", "serde_json", "serde_yaml", "sha2", "tempfile", - "toml 0.9.5", + "toml 0.9.8", + "toml_edit 0.22.27", ] [[package]] name = "vector" -version = "0.51.0" +version = "0.54.0" dependencies = [ - "apache-avro", + "apache-avro 0.16.0", "approx", "arc-swap", "arr_macro", + "arrow", + "arrow-schema", "assert_cmd", "async-compression", "async-graphql", @@ -12083,11 +12516,8 @@ dependencies = [ "aws-smithy-types", "aws-types", "axum 0.6.20", - "azure_core 0.21.0", - "azure_core 0.25.0", - "azure_identity", - "azure_storage", - "azure_storage_blobs", + "azure_core", + "azure_storage_blob", "base64 0.22.1", "bloomy", "bollard", @@ -12098,6 +12528,7 @@ dependencies = [ "chrono", "chrono-tz", "clap", + "clap_complete", "colored", "console-subscriber", "criterion", @@ -12128,17 +12559,17 @@ dependencies = [ "headers", "heim", "hex", - "hickory-proto", + "hickory-proto 0.25.2", "hostname 0.4.0", "http 0.2.9", "http 1.3.1", - "http-body 0.4.5", + "http-body 0.4.6", "http-serde", "humantime", - "hyper 0.14.28", + "hyper 0.14.32", "hyper-openssl 0.9.2", "hyper-proxy", - "indexmap 2.11.0", + "indexmap 2.12.0", "indoc", "inventory", "ipnet", @@ -12149,7 +12580,7 @@ dependencies = [ "libc", "listenfd", "loki-logproto", - "lru 0.16.0", + "lru 0.16.3", "maxminddb", "md-5", "metrics", @@ -12173,7 +12604,6 @@ dependencies = [ "paste", "percent-encoding", "pin-project", - "portpicker", "postgres-openssl", "proptest", "proptest-derive", @@ -12190,18 +12620,20 @@ dependencies = [ "redis", "regex", "reqwest 0.11.26", + "reqwest 0.12.28", "rmp-serde", "rmpv", "roaring", "rstest", "rumqttc", + "rust_decimal", "seahash", - "semver 1.0.26", + "semver", "serde", "serde-toml-merge", "serde_bytes", "serde_json", - "serde_with 3.14.0", + "serde_with", "serde_yaml", "serial_test", "similar-asserts", @@ -12226,7 +12658,7 @@ dependencies = [ "tokio-test", "tokio-tungstenite 0.20.1", "tokio-util", - "toml 0.9.5", + "toml 0.9.8", "tonic 0.11.0", "tonic-build 0.11.0", "tower 0.5.2", @@ -12247,6 +12679,7 @@ dependencies = [ "vector-config-macros", "vector-lib", "vector-vrl-functions", + "vector-vrl-metrics", "vrl", "warp", "windows-service", @@ -12327,7 +12760,8 @@ dependencies = [ "crossbeam-utils", "derivative", "futures 0.3.31", - "indexmap 2.11.0", + "indexmap 2.12.0", + "itertools 0.14.0", "metrics", "paste", "pin-project", @@ -12337,10 +12771,20 @@ dependencies = [ "stream-cancel", "tokio", "tracing 0.1.41", + "vector-common-macros", "vector-config", "vrl", ] +[[package]] +name = "vector-common-macros" +version = "0.1.0" +dependencies = [ + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", +] + [[package]] name = "vector-config" version = "0.1.0" @@ -12350,15 +12794,15 @@ dependencies = [ "chrono-tz", "encoding_rs", "http 0.2.9", - "indexmap 2.11.0", + "indexmap 2.12.0", "inventory", "no-proxy", "num-traits", "serde", "serde_json", - "serde_with 3.14.0", + "serde_with", "snafu 0.8.9", - "toml 0.9.5", + "toml 0.9.8", "tracing 0.1.41", "url", "vector-config-common", @@ -12417,7 +12861,7 @@ dependencies = [ "headers", "http 0.2.9", "hyper-proxy", - "indexmap 2.11.0", + "indexmap 2.12.0", "inventory", "ipnet", "metrics", @@ -12447,7 +12891,7 @@ dependencies = [ "security-framework 3.5.1", "serde", "serde_json", - "serde_with 3.14.0", + "serde_with", "serde_yaml", "similar-asserts", "smallvec", @@ -12458,7 +12902,7 @@ dependencies = [ "tokio-stream", "tokio-test", "tokio-util", - "toml 0.9.5", + "toml 0.9.8", "tonic 0.11.0", "tracing 0.1.41", "tracing-subscriber", @@ -12554,7 +12998,7 @@ version = "0.1.0" dependencies = [ "chrono", "clap", - "crossterm 0.29.0", + "crossterm", "exitcode", "futures 0.3.31", "futures-util", @@ -12562,10 +13006,10 @@ dependencies = [ "humantime", "indoc", "num-format", - "number_prefix", "ratatui", "tokio", "tokio-stream", + "unit-prefix", "url", "vector-api-client", "vector-common", @@ -12584,6 +13028,23 @@ dependencies = [ name = "vector-vrl-functions" version = "0.1.0" dependencies = [ + "dnstap-parser", + "enrichment", + "indoc", + "vector-vrl-metrics", + "vrl", +] + +[[package]] +name = "vector-vrl-metrics" +version = "0.1.0" +dependencies = [ + "arc-swap", + "const-str", + "tokio", + "tokio-stream", + "vector-common", + "vector-core", "vrl", ] @@ -12593,14 +13054,15 @@ version = "0.1.0" dependencies = [ "chrono-tz", "clap", - "dnstap-parser", "enrichment", "glob", "serde", "serde_json", "tikv-jemallocator", "tracing-subscriber", + "vector-core", "vector-vrl-functions", + "vector-vrl-metrics", "vrl", ] @@ -12633,8 +13095,8 @@ checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" [[package]] name = "vrl" -version = "0.27.0" -source = "git+https://github.com/vectordotdev/vrl.git?branch=main#dbfeee575e0527f46264667339f37a795362099e" +version = "0.30.0" +source = "git+https://github.com/vectordotdev/vrl.git?branch=main#ee45ff6caf724ff91ff7fac825baa7267fd7aab7" dependencies = [ "aes", "aes-siv", @@ -12667,15 +13129,15 @@ dependencies = [ "dyn-clone", "encoding_rs", "exitcode", - "fancy-regex 0.15.0", + "fancy-regex", "flate2", "grok", "hex", "hmac", "hostname 0.4.0", "iana-time-zone", - "idna 1.0.3", - "indexmap 2.11.0", + "idna", + "indexmap 2.12.0", "indoc", "influxdb-line-protocol", "ipcrypt-rs", @@ -12709,7 +13171,7 @@ dependencies = [ "quoted_printable", "rand 0.8.5", "regex", - "reqwest 0.12.9", + "reqwest 0.12.28", "reqwest-middleware", "reqwest-retry", "roxmltree", @@ -12728,7 +13190,7 @@ dependencies = [ "strip-ansi-escapes", "syslog_loose 0.22.0", "termcolor", - "thiserror 2.0.3", + "thiserror 2.0.17", "tokio", "tracing 0.1.41", "ua-parser", @@ -12802,7 +13264,7 @@ dependencies = [ "futures-util", "headers", "http 0.2.9", - "hyper 0.14.28", + "hyper 0.14.32", "log", "mime", "mime_guess", @@ -12819,12 +13281,6 @@ dependencies = [ "tracing 0.1.41", ] -[[package]] -name = "wasi" -version = "0.9.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" - [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -12832,12 +13288,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] -name = "wasi" -version = "0.13.3+wasi-0.2.2" +name = "wasip2" +version = "1.0.1+wasi-0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26816d2e1a4a36a2940b96c5296ce403917633dff8f3440e9b236ed6f6bacad2" +checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" dependencies = [ - "wit-bindgen-rt", + "wit-bindgen", ] [[package]] @@ -12996,6 +13452,15 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "webpki-roots" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2878ef029c47c6e8cf779119f20fcf52bde7ad42a731b2a304bc221df17571e" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "which" version = "4.4.2" @@ -13094,7 +13559,7 @@ dependencies = [ "windows-collections", "windows-core 0.60.1", "windows-future", - "windows-link", + "windows-link 0.1.0", "windows-numerics", ] @@ -13124,9 +13589,9 @@ checksum = "ca21a92a9cae9bf4ccae5cf8368dce0837100ddf6e6d57936749e85f152f6247" dependencies = [ "windows-implement", "windows-interface", - "windows-link", - "windows-result 0.3.1", - "windows-strings 0.3.1", + "windows-link 0.1.0", + "windows-result", + "windows-strings", ] [[package]] @@ -13136,7 +13601,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a787db4595e7eb80239b74ce8babfb1363d8e343ab072f2ffe901400c03349f0" dependencies = [ "windows-core 0.60.1", - "windows-link", + "windows-link 0.1.0", ] [[package]] @@ -13167,6 +13632,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dccfd733ce2b1753b03b6d3c65edf020262ea35e20ccdf3e288043e6dd620e3" +[[package]] +name = "windows-link" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45e46c0661abb7180e7b9c281db115305d49ca1709ab8242adf09666d2173c65" + [[package]] name = "windows-numerics" version = "0.1.1" @@ -13174,27 +13645,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "005dea54e2f6499f2cee279b8f703b3cf3b5734a2d8d21867c8f44003182eeed" dependencies = [ "windows-core 0.60.1", - "windows-link", + "windows-link 0.1.0", ] [[package]] name = "windows-registry" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0" -dependencies = [ - "windows-result 0.2.0", - "windows-strings 0.1.0", - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-result" -version = "0.2.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" +checksum = "6c44a98275e31bfd112bb06ba96c8ab13c03383a3753fdddd715406a1824c7e0" dependencies = [ - "windows-targets 0.52.6", + "windows-link 0.1.0", + "windows-result", + "windows-strings", ] [[package]] @@ -13203,7 +13665,7 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06374efe858fab7e4f881500e6e86ec8bc28f9462c47e5a9941a0142ad86b189" dependencies = [ - "windows-link", + "windows-link 0.1.0", ] [[package]] @@ -13212,28 +13674,18 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "193cae8e647981c35bc947fdd57ba7928b1fa0d4a79305f6dd2dc55221ac35ac" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.10.0", "widestring 1.0.2", "windows-sys 0.59.0", ] -[[package]] -name = "windows-strings" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" -dependencies = [ - "windows-result 0.2.0", - "windows-targets 0.52.6", -] - [[package]] name = "windows-strings" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "87fa48cc5d406560701792be122a10132491cff9d0aeb23583cc2dcafc847319" dependencies = [ - "windows-link", + "windows-link 0.1.0", ] [[package]] @@ -13281,6 +13733,15 @@ dependencies = [ "windows-targets 0.53.2", ] +[[package]] +name = "windows-sys" +version = "0.61.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e201184e40b2ede64bc2ea34968b28e33622acdbbf37104f0e4a33f7abe657aa" +dependencies = [ + "windows-link 0.2.0", +] + [[package]] name = "windows-targets" version = "0.42.2" @@ -13534,9 +13995,9 @@ dependencies = [ [[package]] name = "winnow" -version = "0.7.10" +version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06928c8748d81b05c9be96aad92e1b6ff01833332f281e8cfca3be4b35fc9ec" +checksum = "21a0236b59786fed61e2a80582dd500fe61f18b5dca67a4a067d0bc9039339cf" dependencies = [ "memchr", ] @@ -13575,13 +14036,10 @@ dependencies = [ ] [[package]] -name = "wit-bindgen-rt" -version = "0.33.0" +name = "wit-bindgen" +version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c" -dependencies = [ - "bitflags 2.9.0", -] +checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" [[package]] name = "woothee" @@ -13784,10 +14242,11 @@ dependencies = [ [[package]] name = "zstd-sys" -version = "2.0.13+zstd.1.5.6" +version = "2.0.16+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" dependencies = [ + "bindgen", "cc", "pkg-config", ] diff --git a/Cargo.toml b/Cargo.toml index 14e1e219f1297..67c3b8961670a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "vector" -version = "0.51.0" +version = "0.54.0" authors = ["Vector Contributors "] edition = "2024" description = "A lightweight and ultra-fast tool for building observability pipelines" @@ -112,13 +112,13 @@ members = [ "lib/k8s-e2e-tests", "lib/k8s-test-framework", "lib/loki-logproto", - "lib/portpicker", "lib/prometheus-parser", "lib/opentelemetry-proto", "lib/tracing-limit", "lib/vector-api-client", "lib/vector-buffers", "lib/vector-common", + "lib/vector-common-macros", "lib/vector-config", "lib/vector-config-common", "lib/vector-config-macros", @@ -132,11 +132,13 @@ members = [ "lib/vector-vrl/functions", "lib/vector-vrl/tests", "lib/vector-vrl/web-playground", + "lib/vector-vrl-metrics", "vdev", ] [workspace.dependencies] anyhow = { version = "1.0.99", default-features = false, features = ["std"] } +arc-swap = { version = "1.7.1", default-features = false } async-stream = { version = "0.3.6", default-features = false } async-trait = { version = "0.1.89", default-features = false } base64 = { version = "0.22.1", default-features = false } @@ -144,8 +146,10 @@ bytes = { version = "1.10.1", default-features = false, features = ["serde"] } cfg-if = { version = "1.0.3", default-features = false } chrono = { version = "0.4.41", default-features = false, features = ["clock", "serde"] } chrono-tz = { version = "0.10.4", default-features = false, features = ["serde"] } -clap = { version = "4.5.48", default-features = false, features = ["derive", "error-context", "env", "help", "std", "string", "usage", "wrap_help"] } +clap = { version = "4.5.56", default-features = false, features = ["derive", "error-context", "env", "help", "std", "string", "usage", "wrap_help"] } +clap_complete = "4.5.65" colored = { version = "3.0.0", default-features = false } +const-str = { version = "1.0.0", default-features = false } crossbeam-utils = { version = "0.8.21", default-features = false } darling = { version = "0.20.11", default-features = false, features = ["suggestions"] } dashmap = { version = "6.1.0", default-features = false } @@ -178,7 +182,8 @@ prost-types = { version = "0.12", default-features = false } rand = { version = "0.9.2", default-features = false, features = ["small_rng", "thread_rng"] } rand_distr = { version = "0.5.1", default-features = false } regex = { version = "1.11.2", default-features = false, features = ["std", "perf"] } -reqwest = { version = "0.11.26", features = ["json"] } +reqwest = { version = "0.11", features = ["json"] } +rust_decimal = { version = "1.37.0", default-features = false, features = ["std"] } semver = { version = "1.0.26", default-features = false, features = ["serde", "std"] } serde = { version = "1.0.219", default-features = false, features = ["alloc", "derive", "rc"] } serde_json = { version = "1.0.143", default-features = false, features = ["raw_value", "std"] } @@ -186,25 +191,33 @@ serde_yaml = { version = "0.9.34", default-features = false } snafu = { version = "0.8.9", default-features = false, features = ["futures", "std"] } socket2 = { version = "0.5.10", default-features = false } tempfile = "3.23.0" -tokio = { version = "1.45.1", default-features = false } -toml = { version = "0.9.5", default-features = false, features = ["serde", "display", "parse"] } +tokio = { version = "1.49.0", default-features = false } +tokio-stream = { version = "0.1.18", default-features = false } +tokio-test = "0.4.5" +tokio-tungstenite = { version = "0.20.1", default-features = false } +toml = { version = "0.9.8", default-features = false, features = ["serde", "display", "parse"] } tonic = { version = "0.11", default-features = false, features = ["transport", "codegen", "prost", "tls", "tls-roots", "gzip"] } tonic-build = { version = "0.11", default-features = false, features = ["transport", "prost"] } tracing = { version = "0.1.34", default-features = false } tracing-subscriber = { version = "0.3.20", default-features = false, features = ["fmt"] } url = { version = "2.5.4", default-features = false, features = ["serde"] } -uuid = { version = "1.18.1", features = ["v4", "v7", "serde"] } +uuid = { version = "1.18.1", features = ["v4", "v7", "serde", "fast-rng"] } vector-config = { path = "lib/vector-config" } vector-config-common = { path = "lib/vector-config-common" } vector-config-macros = { path = "lib/vector-config-macros" } +vector-common-macros = { path = "lib/vector-common-macros" } vector-lib = { path = "lib/vector-lib", default-features = false, features = ["vrl"] } +vector-vrl-functions = { path = "lib/vector-vrl/functions" } vrl = { git = "https://github.com/vectordotdev/vrl.git", branch = "main", features = ["arbitrary", "cli", "test", "test_framework"] } mock_instant = { version = "0.6" } serial_test = { version = "3.2" } [dependencies] cfg-if.workspace = true +reqwest.workspace = true +reqwest_12 = { package = "reqwest", version = "0.12", features = ["json"] } clap.workspace = true +clap_complete.workspace = true indoc.workspace = true paste.workspace = true pin-project.workspace = true @@ -219,14 +232,14 @@ vrl.workspace = true dnsmsg-parser = { path = "lib/dnsmsg-parser", optional = true } dnstap-parser = { path = "lib/dnstap-parser", optional = true } fakedata = { path = "lib/fakedata", optional = true } -portpicker = { path = "lib/portpicker" } tracing-limit = { path = "lib/tracing-limit" } vector-common = { path = "lib/vector-common", default-features = false } vector-lib.workspace = true vector-config.workspace = true vector-config-common.workspace = true vector-config-macros.workspace = true -vector-vrl-functions = { path = "lib/vector-vrl/functions" } +vector-vrl-functions = { workspace = true, features = ["vrl-metrics"] } +vector-vrl-metrics = { path = "lib/vector-vrl-metrics" } loki-logproto = { path = "lib/loki-logproto", optional = true } # Tokio / Futures @@ -235,7 +248,7 @@ async-trait.workspace = true futures.workspace = true tokio = { workspace = true, features = ["full"] } tokio-openssl = { version = "0.6.5", default-features = false } -tokio-stream = { version = "0.1.17", default-features = false, features = ["net", "sync", "time"] } +tokio-stream = { workspace = true, features = ["net", "sync", "time"] } tokio-util = { version = "0.7", default-features = false, features = ["io", "time"] } console-subscriber = { version = "0.4.1", default-features = false, optional = true } @@ -282,16 +295,10 @@ aws-smithy-runtime-api = { version = "1.7.3", default-features = false, optional aws-smithy-types = { version = "1.2.11", default-features = false, features = ["rt-tokio"], optional = true } # Azure -azure_core = { version = "0.25", default-features = false, features = ["reqwest", "hmac_openssl"], optional = true } -azure_identity = { version = "0.25", default-features = false, features = ["reqwest"], optional = true } +azure_core = { version = "0.30", features = ["reqwest", "hmac_openssl"], optional = true } # Azure Storage -azure_storage = { version = "0.21", default-features = false, optional = true } -azure_storage_blobs = { version = "0.21", default-features = false, optional = true } - -# Needed to bridge with outdated version of azure_core used in azure_storage* -azure_core_for_storage = { package = "azure_core", version = "0.21.0", default-features = false, features = ["enable_reqwest", "hmac_openssl"] } - +azure_storage_blob = { version = "0.7", optional = true } # OpenDAL opendal = { version = "0.54", default-features = false, features = ["services-webhdfs"], optional = true } @@ -336,9 +343,11 @@ hex = { version = "0.4.3", default-features = false, optional = true } greptimedb-ingester = { git = "https://github.com/GreptimeTeam/greptimedb-ingester-rust", rev = "f7243393808640f5123b0d5b7b798da591a4df6e", optional = true } # External libs -arc-swap = { version = "1.7", default-features = false, optional = true } +arc-swap = { workspace = true, default-features = false, optional = true } async-compression = { version = "0.4.27", default-features = false, features = ["tokio", "gzip", "zstd"], optional = true } apache-avro = { version = "0.16.0", default-features = false, optional = true } +arrow = { version = "56.2.0", default-features = false, features = ["ipc"], optional = true } +arrow-schema = { version = "56.2.0", default-features = false, optional = true } axum = { version = "0.6.20", default-features = false } base64 = { workspace = true, optional = true } bloomy = { version = "1.2.0", default-features = false, optional = true } @@ -370,9 +379,9 @@ hostname = { version = "0.4.0", default-features = false } http = { version = "0.2.9", default-features = false } http-1 = { package = "http", version = "1.0", default-features = false, features = ["std"] } http-serde = "1.1.3" -http-body = { version = "0.4.5", default-features = false } +http-body = { version = "0.4.6", default-features = false } humantime.workspace = true -hyper = { version = "0.14.28", default-features = false, features = ["client", "runtime", "http1", "http2", "server", "stream"] } +hyper = { version = "0.14.32", default-features = false, features = ["client", "runtime", "http1", "http2", "server", "stream", "backports", "deprecated"] } hyper-openssl = { version = "0.9.2", default-features = false } hyper-proxy = { version = "0.9.1", default-features = false, features = ["openssl-tls"] } indexmap.workspace = true @@ -382,10 +391,10 @@ itertools.workspace = true k8s-openapi = { version = "0.22.0", default-features = false, features = ["v1_26"], optional = true } kube = { version = "0.93.0", default-features = false, features = ["client", "openssl-tls", "runtime"], optional = true } listenfd = { version = "1.0.2", default-features = false, optional = true } -lru = { version = "0.16.0", default-features = false } -maxminddb = { version = "0.26.0", default-features = false, optional = true, features = ["simdutf8"] } +lru = { version = "0.16.3", default-features = false } +maxminddb = { version = "0.27.0", default-features = false, optional = true, features = ["simdutf8"] } md-5 = { version = "0.10", default-features = false, optional = true } -mongodb = { version = "2.8.2", default-features = false, features = ["tokio-runtime"], optional = true } +mongodb = { version = "3.3.0", default-features = false, optional = true, features = ["compat-3-0-0", "dns-resolver", "rustls-tls"] } async-nats = { version = "0.42.0", default-features = false, optional = true, features = ["ring"] } nkeys = { version = "0.4.5", default-features = false, optional = true } nom = { workspace = true, optional = true } @@ -399,22 +408,23 @@ pulsar = { version = "6.3.1", default-features = false, features = ["tokio-runti quick-junit = { version = "0.5.1" } rand.workspace = true rand_distr.workspace = true -rdkafka = { version = "0.37.0", default-features = false, features = ["curl-static", "tokio", "libz", "ssl", "zstd"], optional = true } +rdkafka = { version = "0.38.0", default-features = false, features = ["curl-static", "tokio", "libz", "ssl", "zstd"], optional = true } redis = { version = "0.32.4", default-features = false, features = ["connection-manager", "sentinel", "tokio-comp", "tokio-native-tls-comp"], optional = true } regex.workspace = true roaring = { version = "0.11.2", default-features = false, features = ["std"], optional = true } rumqttc = { version = "0.24.0", default-features = false, features = ["use-rustls"], optional = true } +rust_decimal = { workspace = true, optional = true } seahash = { version = "4.1.0", default-features = false } smallvec = { version = "1", default-features = false, features = ["union", "serde"] } snap = { version = "1.1.1", default-features = false } socket2.workspace = true -sqlx = { version = "0.8.6", default-features = false, features = ["derive", "postgres", "chrono", "runtime-tokio"], optional = true } +sqlx = { version = "0.8.6", default-features = false, features = ["derive", "postgres", "chrono", "runtime-tokio", "tls-rustls-ring"], optional = true } stream-cancel = { version = "0.8.2", default-features = false } strip-ansi-escapes = { version = "0.2.1", default-features = false } syslog = { version = "6.1.1", default-features = false, optional = true } tikv-jemallocator = { version = "0.6.0", default-features = false, features = ["unprefixed_malloc_on_supported_platforms"], optional = true } tokio-postgres = { version = "0.7.13", default-features = false, features = ["runtime", "with-chrono-0_4"], optional = true } -tokio-tungstenite = { version = "0.20.1", default-features = false, features = ["connect"], optional = true } +tokio-tungstenite = { workspace = true, features = ["connect"], optional = true } toml.workspace = true hickory-proto = { workspace = true, optional = true } tonic = { workspace = true, optional = true } @@ -456,10 +466,6 @@ openssl-src = { version = "300", default-features = false, features = ["force-en approx = "0.5.1" assert_cmd = { version = "2.0.17", default-features = false } aws-smithy-runtime = { version = "1.8.3", default-features = false, features = ["tls-rustls"] } -azure_core = { version = "0.25", default-features = false, features = ["reqwest", "hmac_openssl", "azurite_workaround"] } -azure_identity = { version = "0.25", default-features = false, features = ["reqwest"] } -azure_storage = { version = "0.21", default-features = false, features = ["enable_reqwest", "hmac_openssl"] } -azure_storage_blobs = { version = "0.21", default-features = false, features = ["enable_reqwest", "hmac_openssl", "azurite_workaround"] } base64 = "0.22.1" criterion = { version = "0.7.0", features = ["html_reports", "async_tokio"] } itertools.workspace = true @@ -474,7 +480,7 @@ similar-asserts = "1.7.0" tempfile.workspace = true test-generator = "0.3.1" tokio = { workspace = true, features = ["test-util"] } -tokio-test = "0.4.4" +tokio-test.workspace = true tower-test = "0.4.0" vector-lib = { workspace = true, features = ["test"] } vrl.workspace = true @@ -580,8 +586,9 @@ enrichment-tables-mmdb = ["dep:maxminddb"] enrichment-tables-memory = ["dep:evmap", "dep:evmap-derive", "dep:thread_local"] # Codecs -codecs-syslog = ["vector-lib/syslog"] +codecs-arrow = ["dep:arrow", "dep:arrow-schema", "vector-lib/arrow"] codecs-opentelemetry = ["vector-lib/opentelemetry"] +codecs-syslog = ["vector-lib/syslog"] # Secrets secrets = ["secrets-aws-secrets-manager"] @@ -649,7 +656,7 @@ sources-aws_s3 = ["aws-core", "dep:aws-sdk-sqs", "dep:aws-sdk-s3", "dep:async-co sources-aws_sqs = ["aws-core", "dep:aws-sdk-sqs"] sources-datadog_agent = ["sources-utils-http-encoding", "protobuf-build", "dep:prost"] sources-demo_logs = ["dep:fakedata"] -sources-dnstap = ["sources-utils-net-tcp", "dep:base64", "dep:hickory-proto", "dep:dnsmsg-parser", "dep:dnstap-parser", "protobuf-build", "dep:prost"] +sources-dnstap = ["sources-utils-net-tcp", "dep:base64", "dep:hickory-proto", "dep:dnsmsg-parser", "dep:dnstap-parser", "protobuf-build", "dep:prost", "vector-vrl-functions/dnstap"] sources-docker_logs = ["docker"] sources-eventstoredb_metrics = [] sources-exec = [] @@ -728,6 +735,7 @@ transforms-logs = [ "transforms-exclusive-route", "transforms-sample", "transforms-throttle", + "transforms-trace_to_log" ] transforms-metrics = [ "transforms-aggregate", @@ -757,6 +765,7 @@ transforms-exclusive-route = [] transforms-sample = ["transforms-impl-sample"] transforms-tag_cardinality_limit = ["dep:bloomy", "dep:hashbrown"] transforms-throttle = ["dep:governor"] +transforms-trace_to_log = [] # Implementations of transforms transforms-impl-sample = [] @@ -785,6 +794,7 @@ sinks-logs = [ "sinks-datadog_events", "sinks-datadog_logs", "sinks-datadog_traces", + "sinks-doris", "sinks-elasticsearch", "sinks-file", "sinks-gcp", @@ -841,17 +851,18 @@ sinks-aws_s3 = ["dep:base64", "dep:md-5", "aws-core", "dep:aws-sdk-s3"] sinks-aws_sqs = ["aws-core", "dep:aws-sdk-sqs"] sinks-aws_sns = ["aws-core", "dep:aws-sdk-sns"] sinks-axiom = ["sinks-http"] -sinks-azure_blob = ["dep:azure_core", "dep:azure_identity", "dep:azure_storage", "dep:azure_storage_blobs"] +sinks-azure_blob = ["dep:azure_core", "dep:azure_storage_blob"] sinks-azure_monitor_logs = [] sinks-blackhole = [] sinks-chronicle = [] -sinks-clickhouse = [] +sinks-clickhouse = ["dep:nom", "dep:rust_decimal", "codecs-arrow"] sinks-console = [] sinks-databend = ["dep:databend-client"] sinks-datadog_events = [] sinks-datadog_logs = [] sinks-datadog_metrics = ["protobuf-build", "dep:prost", "dep:prost-reflect"] sinks-datadog_traces = ["protobuf-build", "dep:prost", "dep:rmpv", "dep:rmp-serde", "dep:serde_bytes"] +sinks-doris = ["sqlx/mysql"] sinks-elasticsearch = ["transforms-metric_to_log"] sinks-file = ["dep:async-compression"] sinks-gcp = ["sinks-gcp-chronicle", "dep:base64", "gcp"] @@ -905,6 +916,7 @@ all-integration-tests = [ "datadog-traces-integration-tests", "dnstap-integration-tests", "docker-logs-integration-tests", + "doris-integration-tests", "es-integration-tests", "eventstoredb_metrics-integration-tests", "fluent-integration-tests", @@ -970,6 +982,7 @@ datadog-logs-integration-tests = ["sinks-datadog_logs"] datadog-metrics-integration-tests = ["sinks-datadog_metrics", "dep:prost"] datadog-traces-integration-tests = ["sources-datadog_agent", "sinks-datadog_traces", "axum/tokio"] docker-logs-integration-tests = ["sources-docker_logs", "unix"] +doris-integration-tests = ["sinks-doris"] es-integration-tests = ["sinks-elasticsearch", "aws-core"] eventstoredb_metrics-integration-tests = ["sources-eventstoredb_metrics"] fluent-integration-tests = ["docker", "sources-fluent"] @@ -999,7 +1012,7 @@ webhdfs-integration-tests = ["sinks-webhdfs"] disable-resolv-conf = [] shutdown-tests = ["api", "sinks-blackhole", "sinks-console", "sinks-prometheus", "sources", "transforms-lua", "transforms-remap", "unix"] cli-tests = ["sinks-blackhole", "sinks-socket", "sources-demo_logs", "sources-file"] -test-utils = [] +test-utils = ["vector-lib/test"] # End-to-End testing-related features all-e2e-tests = [ diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index c6050f1474c11..81eb19036dfc5 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -1,7 +1,5 @@ Component,Origin,License,Copyright Inflector,https://github.com/whatisinternet/inflector,BSD-2-Clause,Josh Teeter -RustyXML,https://github.com/Florob/RustyXML,MIT OR Apache-2.0,Florian Zeitz -addr2line,https://github.com/gimli-rs/addr2line,Apache-2.0 OR MIT,The addr2line Authors adler2,https://github.com/oyvindln/adler2,0BSD OR MIT OR Apache-2.0,"Jonas Schievink , oyvindln " adler32,https://github.com/remram44/adler32-rs,Zlib,Remi Rampin aead,https://github.com/RustCrypto/traits,MIT OR Apache-2.0,RustCrypto Developers @@ -10,8 +8,12 @@ aes-siv,https://github.com/RustCrypto/AEADs,Apache-2.0 OR MIT,RustCrypto Develop ahash,https://github.com/tkaitchuck/ahash,MIT OR Apache-2.0,Tom Kaitchuck aho-corasick,https://github.com/BurntSushi/aho-corasick,Unlicense OR MIT,Andrew Gallant alloc-no-stdlib,https://github.com/dropbox/rust-alloc-no-stdlib,BSD-3-Clause,Daniel Reiter Horn +alloc-stdlib,https://github.com/dropbox/rust-alloc-no-stdlib,BSD-3-Clause,Daniel Reiter Horn allocator-api2,https://github.com/zakarumych/allocator-api2,MIT OR Apache-2.0,Zakarum amq-protocol,https://github.com/amqp-rs/amq-protocol,BSD-2-Clause,Marc-Antoine Perennou <%arc-Antoine@Perennou.com> +amq-protocol-tcp,https://github.com/amqp-rs/amq-protocol,BSD-2-Clause,Marc-Antoine Perennou <%arc-Antoine@Perennou.com> +amq-protocol-types,https://github.com/amqp-rs/amq-protocol,BSD-2-Clause,Marc-Antoine Perennou <%arc-Antoine@Perennou.com> +amq-protocol-uri,https://github.com/amqp-rs/amq-protocol,BSD-2-Clause,Marc-Antoine Perennou <%arc-Antoine@Perennou.com> android-tzdata,https://github.com/RumovZ/android-tzdata,MIT OR Apache-2.0,RumovZ android_system_properties,https://github.com/nical/android_system_properties,MIT OR Apache-2.0,Nicolas Silva ansi_term,https://github.com/ogham/rust-ansi-term,MIT,"ogham@bsago.me, Ryan Scheel (Havvy) , Josh Triplett " @@ -22,10 +24,24 @@ anstyle-query,https://github.com/rust-cli/anstyle,MIT OR Apache-2.0,The anstyle- anstyle-wincon,https://github.com/rust-cli/anstyle,MIT OR Apache-2.0,The anstyle-wincon Authors anyhow,https://github.com/dtolnay/anyhow,MIT OR Apache-2.0,David Tolnay apache-avro,https://github.com/apache/avro,Apache-2.0,Apache Avro team +apache-avro,https://github.com/apache/avro-rs,Apache-2.0,The apache-avro Authors arbitrary,https://github.com/rust-fuzz/arbitrary,MIT OR Apache-2.0,"The Rust-Fuzz Project Developers, Nick Fitzgerald , Manish Goregaokar , Simonas Kazlauskas , Brian L. Troutwine , Corey Farwell " arc-swap,https://github.com/vorner/arc-swap,MIT OR Apache-2.0,Michal 'vorner' Vaner arr_macro,https://github.com/JoshMcguigan/arr_macro,MIT OR Apache-2.0,Josh Mcguigan +arr_macro_impl,https://github.com/JoshMcguigan/arr_macro,MIT OR Apache-2.0,Josh Mcguigan arrayvec,https://github.com/bluss/arrayvec,MIT OR Apache-2.0,bluss +arrow,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow +arrow-arith,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow +arrow-array,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow +arrow-buffer,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow +arrow-cast,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow +arrow-data,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow +arrow-ipc,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow +arrow-ord,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow +arrow-row,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow +arrow-schema,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow +arrow-select,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow +arrow-string,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow ascii,https://github.com/tomprogrammer/rust-ascii,Apache-2.0 OR MIT,"Thomas Bahn , Torbjørn Birch Moltu , Simon Sapin " async-broadcast,https://github.com/smol-rs/async-broadcast,MIT OR Apache-2.0,"Stjepan Glavina , Yoshua Wuyts , Zeeshan Ali Khan " async-channel,https://github.com/smol-rs/async-channel,Apache-2.0 OR MIT,Stjepan Glavina @@ -33,7 +49,12 @@ async-compression,https://github.com/Nullus157/async-compression,MIT OR Apache-2 async-executor,https://github.com/smol-rs/async-executor,Apache-2.0 OR MIT,Stjepan Glavina async-fs,https://github.com/smol-rs/async-fs,Apache-2.0 OR MIT,Stjepan Glavina async-global-executor,https://github.com/Keruspe/async-global-executor,Apache-2.0 OR MIT,Marc-Antoine Perennou +async-global-executor-trait,https://github.com/amqp-rs/executor-trait,Apache-2.0 OR MIT,Marc-Antoine Perennou async-graphql,https://github.com/async-graphql/async-graphql,MIT OR Apache-2.0,"sunli , Koxiaet" +async-graphql-derive,https://github.com/async-graphql/async-graphql,MIT OR Apache-2.0,"sunli , Koxiaet" +async-graphql-parser,https://github.com/async-graphql/async-graphql,MIT OR Apache-2.0,"sunli , Koxiaet" +async-graphql-value,https://github.com/async-graphql/async-graphql,MIT OR Apache-2.0,"sunli , Koxiaet" +async-graphql-warp,https://github.com/async-graphql/async-graphql,MIT OR Apache-2.0,"sunli , Koxiaet" async-io,https://github.com/smol-rs/async-io,Apache-2.0 OR MIT,Stjepan Glavina async-lock,https://github.com/smol-rs/async-lock,Apache-2.0 OR MIT,Stjepan Glavina async-nats,https://github.com/nats-io/nats.rs,Apache-2.0,"Tomasz Pietrek , Casper Beyer " @@ -43,6 +64,7 @@ async-reactor-trait,https://github.com/amqp-rs/reactor-trait,Apache-2.0 OR MIT,M async-recursion,https://github.com/dcchut/async-recursion,MIT OR Apache-2.0,Robert Usher <266585+dcchut@users.noreply.github.com> async-signal,https://github.com/smol-rs/async-signal,Apache-2.0 OR MIT,John Nunley async-stream,https://github.com/tokio-rs/async-stream,MIT,Carl Lerche +async-stream-impl,https://github.com/tokio-rs/async-stream,MIT,Carl Lerche async-task,https://github.com/smol-rs/async-task,Apache-2.0 OR MIT,Stjepan Glavina async-trait,https://github.com/dtolnay/async-trait,MIT OR Apache-2.0,David Tolnay atoi,https://github.com/pacman82/atoi-rs,MIT,Markus Klein @@ -80,14 +102,10 @@ aws-types,https://github.com/smithy-lang/smithy-rs,Apache-2.0,"AWS Rust SDK Team axum,https://github.com/tokio-rs/axum,MIT,The axum Authors axum-core,https://github.com/tokio-rs/axum,MIT,The axum-core Authors azure_core,https://github.com/azure/azure-sdk-for-rust,MIT,Microsoft -azure_core,https://github.com/azure/azure-sdk-for-rust,MIT,Microsoft Corp. -azure_identity,https://github.com/azure/azure-sdk-for-rust,MIT,Microsoft -azure_storage,https://github.com/azure/azure-sdk-for-rust,MIT,Microsoft Corp. -azure_storage_blobs,https://github.com/azure/azure-sdk-for-rust,MIT,Microsoft Corp. -azure_svc_blobstorage,https://github.com/azure/azure-sdk-for-rust,MIT,The azure_svc_blobstorage Authors +azure_core_macros,https://github.com/azure/azure-sdk-for-rust,MIT,Microsoft +azure_storage_blob,https://github.com/azure/azure-sdk-for-rust,MIT,Microsoft backoff,https://github.com/ihrwein/backoff,MIT OR Apache-2.0,Tibor Benke backon,https://github.com/Xuanwo/backon,Apache-2.0,The backon Authors -backtrace,https://github.com/rust-lang/backtrace-rs,MIT OR Apache-2.0,The Rust Project Developers base16,https://github.com/thomcc/rust-base16,CC0-1.0,Thom Chiovoloni base16ct,https://github.com/RustCrypto/formats/tree/master/base16ct,Apache-2.0 OR MIT,RustCrypto Developers base62,https://github.com/fbernier/base62,MIT,"François Bernier , Chai T. Rex " @@ -95,6 +113,7 @@ base64,https://github.com/marshallpierce/rust-base64,MIT OR Apache-2.0,"Alice Ma base64,https://github.com/marshallpierce/rust-base64,MIT OR Apache-2.0,Marshall Pierce base64-simd,https://github.com/Nugine/simd,MIT,The base64-simd Authors base64ct,https://github.com/RustCrypto/formats/tree/master/base64ct,Apache-2.0 OR MIT,RustCrypto Developers +bigdecimal,https://github.com/akubera/bigdecimal-rs,MIT OR Apache-2.0,Andrew Kubera bit-set,https://github.com/contain-rs/bit-set,Apache-2.0 OR MIT,Alexis Beingessner bit-vec,https://github.com/contain-rs/bit-vec,Apache-2.0 OR MIT,Alexis Beingessner bitflags,https://github.com/bitflags/bitflags,MIT OR Apache-2.0,The Rust Project Developers @@ -105,21 +124,27 @@ block-padding,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto D blocking,https://github.com/smol-rs/blocking,Apache-2.0 OR MIT,Stjepan Glavina bloomy,https://docs.rs/bloomy/,MIT,"Aleksandr Bezobchuk , Alexis Sellier " bollard,https://github.com/fussybeaver/bollard,Apache-2.0,Bollard contributors +bollard-stubs,https://github.com/fussybeaver/bollard,Apache-2.0,Bollard contributors +bon,https://github.com/elastio/bon,MIT OR Apache-2.0,The bon Authors +bon-macros,https://github.com/elastio/bon,MIT OR Apache-2.0,The bon-macros Authors borrow-or-share,https://github.com/yescallop/borrow-or-share,MIT-0,Scallop Ye +borsh,https://github.com/near/borsh-rs,MIT OR Apache-2.0,Near Inc +borsh-derive,https://github.com/near/borsh-rs,Apache-2.0,Near Inc brotli,https://github.com/dropbox/rust-brotli,BSD-3-Clause AND MIT,"Daniel Reiter Horn , The Brotli Authors" brotli-decompressor,https://github.com/dropbox/rust-brotli-decompressor,BSD-3-Clause OR MIT,"Daniel Reiter Horn , The Brotli Authors" bson,https://github.com/mongodb/bson-rust,MIT,"Y. T. Chung , Kevin Yeh , Saghm Rossi , Patrick Freed , Isabel Atkinson , Abraham Egnor " bstr,https://github.com/BurntSushi/bstr,MIT OR Apache-2.0,Andrew Gallant bumpalo,https://github.com/fitzgen/bumpalo,MIT OR Apache-2.0,Nick Fitzgerald bytecheck,https://github.com/djkoloski/bytecheck,MIT,David Koloski +bytecheck_derive,https://github.com/djkoloski/bytecheck,MIT,David Koloski bytecount,https://github.com/llogiq/bytecount,Apache-2.0 OR MIT,"Andre Bogus , Joshua Landau " bytemuck,https://github.com/Lokathor/bytemuck,Zlib OR Apache-2.0 OR MIT,Lokathor +bytemuck_derive,https://github.com/Lokathor/bytemuck,Zlib OR Apache-2.0 OR MIT,Lokathor byteorder,https://github.com/BurntSushi/byteorder,Unlicense OR MIT,Andrew Gallant bytes,https://github.com/carllerche/bytes,MIT,Carl Lerche bytes,https://github.com/tokio-rs/bytes,MIT,"Carl Lerche , Sean McArthur " bytes-utils,https://github.com/vorner/bytes-utils,Apache-2.0 OR MIT,Michal 'vorner' Vaner bytesize,https://github.com/bytesize-rs/bytesize,Apache-2.0,"Hyunsik Choi , MrCroxx , Rob Ede " -cassowary,https://github.com/dylanede/cassowary-rs,MIT OR Apache-2.0,Dylan Ede castaway,https://github.com/sagebind/castaway,MIT,Stephen M. Coakley cbc,https://github.com/RustCrypto/block-modes,MIT OR Apache-2.0,RustCrypto Developers cesu8,https://github.com/emk/cesu8-rs,Apache-2.0 OR MIT,Eric Kidd @@ -131,10 +156,13 @@ charset,https://github.com/hsivonen/charset,MIT OR Apache-2.0,Henri Sivonen +ciborium-io,https://github.com/enarx/ciborium,Apache-2.0,Nathaniel McCallum +ciborium-ll,https://github.com/enarx/ciborium,Apache-2.0,Nathaniel McCallum cidr,https://github.com/stbuehler/rust-cidr,MIT,Stefan Bühler cipher,https://github.com/RustCrypto/traits,MIT OR Apache-2.0,RustCrypto Developers clap,https://github.com/clap-rs/clap,MIT OR Apache-2.0,The clap Authors clap_builder,https://github.com/clap-rs/clap,MIT OR Apache-2.0,The clap_builder Authors +clap_complete,https://github.com/clap-rs/clap,MIT OR Apache-2.0,The clap_complete Authors clap_derive,https://github.com/clap-rs/clap,MIT OR Apache-2.0,The clap_derive Authors clap_lex,https://github.com/clap-rs/clap,MIT OR Apache-2.0,The clap_lex Authors clipboard-win,https://github.com/DoumanAsh/clipboard-win,BSL-1.0,Douman @@ -145,8 +173,13 @@ colored,https://github.com/mackwic/colored,MPL-2.0,Thomas Wickham community-id,https://github.com/traceflight/rs-community-id,MIT OR Apache-2.0,Julian Wang compact_str,https://github.com/ParkMyCar/compact_str,MIT,Parker Timmerman +compression-codecs,https://github.com/Nullus157/async-compression,MIT OR Apache-2.0,"Wim Looman , Allen Bui " +compression-core,https://github.com/Nullus157/async-compression,MIT OR Apache-2.0,"Wim Looman , Allen Bui " concurrent-queue,https://github.com/smol-rs/concurrent-queue,Apache-2.0 OR MIT,"Stjepan Glavina , Taiki Endo , John Nunley " const-oid,https://github.com/RustCrypto/formats/tree/master/const-oid,Apache-2.0 OR MIT,RustCrypto Developers +const-random,https://github.com/tkaitchuck/constrandom,MIT OR Apache-2.0,Tom Kaitchuck +const-random-macro,https://github.com/tkaitchuck/constrandom,MIT OR Apache-2.0,Tom Kaitchuck +const-str,https://github.com/Nugine/const-str,MIT,Nugine convert_case,https://github.com/rutrum/convert-case,MIT,David Purdum convert_case,https://github.com/rutrum/convert-case,MIT,rutrum cookie,https://github.com/SergioBenitez/cookie-rs,MIT OR Apache-2.0,"Sergio Benitez , Alex Crichton " @@ -154,6 +187,7 @@ cookie-factory,https://github.com/rust-bakery/cookie-factory,MIT,"Geoffroy Coupr cookie_store,https://github.com/pfernie/cookie_store,MIT OR Apache-2.0,Patrick Fernie core-foundation,https://github.com/servo/core-foundation-rs,MIT OR Apache-2.0,The Servo Project Developers core-foundation,https://github.com/servo/core-foundation-rs,MIT OR Apache-2.0,The Servo Project Developers +core-foundation-sys,https://github.com/servo/core-foundation-rs,MIT OR Apache-2.0,The Servo Project Developers core2,https://github.com/bbqsrc/core2,Apache-2.0 OR MIT,Brendan Molloy cpufeatures,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers crc,https://github.com/mrhooray/crc-rs,MIT OR Apache-2.0,"Rui Hu , Akhil Velagapudi <4@4khil.com>" @@ -173,11 +207,14 @@ crypto-bigint,https://github.com/RustCrypto/crypto-bigint,Apache-2.0 OR MIT,Rust crypto-common,https://github.com/RustCrypto/traits,MIT OR Apache-2.0,RustCrypto Developers crypto_secretbox,https://github.com/RustCrypto/nacl-compat/tree/master/crypto_secretbox,Apache-2.0 OR MIT,RustCrypto Developers csv,https://github.com/BurntSushi/rust-csv,Unlicense OR MIT,Andrew Gallant +csv-core,https://github.com/BurntSushi/rust-csv,Unlicense OR MIT,Andrew Gallant ctr,https://github.com/RustCrypto/block-modes,MIT OR Apache-2.0,RustCrypto Developers curl-sys,https://github.com/alexcrichton/curl-rust,MIT,Alex Crichton curve25519-dalek,https://github.com/dalek-cryptography/curve25519-dalek/tree/main/curve25519-dalek,BSD-3-Clause,"Isis Lovecruft , Henry de Valence " curve25519-dalek-derive,https://github.com/dalek-cryptography/curve25519-dalek,MIT OR Apache-2.0,The curve25519-dalek-derive Authors darling,https://github.com/TedDriggs/darling,MIT,Ted Driggs +darling_core,https://github.com/TedDriggs/darling,MIT,Ted Driggs +darling_macro,https://github.com/TedDriggs/darling,MIT,Ted Driggs dary_heap,https://github.com/hanmertens/dary_heap,MIT OR Apache-2.0,Han Mertens dashmap,https://github.com/xacrimon/dashmap,MIT,Acrimon data-encoding,https://github.com/ia0/data-encoding,MIT,Julien Cretin @@ -185,14 +222,18 @@ data-url,https://github.com/servo/rust-url,MIT OR Apache-2.0,Simon Sapin dbl,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers deadpool,https://github.com/bikeshedder/deadpool,MIT OR Apache-2.0,Michael P. Jung +deadpool-runtime,https://github.com/bikeshedder/deadpool,MIT OR Apache-2.0,Michael P. Jung der,https://github.com/RustCrypto/formats/tree/master/der,Apache-2.0 OR MIT,RustCrypto Developers deranged,https://github.com/jhpratt/deranged,MIT OR Apache-2.0,Jacob Pratt derivative,https://github.com/mcarton/rust-derivative,MIT OR Apache-2.0,mcarton +derive-syn-parse,https://github.com/sharnoff/derive-syn-parse,MIT OR Apache-2.0,sharnoff +derive-where,https://github.com/ModProg/derive-where,MIT OR Apache-2.0,The derive-where Authors derive_arbitrary,https://github.com/rust-fuzz/arbitrary,MIT OR Apache-2.0,"The Rust-Fuzz Project Developers, Nick Fitzgerald , Manish Goregaokar , Andre Bogus , Corey Farwell " derive_builder,https://github.com/colin-kiegel/rust-derive-builder,MIT OR Apache-2.0,"Colin Kiegel , Pascal Hertleif , Jan-Erik Rediger , Ted Driggs " derive_builder_core,https://github.com/colin-kiegel/rust-derive-builder,MIT OR Apache-2.0,"Colin Kiegel , Pascal Hertleif , Jan-Erik Rediger , Ted Driggs " derive_builder_macro,https://github.com/colin-kiegel/rust-derive-builder,MIT OR Apache-2.0,"Colin Kiegel , Pascal Hertleif , Jan-Erik Rediger , Ted Driggs " derive_more,https://github.com/JelteF/derive_more,MIT,Jelte Fennema +derive_more-impl,https://github.com/JelteF/derive_more,MIT,Jelte Fennema digest,https://github.com/RustCrypto/traits,MIT OR Apache-2.0,RustCrypto Developers dirs-next,https://github.com/xdg-rs/dirs,MIT OR Apache-2.0,The @xdg-rs members dirs-sys-next,https://github.com/xdg-rs/dirs/tree/master/dirs-sys,MIT OR Apache-2.0,The @xdg-rs members @@ -201,6 +242,7 @@ dns-lookup,https://github.com/keeperofdakeys/dns-lookup,MIT OR Apache-2.0,Josh D doc-comment,https://github.com/GuillaumeGomez/doc-comment,MIT,Guillaume Gomez document-features,https://github.com/slint-ui/document-features,MIT OR Apache-2.0,Slint Developers domain,https://github.com/nlnetlabs/domain,BSD-3-Clause,NLnet Labs +domain-macros,https://github.com/nlnetlabs/domain,BSD-3-Clause,NLnet Labs dotenvy,https://github.com/allan2/dotenvy,MIT,"Noemi Lapresta , Craig Hills , Mike Piccolo , Alice Maz , Sean Griffin , Adam Sharp , Arpad Borsos , Allan Zhang " dyn-clone,https://github.com/dtolnay/dyn-clone,MIT OR Apache-2.0,David Tolnay ecdsa,https://github.com/RustCrypto/signatures/tree/master/ecdsa,Apache-2.0 OR MIT,RustCrypto Developers @@ -215,6 +257,7 @@ endian-type,https://github.com/Lolirofle/endian-type,MIT,Lolirofle enum_dispatch,https://gitlab.com/antonok/enum_dispatch,MIT OR Apache-2.0,Anton Lazarev enumflags2,https://github.com/meithecatte/enumflags2,MIT OR Apache-2.0,"maik klein , Maja Kądziołka " +enumflags2_derive,https://github.com/meithecatte/enumflags2,MIT OR Apache-2.0,"maik klein , Maja Kądziołka " env_logger,https://github.com/env-logger-rs/env_logger,MIT OR Apache-2.0,The Rust Project Developers equivalent,https://github.com/cuviper/equivalent,Apache-2.0 OR MIT,The equivalent Authors erased-serde,https://github.com/dtolnay/erased-serde,MIT OR Apache-2.0,David Tolnay @@ -226,16 +269,17 @@ event-listener,https://github.com/smol-rs/event-listener,Apache-2.0 OR MIT,Stjep event-listener,https://github.com/smol-rs/event-listener,Apache-2.0 OR MIT,"Stjepan Glavina , John Nunley " event-listener-strategy,https://github.com/smol-rs/event-listener-strategy,Apache-2.0 OR MIT,John Nunley evmap,https://github.com/jonhoo/rust-evmap,MIT OR Apache-2.0,Jon Gjengset +evmap-derive,https://github.com/jonhoo/rust-evmap,MIT OR Apache-2.0,Jon Gjengset executor-trait,https://github.com/amqp-rs/executor-trait,Apache-2.0 OR MIT,Marc-Antoine Perennou exitcode,https://github.com/benwilber/exitcode,Apache-2.0,Ben Wilber fakedata_generator,https://github.com/kevingimbel/fakedata_generator,MIT,Kevin Gimbel fallible-iterator,https://github.com/sfackler/rust-fallible-iterator,MIT OR Apache-2.0,Steven Fackler -fancy-regex,https://github.com/fancy-regex/fancy-regex,MIT,"Raph Levien , Robin Stocker " fancy-regex,https://github.com/fancy-regex/fancy-regex,MIT,"Raph Levien , Robin Stocker , Keith Hall " fastrand,https://github.com/smol-rs/fastrand,Apache-2.0 OR MIT,Stjepan Glavina ff,https://github.com/zkcrypto/ff,MIT OR Apache-2.0,"Sean Bowe , Jack Grigg " fiat-crypto,https://github.com/mit-plv/fiat-crypto,MIT OR Apache-2.0 OR BSD-1-Clause,Fiat Crypto library authors finl_unicode,https://github.com/dahosek/finl_unicode,MIT OR Apache-2.0,The finl_unicode Authors +flatbuffers,https://github.com/google/flatbuffers,Apache-2.0,"Robert Winslow , FlatBuffers Maintainers" flate2,https://github.com/rust-lang/flate2-rs,MIT OR Apache-2.0,"Alex Crichton , Josh Triplett " float_eq,https://github.com/jtempest/float_eq-rs,MIT OR Apache-2.0,jtempest fluent-uri,https://github.com/yescallop/fluent-uri-rs,MIT,Scallop Ye @@ -243,6 +287,8 @@ flume,https://github.com/zesterer/flume,Apache-2.0 OR MIT,Joshua Barretto foldhash,https://github.com/orlp/foldhash,Zlib,Orson Peters foreign-types,https://github.com/sfackler/foreign-types,MIT OR Apache-2.0,Steven Fackler +foreign-types-shared,https://github.com/sfackler/foreign-types,MIT OR Apache-2.0,Steven Fackler +form_urlencoded,https://github.com/servo/rust-url,MIT OR Apache-2.0,The rust-url developers fraction,https://github.com/dnsl48/fraction,MIT OR Apache-2.0,dnsl48 fsevent-sys,https://github.com/octplane/fsevent-rust/tree/master/fsevent-sys,MIT,Pierre Baillet fslock,https://github.com/brunoczim/fslock,MIT,The fslock Authors @@ -262,7 +308,6 @@ futures-timer,https://github.com/async-rs/futures-timer,MIT OR Apache-2.0,Alex C futures-util,https://github.com/rust-lang/futures-rs,MIT OR Apache-2.0,The futures-util Authors generic-array,https://github.com/fizyk20/generic-array,MIT,"Bartłomiej Kamiński , Aaron Trent " getrandom,https://github.com/rust-random/getrandom,MIT OR Apache-2.0,The Rand Project Developers -gimli,https://github.com/gimli-rs/gimli,MIT OR Apache-2.0,The gimli Authors glob,https://github.com/rust-lang/glob,MIT OR Apache-2.0,The Rust Project Developers gloo-timers,https://github.com/rustwasm/gloo/tree/master/crates/timers,MIT OR Apache-2.0,Rust and WebAssembly Working Group goauth,https://github.com/durch/rust-goauth,MIT,Drazen Urch @@ -283,21 +328,30 @@ hashbag,https://github.com/jonhoo/hashbag,MIT OR Apache-2.0,Jon Gjengset hashlink,https://github.com/kyren/hashlink,MIT OR Apache-2.0,kyren headers,https://github.com/hyperium/headers,MIT,Sean McArthur +headers-core,https://github.com/hyperium/headers,MIT,Sean McArthur heck,https://github.com/withoutboats/heck,MIT OR Apache-2.0,The heck Authors heck,https://github.com/withoutboats/heck,MIT OR Apache-2.0,Without Boats heim,https://github.com/heim-rs/heim,Apache-2.0 OR MIT,svartalf +heim-common,https://github.com/heim-rs/heim,Apache-2.0 OR MIT,svartalf +heim-cpu,https://github.com/heim-rs/heim,Apache-2.0 OR MIT,svartalf +heim-disk,https://github.com/heim-rs/heim,Apache-2.0 OR MIT,svartalf +heim-host,https://github.com/heim-rs/heim,Apache-2.0 OR MIT,svartalf +heim-memory,https://github.com/heim-rs/heim,Apache-2.0 OR MIT,svartalf +heim-net,https://github.com/heim-rs/heim,Apache-2.0 OR MIT,svartalf +heim-runtime,https://github.com/heim-rs/heim,Apache-2.0 OR MIT,svartalf hermit-abi,https://github.com/hermit-os/hermit-rs,MIT OR Apache-2.0,Stefan Lankes hex,https://github.com/KokaKiwi/rust-hex,MIT OR Apache-2.0,KokaKiwi hickory-proto,https://github.com/hickory-dns/hickory-dns,MIT OR Apache-2.0,The contributors to Hickory DNS +hickory-resolver,https://github.com/hickory-dns/hickory-dns,MIT OR Apache-2.0,The contributors to Hickory DNS hkdf,https://github.com/RustCrypto/KDFs,MIT OR Apache-2.0,RustCrypto Developers hmac,https://github.com/RustCrypto/MACs,MIT OR Apache-2.0,RustCrypto Developers home,https://github.com/rust-lang/cargo,MIT OR Apache-2.0,Brian Anderson hostname,https://github.com/svartalf/hostname,MIT,"fengcen , svartalf " http,https://github.com/hyperium/http,MIT OR Apache-2.0,"Alex Crichton , Carl Lerche , Sean McArthur " http-body,https://github.com/hyperium/http-body,MIT,"Carl Lerche , Lucio Franco , Sean McArthur " +http-body-util,https://github.com/hyperium/http-body,MIT,"Carl Lerche , Lucio Franco , Sean McArthur " http-range-header,https://github.com/MarcusGrass/parse-range-headers,MIT,The http-range-header Authors http-serde,https://gitlab.com/kornelski/http-serde,Apache-2.0 OR MIT,Kornel -http-types,https://github.com/http-rs/http-types,MIT OR Apache-2.0,Yoshua Wuyts httparse,https://github.com/seanmonstar/httparse,MIT OR Apache-2.0,Sean McArthur httpdate,https://github.com/pyfisch/httpdate,MIT OR Apache-2.0,Pyfisch humantime,https://github.com/chronotope/humantime,MIT OR Apache-2.0,The humantime Authors @@ -323,11 +377,11 @@ icu_properties_data,https://github.com/unicode-org/icu4x,Unicode-3.0,The ICU4X P icu_provider,https://github.com/unicode-org/icu4x,Unicode-3.0,The ICU4X Project Developers icu_provider_macros,https://github.com/unicode-org/icu4x,Unicode-3.0,The ICU4X Project Developers ident_case,https://github.com/TedDriggs/ident_case,MIT OR Apache-2.0,Ted Driggs +idna,https://github.com/servo/rust-url,MIT OR Apache-2.0,The rust-url developers idna_adapter,https://github.com/hsivonen/idna_adapter,Apache-2.0 OR MIT,The rust-url developers indexmap,https://github.com/bluss/indexmap,Apache-2.0 OR MIT,The indexmap Authors indexmap,https://github.com/indexmap-rs/indexmap,Apache-2.0 OR MIT,The indexmap Authors indoc,https://github.com/dtolnay/indoc,MIT OR Apache-2.0,David Tolnay -infer,https://github.com/bojand/infer,MIT,Bojan influxdb-line-protocol,https://github.com/influxdata/influxdb_iox/tree/main/influxdb_line_protocol,MIT OR Apache-2.0,InfluxDB IOx Project Developers inotify,https://github.com/hannobraun/inotify,ISC,"Hanno Braun , Félix Saparelli , Cristian Kubis , Frank Denis " inotify-sys,https://github.com/hannobraun/inotify-sys,ISC,Hanno Braun @@ -336,12 +390,12 @@ instability,https://github.com/ratatui-org/instability,MIT,"Stephen M. Coakley < instant,https://github.com/sebcrozet/instant,BSD-3-Clause,sebcrozet inventory,https://github.com/dtolnay/inventory,MIT OR Apache-2.0,David Tolnay io-lifetimes,https://github.com/sunfishcode/io-lifetimes,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,Dan Gohman -io-uring,https://github.com/tokio-rs/io-uring,MIT OR Apache-2.0,quininer iovec,https://github.com/carllerche/iovec,MIT OR Apache-2.0,Carl Lerche ipconfig,https://github.com/liranringel/ipconfig,MIT OR Apache-2.0,Liran Ringel ipcrypt-rs,https://github.com/jedisct1/rust-ipcrypt2,ISC,Frank Denis ipnet,https://github.com/krisprice/ipnet,MIT OR Apache-2.0,Kris Price ipnetwork,https://github.com/achanda/ipnetwork,MIT OR Apache-2.0,"Abhishek Chanda , Linus Färnstrand " +iri-string,https://github.com/lo48576/iri-string,MIT OR Apache-2.0,YOSHIOKA Takuma is-terminal,https://github.com/sunfishcode/is-terminal,MIT,"softprops , Dan Gohman " is_ci,https://github.com/zkat/is_ci,ISC,Kat Marchán itertools,https://github.com/rust-itertools/itertools,MIT OR Apache-2.0,bluss @@ -354,38 +408,56 @@ jsonpath-rust,https://github.com/besok/jsonpath-rust,MIT,BorisZhguchev k8s-openapi,https://github.com/Arnavion/k8s-openapi,Apache-2.0,Arnav Singh +kasuari,https://github.com/ratatui/kasuari,MIT OR Apache-2.0,"Dylan Ede , The Ratatui Developers" keccak,https://github.com/RustCrypto/sponges/tree/master/keccak,Apache-2.0 OR MIT,RustCrypto Developers kqueue,https://gitlab.com/rust-kqueue/rust-kqueue,MIT,William Orr kqueue-sys,https://gitlab.com/rust-kqueue/rust-kqueue-sys,MIT,"William Orr , Daniel (dmilith) Dettlaff " krb5-src,https://github.com/MaterializeInc/rust-krb5-src,Apache-2.0,"Materialize, Inc." kube,https://github.com/kube-rs/kube,Apache-2.0,"clux , Natalie Klestrup Röijezon , kazk " +kube-client,https://github.com/kube-rs/kube,Apache-2.0,"clux , Natalie Klestrup Röijezon , kazk " +kube-core,https://github.com/kube-rs/kube,Apache-2.0,"clux , Natalie Klestrup Röijezon , kazk " +kube-runtime,https://github.com/kube-rs/kube,Apache-2.0,"clux , Natalie Klestrup Röijezon , kazk " lalrpop-util,https://github.com/lalrpop/lalrpop,Apache-2.0 OR MIT,Niko Matsakis lapin,https://github.com/amqp-rs/lapin,MIT,"Geoffroy Couprie , Marc-Antoine Perennou " lazy_static,https://github.com/rust-lang-nursery/lazy-static.rs,MIT OR Apache-2.0,Marvin Löbel +lexical-core,https://github.com/Alexhuszagh/rust-lexical,MIT OR Apache-2.0,Alex Huszagh +lexical-parse-float,https://github.com/Alexhuszagh/rust-lexical,MIT OR Apache-2.0,Alex Huszagh +lexical-parse-integer,https://github.com/Alexhuszagh/rust-lexical,MIT OR Apache-2.0,Alex Huszagh +lexical-util,https://github.com/Alexhuszagh/rust-lexical,MIT OR Apache-2.0,Alex Huszagh +lexical-write-float,https://github.com/Alexhuszagh/rust-lexical,MIT OR Apache-2.0,Alex Huszagh +lexical-write-integer,https://github.com/Alexhuszagh/rust-lexical,MIT OR Apache-2.0,Alex Huszagh libc,https://github.com/rust-lang/libc,MIT OR Apache-2.0,The Rust Project Developers libflate,https://github.com/sile/libflate,MIT,Takeru Ohta +libflate_lz77,https://github.com/sile/libflate,MIT,Takeru Ohta libm,https://github.com/rust-lang/libm,MIT OR Apache-2.0,Jorge Aparicio libsqlite3-sys,https://github.com/rusqlite/rusqlite,MIT,The rusqlite developers libz-rs-sys,https://github.com/trifectatechfoundation/zlib-rs,Zlib,The libz-rs-sys Authors libz-sys,https://github.com/rust-lang/libz-sys,MIT OR Apache-2.0,"Alex Crichton , Josh Triplett , Sebastian Thiel " +line-clipping,https://github.com/joshka/line-clipping,MIT OR Apache-2.0,Josh McKinney linked-hash-map,https://github.com/contain-rs/linked-hash-map,MIT OR Apache-2.0,"Stepan Koltsov , Andrew Paseltiner " linked_hash_set,https://github.com/alexheretic/linked-hash-set,Apache-2.0,Alex Butler linux-raw-sys,https://github.com/sunfishcode/linux-raw-sys,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,Dan Gohman listenfd,https://github.com/mitsuhiko/listenfd,Apache-2.0,Armin Ronacher litemap,https://github.com/unicode-org/icu4x,Unicode-3.0,The ICU4X Project Developers litrs,https://github.com/LukasKalbertodt/litrs,MIT OR Apache-2.0,Lukas Kalbertodt +lock_api,https://github.com/Amanieu/parking_lot,MIT OR Apache-2.0,Amanieu d'Antras lockfree-object-pool,https://github.com/EVaillant/lockfree-object-pool,BSL-1.0,Etienne Vaillant log,https://github.com/rust-lang/log,MIT OR Apache-2.0,The Rust Project Developers lru,https://github.com/jeromefroe/lru-rs,MIT,Jerome Froelich lru-cache,https://github.com/contain-rs/lru-cache,MIT OR Apache-2.0,Stepan Koltsov lz4,https://github.com/10xGenomics/lz4-rs,MIT,"Jens Heyens , Artem V. Navrotskiy , Patrick Marks " +lz4-sys,https://github.com/10xGenomics/lz4-rs,MIT,"Jens Heyens , Artem V. Navrotskiy , Patrick Marks " lz4_flex,https://github.com/pseitz/lz4_flex,MIT,"Pascal Seitz , Arthur Silva , ticki " macaddr,https://github.com/svartalf/rust-macaddr,Apache-2.0 OR MIT,svartalf mach,https://github.com/fitzgen/mach,BSD-2-Clause,"Nick Fitzgerald , David Cuddeback , Gonzalo Brito Gadeschi " +macro_magic,https://github.com/sam0x17/macro_magic,MIT,sam0x17 +macro_magic_core,https://github.com/sam0x17/macro_magic,MIT,The macro_magic_core Authors +macro_magic_core_macros,https://github.com/sam0x17/macro_magic,MIT,The macro_magic_core_macros Authors +macro_magic_macros,https://github.com/sam0x17/macro_magic,MIT,The macro_magic_macros Authors malloc_buf,https://github.com/SSheldon/malloc_buf,MIT,Steven Sheldon +marrow,https://github.com/chmp/marrow,MIT,Christopher Prohm match_cfg,https://github.com/gnzlbg/match_cfg,MIT OR Apache-2.0,gnzlbg matchers,https://github.com/hawkw/matchers,MIT,Eliza Weisman -matches,https://github.com/SimonSapin/rust-std-candidates,MIT,The matches Authors matchit,https://github.com/ibraheemdev/matchit,MIT AND BSD-3-Clause,Ibraheem Ahmed maxminddb,https://github.com/oschwald/maxminddb-rust,ISC,Gregory J. Oschwald md-5,https://github.com/RustCrypto/hashes,MIT OR Apache-2.0,RustCrypto Developers @@ -394,6 +466,7 @@ memmap2,https://github.com/RazrFalcon/memmap2-rs,MIT OR Apache-2.0,"Dan Burkert memoffset,https://github.com/Gilnaa/memoffset,MIT,Gilad Naaman metrics,https://github.com/metrics-rs/metrics,MIT,Toby Lawrence metrics-tracing-context,https://github.com/metrics-rs/metrics,MIT,MOZGIII +metrics-util,https://github.com/metrics-rs/metrics,MIT,Toby Lawrence mime,https://github.com/hyperium/mime,MIT OR Apache-2.0,Sean McArthur mime_guess,https://github.com/abonander/mime_guess,MIT,Austin Bonander minimal-lexical,https://github.com/Alexhuszagh/minimal-lexical,MIT OR Apache-2.0,Alex Huszagh @@ -403,7 +476,10 @@ mlua,https://github.com/mlua-rs/mlua,MIT,"Aleksandr Orlenko , kyre mlua-sys,https://github.com/mlua-rs/mlua,MIT,Aleksandr Orlenko mlua_derive,https://github.com/khvzak/mlua,MIT,Aleksandr Orlenko moka,https://github.com/moka-rs/moka,MIT OR Apache-2.0,The moka Authors -mongodb,https://github.com/mongodb/mongo-rust-driver,Apache-2.0,"Saghm Rossi , Patrick Freed , Isabel Atkinson , Abraham Egnor , Kaitlin Mahar " +mongocrypt,https://github.com/mongodb/libmongocrypt-rust,Apache-2.0,"Abraham Egnor , Isabel Atkinson " +mongocrypt-sys,https://github.com/mongodb/libmongocrypt-rust,Apache-2.0,"Abraham Egnor , Isabel Atkinson " +mongodb,https://github.com/mongodb/mongo-rust-driver,Apache-2.0,"Saghm Rossi , Patrick Freed , Isabel Atkinson , Abraham Egnor , Kaitlin Mahar , Patrick Meredith " +mongodb-internal-macros,https://github.com/mongodb/mongo-rust-driver,Apache-2.0,The mongodb-internal-macros Authors multer,https://github.com/rousan/multer-rs,MIT,Rousan Ali native-tls,https://github.com/sfackler/rust-native-tls,MIT OR Apache-2.0,Steven Fackler ndk-context,https://github.com/rust-windowing/android-ndk-rs,MIT OR Apache-2.0,The Rust Windowing contributors @@ -419,6 +495,7 @@ no-proxy,https://github.com/jdrouet/no-proxy,MIT,Jérémie Drouet nom,https://github.com/Geal/nom,MIT,contact@geoffroycouprie.com nom,https://github.com/rust-bakery/nom,MIT,contact@geoffroycouprie.com +nom-language,https://github.com/rust-bakery/nom,MIT,contact@geoffroycouprie.com nonzero_ext,https://github.com/antifuchs/nonzero_ext,Apache-2.0,Andreas Fuchs notify,https://github.com/notify-rs/notify,CC0-1.0,"Félix Saparelli , Daniel Faust , Aron Heinecke " notify-types,https://github.com/notify-rs/notify,MIT OR Apache-2.0,Daniel Faust @@ -438,17 +515,17 @@ num-rational,https://github.com/rust-num/num-rational,MIT OR Apache-2.0,The Rust num-traits,https://github.com/rust-num/num-traits,MIT OR Apache-2.0,The Rust Project Developers num_cpus,https://github.com/seanmonstar/num_cpus,MIT OR Apache-2.0,Sean McArthur num_enum,https://github.com/illicitonion/num_enum,BSD-3-Clause OR MIT OR Apache-2.0,"Daniel Wagner-Hall , Daniel Henry-Mantilla , Vincent Esche " +num_enum_derive,https://github.com/illicitonion/num_enum,BSD-3-Clause OR MIT OR Apache-2.0,"Daniel Wagner-Hall , Daniel Henry-Mantilla , Vincent Esche " num_threads,https://github.com/jhpratt/num_threads,MIT OR Apache-2.0,Jacob Pratt -number_prefix,https://github.com/ogham/rust-number-prefix,MIT,Benjamin Sago oauth2,https://github.com/ramosbugs/oauth2-rs,MIT OR Apache-2.0,"Alex Crichton , Florin Lipan , David A. Ramos " objc,http://github.com/SSheldon/rust-objc,MIT,Steven Sheldon objc2-core-foundation,https://github.com/madsmtm/objc2,Zlib OR Apache-2.0 OR MIT,The objc2-core-foundation Authors objc2-io-kit,https://github.com/madsmtm/objc2,Zlib OR Apache-2.0 OR MIT,The objc2-io-kit Authors -object,https://github.com/gimli-rs/object,Apache-2.0 OR MIT,The object Authors octseq,https://github.com/NLnetLabs/octets/,BSD-3-Clause,NLnet Labs ofb,https://github.com/RustCrypto/block-modes,MIT OR Apache-2.0,RustCrypto Developers once_cell,https://github.com/matklad/once_cell,MIT OR Apache-2.0,Aleksey Kladov onig,https://github.com/iwillspeak/rust-onig,MIT,"Will Speak , Ivan Ivashchenko " +onig_sys,https://github.com/iwillspeak/rust-onig,MIT,"Will Speak , Ivan Ivashchenko " opaque-debug,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers opendal,https://github.com/apache/opendal,Apache-2.0,Apache OpenDAL openidconnect,https://github.com/ramosbugs/openidconnect-rs,MIT,David A. Ramos @@ -465,6 +542,8 @@ pad,https://github.com/ogham/rust-pad,MIT,Ben S parking,https://github.com/smol-rs/parking,Apache-2.0 OR MIT,"Stjepan Glavina , The Rust Project Developers" parking_lot,https://github.com/Amanieu/parking_lot,Apache-2.0 OR MIT,Amanieu d'Antras parking_lot,https://github.com/Amanieu/parking_lot,MIT OR Apache-2.0,Amanieu d'Antras +parking_lot_core,https://github.com/Amanieu/parking_lot,Apache-2.0 OR MIT,Amanieu d'Antras +parking_lot_core,https://github.com/Amanieu/parking_lot,MIT OR Apache-2.0,Amanieu d'Antras parse-size,https://github.com/kennytm/parse-size,MIT,kennytm passt,https://github.com/kevingimbel/passt,MIT OR Apache-2.0,Kevin Gimbel paste,https://github.com/dtolnay/paste,MIT OR Apache-2.0,David Tolnay @@ -472,8 +551,13 @@ pbkdf2,https://github.com/RustCrypto/password-hashes/tree/master/pbkdf2,MIT OR A peeking_take_while,https://github.com/fitzgen/peeking_take_while,MIT OR Apache-2.0,Nick Fitzgerald pem,https://github.com/jcreekmore/pem-rs,MIT,Jonathan Creekmore pem-rfc7468,https://github.com/RustCrypto/formats/tree/master/pem-rfc7468,Apache-2.0 OR MIT,RustCrypto Developers +percent-encoding,https://github.com/servo/rust-url,MIT OR Apache-2.0,The rust-url developers pest,https://github.com/pest-parser/pest,MIT OR Apache-2.0,Dragoș Tiselice +pest_derive,https://github.com/pest-parser/pest,MIT OR Apache-2.0,Dragoș Tiselice +pest_generator,https://github.com/pest-parser/pest,MIT OR Apache-2.0,Dragoș Tiselice +pest_meta,https://github.com/pest-parser/pest,MIT OR Apache-2.0,Dragoș Tiselice phf,https://github.com/rust-phf/rust-phf,MIT,Steven Fackler +phf_shared,https://github.com/rust-phf/rust-phf,MIT,Steven Fackler pin-project,https://github.com/taiki-e/pin-project,Apache-2.0 OR MIT,The pin-project Authors pin-project-internal,https://github.com/taiki-e/pin-project,Apache-2.0 OR MIT,The pin-project-internal Authors pin-project-lite,https://github.com/taiki-e/pin-project-lite,Apache-2.0 OR MIT,The pin-project-lite Authors @@ -488,24 +572,29 @@ polling,https://github.com/smol-rs/polling,Apache-2.0 OR MIT,"Stjepan Glavina -postgres-protocol,https://github.com/sfackler/rust-postgres,MIT OR Apache-2.0,Steven Fackler -postgres-types,https://github.com/sfackler/rust-postgres,MIT OR Apache-2.0,Steven Fackler +postgres-protocol,https://github.com/rust-postgres/rust-postgres,MIT OR Apache-2.0,Steven Fackler +postgres-types,https://github.com/rust-postgres/rust-postgres,MIT OR Apache-2.0,Steven Fackler powerfmt,https://github.com/jhpratt/powerfmt,MIT OR Apache-2.0,Jacob Pratt ppv-lite86,https://github.com/cryptocorrosion/cryptocorrosion,MIT OR Apache-2.0,The CryptoCorrosion Contributors prettydiff,https://github.com/romankoblov/prettydiff,MIT,Roman Koblov +prettyplease,https://github.com/dtolnay/prettyplease,MIT OR Apache-2.0,David Tolnay prettytable-rs,https://github.com/phsym/prettytable-rs,BSD-3-Clause,Pierre-Henri Symoneaux primeorder,https://github.com/RustCrypto/elliptic-curves/tree/master/primeorder,Apache-2.0 OR MIT,RustCrypto Developers proc-macro-crate,https://github.com/bkchr/proc-macro-crate,MIT OR Apache-2.0,Bastian Köcher proc-macro-error-attr2,https://github.com/GnomedDev/proc-macro-error-2,MIT OR Apache-2.0,"CreepySkeleton , GnomedDev " proc-macro-error2,https://github.com/GnomedDev/proc-macro-error-2,MIT OR Apache-2.0,"CreepySkeleton , GnomedDev " proc-macro-hack,https://github.com/dtolnay/proc-macro-hack,MIT OR Apache-2.0,David Tolnay +proc-macro-nested,https://github.com/dtolnay/proc-macro-hack,MIT OR Apache-2.0,David Tolnay proc-macro2,https://github.com/dtolnay/proc-macro2,MIT OR Apache-2.0,"David Tolnay , Alex Crichton " proptest,https://github.com/proptest-rs/proptest,MIT OR Apache-2.0,Jason Lingle prost,https://github.com/tokio-rs/prost,Apache-2.0,"Dan Burkert , Lucio Franco , Casper Meijn , Tokio Contributors " +prost-derive,https://github.com/tokio-rs/prost,Apache-2.0,"Dan Burkert , Lucio Franco , Casper Meijn , Tokio Contributors " prost-reflect,https://github.com/andrewhickman/prost-reflect,MIT OR Apache-2.0,Andrew Hickman +prost-types,https://github.com/tokio-rs/prost,Apache-2.0,"Dan Burkert , Lucio Franco , Casper Meijn , Tokio Contributors " psl,https://github.com/addr-rs/psl,MIT OR Apache-2.0,rushmorem psl-types,https://github.com/addr-rs/psl-types,MIT OR Apache-2.0,rushmorem ptr_meta,https://github.com/djkoloski/ptr_meta,MIT,David Koloski +ptr_meta_derive,https://github.com/djkoloski/ptr_meta,MIT,David Koloski publicsuffix,https://github.com/rushmorem/publicsuffix,MIT OR Apache-2.0,rushmorem pulsar,https://github.com/streamnative/pulsar-rs,MIT OR Apache-2.0,"Colin Stearns , Kevin Stenerson , Geoffroy Couprie " quad-rand,https://github.com/not-fl3/quad-rand,MIT,not-fl3 @@ -519,34 +608,44 @@ quinn-proto,https://github.com/quinn-rs/quinn,MIT OR Apache-2.0,The quinn-proto quinn-udp,https://github.com/quinn-rs/quinn,MIT OR Apache-2.0,The quinn-udp Authors quote,https://github.com/dtolnay/quote,MIT OR Apache-2.0,David Tolnay quoted_printable,https://github.com/staktrace/quoted-printable,0BSD,Kartikaya Gupta +r-efi,https://github.com/r-efi/r-efi,MIT OR Apache-2.0 OR LGPL-2.1-or-later,The r-efi Authors radium,https://github.com/bitvecto-rs/radium,MIT,"Nika Layzell , myrrlyn " radix_trie,https://github.com/michaelsproul/rust_radix_trie,MIT,Michael Sproul rand,https://github.com/rust-random/rand,MIT OR Apache-2.0,"The Rand Project Developers, The Rust Project Developers" rand_chacha,https://github.com/rust-random/rand,MIT OR Apache-2.0,"The Rand Project Developers, The Rust Project Developers, The CryptoCorrosion Contributors" +rand_core,https://github.com/rust-random/rand,MIT OR Apache-2.0,"The Rand Project Developers, The Rust Project Developers" rand_distr,https://github.com/rust-random/rand_distr,MIT OR Apache-2.0,The Rand Project Developers -rand_hc,https://github.com/rust-random/rand,MIT OR Apache-2.0,The Rand Project Developers rand_xorshift,https://github.com/rust-random/rngs,MIT OR Apache-2.0,"The Rand Project Developers, The Rust Project Developers" ratatui,https://github.com/ratatui/ratatui,MIT,"Florian Dehau , The Ratatui Developers" +ratatui-core,https://github.com/ratatui/ratatui,MIT,"Florian Dehau , The Ratatui Developers" +ratatui-crossterm,https://github.com/ratatui/ratatui,MIT,"Florian Dehau , The Ratatui Developers" +ratatui-widgets,https://github.com/ratatui/ratatui,MIT,"Florian Dehau , The Ratatui Developers" raw-cpuid,https://github.com/gz/rust-cpuid,MIT,Gerd Zellweger raw-window-handle,https://github.com/rust-windowing/raw-window-handle,MIT OR Apache-2.0 OR Zlib,Osspial rdkafka,https://github.com/fede1024/rust-rdkafka,MIT,Federico Giraud +rdkafka-sys,https://github.com/fede1024/rust-rdkafka,MIT,Federico Giraud +reactor-trait,https://github.com/amqp-rs/executor-trait,Apache-2.0 OR MIT,Marc-Antoine Perennou redis,https://github.com/redis-rs/redis-rs,BSD-3-Clause,The redis Authors redox_syscall,https://gitlab.redox-os.org/redox-os/syscall,MIT,Jeremy Soller redox_users,https://gitlab.redox-os.org/redox-os/users,MIT,"Jose Narvaez , Wesley Hershberger " ref-cast,https://github.com/dtolnay/ref-cast,MIT OR Apache-2.0,David Tolnay +ref-cast-impl,https://github.com/dtolnay/ref-cast,MIT OR Apache-2.0,David Tolnay +referencing,https://github.com/Stranger6667/jsonschema,MIT,Dmitry Dygalo regex,https://github.com/rust-lang/regex,MIT OR Apache-2.0,"The Rust Project Developers, Andrew Gallant " regex-automata,https://github.com/rust-lang/regex/tree/master/regex-automata,MIT OR Apache-2.0,"The Rust Project Developers, Andrew Gallant " regex-filtered,https://github.com/ua-parser/uap-rust,BSD-3-Clause,The regex-filtered Authors -regex-lite,https://github.com/rust-lang/regex/tree/master/regex-lite,MIT OR Apache-2.0,"The Rust Project Developers, Andrew Gallant " +regex-lite,https://github.com/rust-lang/regex,MIT OR Apache-2.0,"The Rust Project Developers, Andrew Gallant " regex-syntax,https://github.com/rust-lang/regex/tree/master/regex-syntax,MIT OR Apache-2.0,"The Rust Project Developers, Andrew Gallant " rend,https://github.com/djkoloski/rend,MIT,David Koloski reqwest,https://github.com/seanmonstar/reqwest,MIT OR Apache-2.0,Sean McArthur reqwest-middleware,https://github.com/TrueLayer/reqwest-middleware,MIT OR Apache-2.0,Rodrigo Gryzinski -resolv-conf,http://github.com/tailhook/resolv-conf,MIT OR Apache-2.0,paul@colomiets.name +reqwest-retry,https://github.com/TrueLayer/reqwest-middleware,MIT OR Apache-2.0,Rodrigo Gryzinski +resolv-conf,https://github.com/hickory-dns/resolv-conf,MIT OR Apache-2.0,The resolv-conf Authors retry-policies,https://github.com/TrueLayer/retry-policies,MIT OR Apache-2.0,Luca Palmieri rfc6979,https://github.com/RustCrypto/signatures/tree/master/rfc6979,Apache-2.0 OR MIT,RustCrypto Developers ring,https://github.com/briansmith/ring,Apache-2.0 AND ISC,The ring Authors rkyv,https://github.com/rkyv/rkyv,MIT,David Koloski +rkyv_derive,https://github.com/rkyv/rkyv,MIT,David Koloski rle-decode-fast,https://github.com/WanzenBug/rle-decode-helper,MIT OR Apache-2.0,Moritz Wanzenböck rmp,https://github.com/3Hren/msgpack-rust,MIT,Evgeny Safronov rmp-serde,https://github.com/3Hren/msgpack-rust,MIT,Evgeny Safronov @@ -556,9 +655,7 @@ roxmltree,https://github.com/RazrFalcon/roxmltree,MIT OR Apache-2.0,Yevhenii Rei rsa,https://github.com/RustCrypto/RSA,MIT OR Apache-2.0,"RustCrypto Developers, dignifiedquire " rumqttc,https://github.com/bytebeamio/rumqtt,Apache-2.0,tekjar rust_decimal,https://github.com/paupino/rust-decimal,MIT,Paul Mason -rustc-demangle,https://github.com/rust-lang/rustc-demangle,MIT OR Apache-2.0,Alex Crichton rustc-hash,https://github.com/rust-lang/rustc-hash,Apache-2.0 OR MIT,The Rust Project Developers -rustc_version,https://github.com/Kimundi/rustc-version-rs,MIT OR Apache-2.0,Marvin Löbel rustc_version,https://github.com/djc/rustc-version-rs,MIT OR Apache-2.0,The rustc_version Authors rustc_version_runtime,https://github.com/seppo0010/rustc-version-runtime-rs,MIT,Sebastian Waisbrot rustix,https://github.com/bytecodealliance/rustix,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,"Dan Gohman , Jakub Konka " @@ -584,18 +681,21 @@ seahash,https://gitlab.redox-os.org/redox-os/seahash,MIT,"ticki security-framework,https://github.com/kornelski/rust-security-framework,MIT OR Apache-2.0,"Steven Fackler , Kornel " +security-framework-sys,https://github.com/kornelski/rust-security-framework,MIT OR Apache-2.0,"Steven Fackler , Kornel " semver,https://github.com/dtolnay/semver,MIT OR Apache-2.0,David Tolnay -semver,https://github.com/steveklabnik/semver,MIT OR Apache-2.0,"Steve Klabnik , The Rust Project Developers" -semver-parser,https://github.com/steveklabnik/semver-parser,MIT OR Apache-2.0,Steve Klabnik serde,https://github.com/serde-rs/serde,MIT OR Apache-2.0,"Erick Tryzelaar , David Tolnay " +serde-aux,https://github.com/iddm/serde-aux,MIT,Victor Polevoy serde-toml-merge,https://github.com/jdrouet/serde-toml-merge,MIT,Jeremie Drouet serde-value,https://github.com/arcnmx/serde-value,MIT,arcnmx +serde_arrow,https://github.com/chmp/serde_arrow,MIT,Christopher Prohm serde_bytes,https://github.com/serde-rs/bytes,MIT OR Apache-2.0,David Tolnay +serde_core,https://github.com/serde-rs/serde,MIT OR Apache-2.0,"Erick Tryzelaar , David Tolnay " +serde_derive,https://github.com/serde-rs/serde,MIT OR Apache-2.0,"Erick Tryzelaar , David Tolnay " +serde_derive_internals,https://github.com/serde-rs/serde,MIT OR Apache-2.0,"Erick Tryzelaar , David Tolnay " serde_json,https://github.com/serde-rs/json,MIT OR Apache-2.0,"Erick Tryzelaar , David Tolnay " serde_nanos,https://github.com/caspervonb/serde_nanos,MIT OR Apache-2.0,Casper Beyer serde_path_to_error,https://github.com/dtolnay/path-to-error,MIT OR Apache-2.0,David Tolnay serde_plain,https://github.com/mitsuhiko/serde-plain,MIT OR Apache-2.0,Armin Ronacher -serde_qs,https://github.com/samscott89/serde_qs,MIT OR Apache-2.0,Sam Scott serde_repr,https://github.com/dtolnay/serde-repr,MIT OR Apache-2.0,David Tolnay serde_spanned,https://github.com/toml-rs/toml,MIT OR Apache-2.0,The serde_spanned Authors serde_urlencoded,https://github.com/nox/serde_urlencoded,MIT OR Apache-2.0,Anthony Ramine @@ -608,6 +708,7 @@ sha2,https://github.com/RustCrypto/hashes,MIT OR Apache-2.0,RustCrypto Developer sha3,https://github.com/RustCrypto/hashes,MIT OR Apache-2.0,RustCrypto Developers sharded-slab,https://github.com/hawkw/sharded-slab,MIT,Eliza Weisman signal-hook,https://github.com/vorner/signal-hook,Apache-2.0 OR MIT,"Michal 'vorner' Vaner , Thomas Himmelstoss " +signal-hook-mio,https://github.com/vorner/signal-hook,Apache-2.0 OR MIT,"Michal 'vorner' Vaner , Thomas Himmelstoss " signal-hook-registry,https://github.com/vorner/signal-hook,Apache-2.0 OR MIT,"Michal 'vorner' Vaner , Masaki Hara " signatory,https://github.com/iqlusioninc/crates/tree/main/signatory,Apache-2.0 OR MIT,Tony Arcieri signature,https://github.com/RustCrypto/traits/tree/master/signature,Apache-2.0 OR MIT,RustCrypto Developers @@ -620,6 +721,7 @@ smallvec,https://github.com/servo/rust-smallvec,MIT OR Apache-2.0,The Servo Proj smol,https://github.com/smol-rs/smol,Apache-2.0 OR MIT,Stjepan Glavina smpl_jwt,https://github.com/durch/rust-jwt,MIT,Drazen Urch snafu,https://github.com/shepmaster/snafu,MIT OR Apache-2.0,Jake Goulding +snafu-derive,https://github.com/shepmaster/snafu,MIT OR Apache-2.0,Jake Goulding snap,https://github.com/BurntSushi/rust-snappy,BSD-3-Clause,Andrew Gallant socket2,https://github.com/rust-lang/socket2,MIT OR Apache-2.0,"Alex Crichton , Thomas de Zeeuw " spin,https://github.com/mvdnes/spin-rs,MIT,"Mathijs van de Nes , John Ericson " @@ -627,15 +729,21 @@ spin,https://github.com/mvdnes/spin-rs,MIT,"Mathijs van de Nes spki,https://github.com/RustCrypto/formats/tree/master/spki,Apache-2.0 OR MIT,RustCrypto Developers sqlx,https://github.com/launchbadge/sqlx,MIT OR Apache-2.0,"Ryan Leckey , Austin Bonander , Chloe Ross , Daniel Akhterov " +sqlx-core,https://github.com/launchbadge/sqlx,MIT OR Apache-2.0,"Ryan Leckey , Austin Bonander , Chloe Ross , Daniel Akhterov " +sqlx-macros,https://github.com/launchbadge/sqlx,MIT OR Apache-2.0,"Ryan Leckey , Austin Bonander , Chloe Ross , Daniel Akhterov " +sqlx-macros-core,https://github.com/launchbadge/sqlx,MIT OR Apache-2.0,"Ryan Leckey , Austin Bonander , Chloe Ross , Daniel Akhterov " +sqlx-mysql,https://github.com/launchbadge/sqlx,MIT OR Apache-2.0,"Ryan Leckey , Austin Bonander , Chloe Ross , Daniel Akhterov " +sqlx-postgres,https://github.com/launchbadge/sqlx,MIT OR Apache-2.0,"Ryan Leckey , Austin Bonander , Chloe Ross , Daniel Akhterov " +sqlx-sqlite,https://github.com/launchbadge/sqlx,MIT OR Apache-2.0,"Ryan Leckey , Austin Bonander , Chloe Ross , Daniel Akhterov " stable_deref_trait,https://github.com/storyyeller/stable_deref_trait,MIT OR Apache-2.0,Robert Grosse static_assertions,https://github.com/nvzqz/static-assertions-rs,MIT OR Apache-2.0,Nikolai Vazquez static_assertions_next,https://github.com/scuffletv/static-assertions,MIT OR Apache-2.0,Nikolai Vazquez stream-cancel,https://github.com/jonhoo/stream-cancel,MIT OR Apache-2.0,Jon Gjengset stringprep,https://github.com/sfackler/rust-stringprep,MIT OR Apache-2.0,Steven Fackler strip-ansi-escapes,https://github.com/luser/strip-ansi-escapes,Apache-2.0 OR MIT,Ted Mielczarek -strsim,https://github.com/dguo/strsim-rs,MIT,Danny Guo strsim,https://github.com/rapidfuzz/strsim-rs,MIT,"Danny Guo , maxbachmann " strum,https://github.com/Peternator7/strum,MIT,Peter Glotfelty +strum_macros,https://github.com/Peternator7/strum,MIT,Peter Glotfelty subtle,https://github.com/dalek-cryptography/subtle,BSD-3-Clause,"Isis Lovecruft , Henry de Valence " supports-color,https://github.com/zkat/supports-color,Apache-2.0,Kat Marchán syn,https://github.com/dtolnay/syn,MIT OR Apache-2.0,David Tolnay @@ -645,6 +753,7 @@ sysinfo,https://github.com/GuillaumeGomez/sysinfo,MIT,Guillaume Gomez system-configuration,https://github.com/mullvad/system-configuration-rs,MIT OR Apache-2.0,Mullvad VPN +system-configuration-sys,https://github.com/mullvad/system-configuration-rs,MIT OR Apache-2.0,Mullvad VPN tagptr,https://github.com/oliver-giersch/tagptr,MIT OR Apache-2.0,Oliver Giersch take_mut,https://github.com/Sgeo/take_mut,MIT,Sgeo tap,https://github.com/myrrlyn/tap,MIT,"Elliott Linder , myrrlyn " @@ -654,22 +763,29 @@ term,https://github.com/Stebalien/term,MIT OR Apache-2.0,"The Rust Project Devel termcolor,https://github.com/BurntSushi/termcolor,Unlicense OR MIT,Andrew Gallant terminal_size,https://github.com/eminence/terminal-size,MIT OR Apache-2.0,Andrew Chin thiserror,https://github.com/dtolnay/thiserror,MIT OR Apache-2.0,David Tolnay +thiserror-impl,https://github.com/dtolnay/thiserror,MIT OR Apache-2.0,David Tolnay thread_local,https://github.com/Amanieu/thread_local-rs,MIT OR Apache-2.0,Amanieu d'Antras tikv-jemalloc-sys,https://github.com/tikv/jemallocator,MIT OR Apache-2.0,"Alex Crichton , Gonzalo Brito Gadeschi , The TiKV Project Developers" tikv-jemallocator,https://github.com/tikv/jemallocator,MIT OR Apache-2.0,"Alex Crichton , Gonzalo Brito Gadeschi , Simon Sapin , Steven Fackler , The TiKV Project Developers" time,https://github.com/time-rs/time,MIT OR Apache-2.0,"Jacob Pratt , Time contributors" +time-core,https://github.com/time-rs/time,MIT OR Apache-2.0,"Jacob Pratt , Time contributors" +time-macros,https://github.com/time-rs/time,MIT OR Apache-2.0,"Jacob Pratt , Time contributors" +tiny-keccak,https://github.com/debris/tiny-keccak,CC0-1.0,debris tinystr,https://github.com/unicode-org/icu4x,Unicode-3.0,The ICU4X Project Developers tinyvec,https://github.com/Lokathor/tinyvec,Zlib OR Apache-2.0 OR MIT,Lokathor tinyvec_macros,https://github.com/Soveu/tinyvec_macros,MIT OR Apache-2.0 OR Zlib,Soveu tokio,https://github.com/tokio-rs/tokio,MIT,Tokio Contributors tokio-io,https://github.com/tokio-rs/tokio,MIT,Carl Lerche tokio-io-timeout,https://github.com/sfackler/tokio-io-timeout,MIT OR Apache-2.0,Steven Fackler +tokio-macros,https://github.com/tokio-rs/tokio,MIT,Tokio Contributors tokio-native-tls,https://github.com/tokio-rs/tls,MIT,Tokio Contributors tokio-openssl,https://github.com/tokio-rs/tokio-openssl,MIT OR Apache-2.0,Alex Crichton -tokio-postgres,https://github.com/sfackler/rust-postgres,MIT OR Apache-2.0,Steven Fackler +tokio-postgres,https://github.com/rust-postgres/rust-postgres,MIT OR Apache-2.0,Steven Fackler tokio-retry,https://github.com/srijs/rust-tokio-retry,MIT,Sam Rijs tokio-rustls,https://github.com/rustls/tokio-rustls,MIT OR Apache-2.0,The tokio-rustls Authors +tokio-stream,https://github.com/tokio-rs/tokio,MIT,Tokio Contributors tokio-tungstenite,https://github.com/snapview/tokio-tungstenite,MIT,"Daniel Abramov , Alexey Galakhov " +tokio-util,https://github.com/tokio-rs/tokio,MIT,Tokio Contributors tokio-websockets,https://github.com/Gelbpunkt/tokio-websockets,MIT,The tokio-websockets Authors toml,https://github.com/toml-rs/toml,MIT OR Apache-2.0,The toml Authors toml_datetime,https://github.com/toml-rs/toml,MIT OR Apache-2.0,The toml_datetime Authors @@ -681,36 +797,42 @@ toml_writer,https://github.com/toml-rs/toml,MIT OR Apache-2.0,The toml_writer Au tonic,https://github.com/hyperium/tonic,MIT,Lucio Franco tower,https://github.com/tower-rs/tower,MIT,Tower Maintainers tower-http,https://github.com/tower-rs/tower-http,MIT,Tower Maintainers +tower-layer,https://github.com/tower-rs/tower,MIT,Tower Maintainers +tower-service,https://github.com/tower-rs/tower,MIT,Tower Maintainers tracing,https://github.com/tokio-rs/tracing,MIT,"Eliza Weisman , Tokio Contributors " tracing-attributes,https://github.com/tokio-rs/tracing,MIT,"Tokio Contributors , Eliza Weisman , David Barsky " tracing-core,https://github.com/tokio-rs/tracing,MIT,Tokio Contributors +tracing-futures,https://github.com/tokio-rs/tracing,MIT,"Eliza Weisman , Tokio Contributors " tracing-log,https://github.com/tokio-rs/tracing,MIT,Tokio Contributors tracing-serde,https://github.com/tokio-rs/tracing,MIT,Tokio Contributors tracing-subscriber,https://github.com/tokio-rs/tracing,MIT,"Eliza Weisman , David Barsky , Tokio Contributors " tracing-tower,https://github.com/tokio-rs/tracing,MIT,Eliza Weisman triomphe,https://github.com/Manishearth/triomphe,MIT OR Apache-2.0,"Manish Goregaokar , The Servo Project Developers" -trust-dns-proto,https://github.com/bluejekyll/trust-dns,MIT OR Apache-2.0,Benjamin Fry -trust-dns-resolver,https://github.com/bluejekyll/trust-dns,MIT OR Apache-2.0,Benjamin Fry try-lock,https://github.com/seanmonstar/try-lock,MIT,Sean McArthur tryhard,https://github.com/EmbarkStudios/tryhard,MIT OR Apache-2.0,Embark tungstenite,https://github.com/snapview/tungstenite-rs,MIT OR Apache-2.0,"Alexey Galakhov, Daniel Abramov" twox-hash,https://github.com/shepmaster/twox-hash,MIT,Jake Goulding typed-builder,https://github.com/idanarye/rust-typed-builder,MIT OR Apache-2.0,"IdanArye , Chris Morgan " +typed-builder-macro,https://github.com/idanarye/rust-typed-builder,MIT OR Apache-2.0,"IdanArye , Chris Morgan " typenum,https://github.com/paholg/typenum,MIT OR Apache-2.0,"Paho Lurie-Gregg , Andre Bogus " typespec,https://github.com/azure/azure-sdk-for-rust,MIT,Microsoft typespec_client_core,https://github.com/azure/azure-sdk-for-rust,MIT,Microsoft typespec_macros,https://github.com/azure/azure-sdk-for-rust,MIT,Microsoft typetag,https://github.com/dtolnay/typetag,MIT OR Apache-2.0,David Tolnay +typetag-impl,https://github.com/dtolnay/typetag,MIT OR Apache-2.0,David Tolnay ua-parser,https://github.com/ua-parser/uap-rust,Apache-2.0,The ua-parser Authors ucd-trie,https://github.com/BurntSushi/ucd-generate,MIT OR Apache-2.0,Andrew Gallant unarray,https://github.com/cameron1024/unarray,MIT OR Apache-2.0,The unarray Authors unicase,https://github.com/seanmonstar/unicase,MIT OR Apache-2.0,Sean McArthur unicode-bidi,https://github.com/servo/unicode-bidi,MIT OR Apache-2.0,The Servo Project Developers +unicode-general-category,https://github.com/yeslogic/unicode-general-category,Apache-2.0,YesLogic Pty. Ltd. unicode-ident,https://github.com/dtolnay/unicode-ident,(MIT OR Apache-2.0) AND Unicode-DFS-2016,David Tolnay unicode-normalization,https://github.com/unicode-rs/unicode-normalization,MIT OR Apache-2.0,"kwantam , Manish Goregaokar " unicode-segmentation,https://github.com/unicode-rs/unicode-segmentation,MIT OR Apache-2.0,"kwantam , Manish Goregaokar " unicode-truncate,https://github.com/Aetf/unicode-truncate,MIT OR Apache-2.0,Aetf unicode-width,https://github.com/unicode-rs/unicode-width,MIT OR Apache-2.0,"kwantam , Manish Goregaokar " +unicode-xid,https://github.com/unicode-rs/unicode-xid,MIT OR Apache-2.0,"erick.tryzelaar , kwantam , Manish Goregaokar " +unit-prefix,https://codeberg.org/commons-rs/unit-prefix,MIT,"Fabio Valentini , Benjamin Sago " universal-hash,https://github.com/RustCrypto/traits,MIT OR Apache-2.0,RustCrypto Developers unreachable,https://github.com/reem/rust-unreachable,MIT OR Apache-2.0,Jonathan Reem unsafe-libyaml,https://github.com/dtolnay/unsafe-libyaml,MIT,David Tolnay @@ -722,6 +844,7 @@ utf-8,https://github.com/SimonSapin/rust-utf8,MIT OR Apache-2.0,Simon Sapin utf8-width,https://github.com/magiclen/utf8-width,MIT,Magic Len utf8_iter,https://github.com/hsivonen/utf8_iter,Apache-2.0 OR MIT,Henri Sivonen +utf8parse,https://github.com/alacritty/vte,Apache-2.0 OR MIT,"Joe Wilm , Christian Duerr " uuid,https://github.com/uuid-rs/uuid,Apache-2.0 OR MIT,"Ashley Mannix, Dylan DPC, Hunar Roop Kahlon" uuid-simd,https://github.com/Nugine/simd,MIT,The uuid-simd Authors valuable,https://github.com/tokio-rs/valuable,MIT,The valuable Authors @@ -735,7 +858,7 @@ walkdir,https://github.com/BurntSushi/walkdir,Unlicense OR MIT,Andrew Gallant warp,https://github.com/seanmonstar/warp,MIT,Sean McArthur wasi,https://github.com/bytecodealliance/wasi,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,The Cranelift Project Developers -wasi,https://github.com/bytecodealliance/wasi-rs,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,The Cranelift Project Developers +wasip2,https://github.com/bytecodealliance/wasi-rs,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,The wasip2 Authors wasite,https://github.com/ardaku/wasite,Apache-2.0 OR BSL-1.0 OR MIT,The wasite Authors wasm-bindgen,https://github.com/rustwasm/wasm-bindgen,MIT OR Apache-2.0,The wasm-bindgen Developers wasm-bindgen-backend,https://github.com/rustwasm/wasm-bindgen/tree/master/crates/backend,MIT OR Apache-2.0,The wasm-bindgen Developers @@ -748,20 +871,42 @@ wasm-timer,https://github.com/tomaka/wasm-timer,MIT,Pierre Krieger widestring,https://github.com/starkat99/widestring-rs,MIT OR Apache-2.0,The widestring Authors winapi,https://github.com/retep998/winapi-rs,MIT OR Apache-2.0,Peter Atashian +winapi-i686-pc-windows-gnu,https://github.com/retep998/winapi-rs,MIT OR Apache-2.0,Peter Atashian winapi-util,https://github.com/BurntSushi/winapi-util,Unlicense OR MIT,Andrew Gallant +winapi-x86_64-pc-windows-gnu,https://github.com/retep998/winapi-rs,MIT OR Apache-2.0,Peter Atashian windows,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft windows-collections,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The windows-collections Authors +windows-core,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft windows-future,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The windows-future Authors +windows-implement,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows-interface,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows-link,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows-link,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The windows-link Authors windows-numerics,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The windows-numerics Authors +windows-registry,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows-result,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft windows-service,https://github.com/mullvad/windows-service-rs,MIT OR Apache-2.0,Mullvad VPN +windows-strings,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows-sys,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows-sys,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The windows-sys Authors +windows-targets,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows_aarch64_gnullvm,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows_aarch64_msvc,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows_i686_gnu,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows_i686_gnullvm,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows_i686_msvc,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows_x86_64_gnu,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows_x86_64_gnullvm,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows_x86_64_msvc,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft winnow,https://github.com/winnow-rs/winnow,MIT,The winnow Authors winreg,https://github.com/gentoo90/winreg-rs,MIT,Igor Shaula -wit-bindgen-rt,https://github.com/bytecodealliance/wasi-rs,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,The wit-bindgen-rt Authors +wit-bindgen,https://github.com/bytecodealliance/wit-bindgen,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,Alex Crichton woothee,https://github.com/woothee/woothee-rust,Apache-2.0,hhatto write16,https://github.com/hsivonen/write16,Apache-2.0 OR MIT,The write16 Authors writeable,https://github.com/unicode-org/icu4x,Unicode-3.0,The ICU4X Project Developers @@ -771,6 +916,7 @@ xxhash-rust,https://github.com/DoumanAsh/xxhash-rust,BSL-1.0,Douman yoke-derive,https://github.com/unicode-org/icu4x,Unicode-3.0,Manish Goregaokar zerocopy,https://github.com/google/zerocopy,BSD-2-Clause OR Apache-2.0 OR MIT,Joshua Liebow-Feeser +zerocopy-derive,https://github.com/google/zerocopy,BSD-2-Clause OR Apache-2.0 OR MIT,Joshua Liebow-Feeser zerofrom,https://github.com/unicode-org/icu4x,Unicode-3.0,Manish Goregaokar zerofrom-derive,https://github.com/unicode-org/icu4x,Unicode-3.0,Manish Goregaokar zeroize,https://github.com/RustCrypto/utils/tree/master/zeroize,Apache-2.0 OR MIT,The RustCrypto Project Developers diff --git a/Makefile b/Makefile index e5fc317a5249b..f259f6c200cca 100644 --- a/Makefile +++ b/Makefile @@ -62,12 +62,23 @@ export ENVIRONMENT_NETWORK ?= host # Multiple port publishing can be provided using spaces, for example: 8686:8686 8080:8080/udp export ENVIRONMENT_PUBLISH ?= +# If ENVIRONMENT is true, always use cargo vdev since it may be running inside the container +ifeq ($(origin VDEV), environment) +ifeq ($(ENVIRONMENT), true) +VDEV := cargo vdev +else +# VDEV is already set from environment, keep it +endif +else +VDEV := cargo vdev +endif + # Set dummy AWS credentials if not present - used for AWS and ES integration tests export AWS_ACCESS_KEY_ID ?= "dummy" export AWS_SECRET_ACCESS_KEY ?= "dummy" # Set version -export VERSION ?= $(shell command -v cargo >/dev/null && cargo vdev version || echo unknown) +export VERSION ?= $(shell command -v cargo >/dev/null && $(VDEV) version || echo unknown) # Set if you are on the CI and actually want the things to happen. (Non-CI users should never set this.) export CI ?= false @@ -379,10 +390,10 @@ test-integration: test-integration-redis test-integration-splunk test-integratio test-integration: test-integration-datadog-traces test-integration-shutdown test-integration-%-cleanup: - cargo vdev --verbose integration stop $* + $(VDEV) --verbose integration stop $* test-integration-%: - cargo vdev --verbose integration test $* + $(VDEV) --verbose integration test $* ifeq ($(AUTODESPAWN), true) make test-integration-$*-cleanup endif @@ -451,7 +462,7 @@ bench-all: bench-remap-functions .PHONY: check check: ## Run prerequisite code checks - ${MAYBE_ENVIRONMENT_EXEC} cargo vdev check rust + ${MAYBE_ENVIRONMENT_EXEC} $(VDEV) check rust .PHONY: check-all check-all: ## Check everything @@ -461,47 +472,47 @@ check-all: check-scripts check-deny check-component-docs check-licenses .PHONY: check-component-features check-component-features: ## Check that all component features are setup properly - ${MAYBE_ENVIRONMENT_EXEC} cargo vdev check component-features + ${MAYBE_ENVIRONMENT_EXEC} $(VDEV) check component-features .PHONY: check-clippy check-clippy: ## Check code with Clippy - ${MAYBE_ENVIRONMENT_EXEC} cargo vdev check rust + ${MAYBE_ENVIRONMENT_EXEC} $(VDEV) check rust .PHONY: check-docs check-docs: ## Check that all /docs file are valid - ${MAYBE_ENVIRONMENT_EXEC} cargo vdev check docs + ${MAYBE_ENVIRONMENT_EXEC} $(VDEV) check docs .PHONY: check-fmt check-fmt: ## Check that all files are formatted properly - ${MAYBE_ENVIRONMENT_EXEC} cargo vdev check fmt + ${MAYBE_ENVIRONMENT_EXEC} $(VDEV) check fmt .PHONY: check-licenses check-licenses: ## Check that the 3rd-party license file is up to date - ${MAYBE_ENVIRONMENT_EXEC} cargo vdev check licenses + ${MAYBE_ENVIRONMENT_EXEC} $(VDEV) check licenses .PHONY: check-markdown check-markdown: ## Check that markdown is styled properly - ${MAYBE_ENVIRONMENT_EXEC} cargo vdev check markdown + ${MAYBE_ENVIRONMENT_EXEC} $(VDEV) check markdown .PHONY: check-examples check-examples: ## Check that the config/examples files are valid - ${MAYBE_ENVIRONMENT_EXEC} cargo vdev check examples + ${MAYBE_ENVIRONMENT_EXEC} $(VDEV) check examples .PHONY: check-scripts check-scripts: ## Check that scripts do not have common mistakes - ${MAYBE_ENVIRONMENT_EXEC} cargo vdev check scripts + ${MAYBE_ENVIRONMENT_EXEC} $(VDEV) check scripts .PHONY: check-deny check-deny: ## Check advisories licenses and sources for crate dependencies - ${MAYBE_ENVIRONMENT_EXEC} cargo vdev check deny + ${MAYBE_ENVIRONMENT_EXEC} $(VDEV) check deny .PHONY: check-events check-events: ## Check that events satisfy patterns set in https://github.com/vectordotdev/vector/blob/master/rfcs/2020-03-17-2064-event-driven-observability.md - ${MAYBE_ENVIRONMENT_EXEC} cargo vdev check events + ${MAYBE_ENVIRONMENT_EXEC} $(VDEV) check events .PHONY: check-component-docs check-component-docs: generate-component-docs ## Checks that the machine-generated component Cue docs are up-to-date. - ${MAYBE_ENVIRONMENT_EXEC} cargo vdev check component-docs + ${MAYBE_ENVIRONMENT_EXEC} $(VDEV) check component-docs ##@ Rustdoc build-rustdoc: ## Build Vector's Rustdocs @@ -522,7 +533,7 @@ target/artifacts/vector-${VERSION}-%.tar.gz: target/%/release/vector.tar.gz .PHONY: package package: build ## Build the Vector archive - ${MAYBE_ENVIRONMENT_EXEC} cargo vdev package archive + ${MAYBE_ENVIRONMENT_EXEC} $(VDEV) package archive .PHONY: package-x86_64-unknown-linux-gnu-all package-x86_64-unknown-linux-gnu-all: package-x86_64-unknown-linux-gnu package-deb-x86_64-unknown-linux-gnu package-rpm-x86_64-unknown-linux-gnu # Build all x86_64 GNU packages @@ -621,31 +632,31 @@ release: release-prepare generate release-commit ## Release a new Vector version .PHONY: release-commit release-commit: ## Commits release changes - @cargo vdev release commit + @$(VDEV) release commit .PHONY: release-docker release-docker: ## Release to Docker Hub - @cargo vdev release docker + @$(VDEV) release docker .PHONY: release-github release-github: ## Release to GitHub - @cargo vdev release github + @$(VDEV) release github .PHONY: release-homebrew release-homebrew: ## Release to vectordotdev Homebrew tap - @cargo vdev release homebrew --vector-version $(VECTOR_VERSION) + @$(VDEV) release homebrew --vector-version $(VECTOR_VERSION) .PHONY: release-prepare release-prepare: ## Prepares the release with metadata and highlights - @cargo vdev release prepare + @$(VDEV) release prepare .PHONY: release-push release-push: ## Push new Vector version - @cargo vdev release push + @$(VDEV) release push .PHONY: release-s3 release-s3: ## Release artifacts to S3 - @cargo vdev release s3 + @$(VDEV) release s3 .PHONY: sha256sum sha256sum: ## Generate SHA256 checksums of CI artifacts @@ -655,11 +666,11 @@ sha256sum: ## Generate SHA256 checksums of CI artifacts .PHONY: test-vrl test-vrl: ## Run the VRL test suite - @cargo vdev test-vrl + @$(VDEV) test-vrl .PHONY: compile-vrl-wasm compile-vrl-wasm: ## Compile VRL crates to WASM target - cargo vdev build vrl-wasm + $(VDEV) build vrl-wasm ##@ Utility @@ -669,13 +680,13 @@ clean: environment-clean ## Clean everything .PHONY: generate-kubernetes-manifests generate-kubernetes-manifests: ## Generate Kubernetes manifests from latest Helm chart - cargo vdev build manifests + $(VDEV) build manifests .PHONY: generate-component-docs generate-component-docs: ## Generate per-component Cue docs from the configuration schema. ${MAYBE_ENVIRONMENT_EXEC} cargo build $(if $(findstring true,$(CI)),--quiet,) target/debug/vector generate-schema > /tmp/vector-config-schema.json 2>/dev/null - ${MAYBE_ENVIRONMENT_EXEC} cargo vdev build component-docs /tmp/vector-config-schema.json \ + ${MAYBE_ENVIRONMENT_EXEC} $(VDEV) build component-docs /tmp/vector-config-schema.json \ $(if $(findstring true,$(CI)),>/dev/null,) .PHONY: signoff @@ -684,7 +695,7 @@ signoff: ## Signsoff all previous commits since branch creation .PHONY: version version: ## Get the current Vector version - @cargo vdev version + @$(VDEV) version .PHONY: git-hooks git-hooks: ## Add Vector-local git hooks for commit sign-off @@ -697,16 +708,16 @@ cargo-install-%: .PHONY: ci-generate-publish-metadata ci-generate-publish-metadata: ## Generates the necessary metadata required for building/publishing Vector. - cargo vdev build publish-metadata + $(VDEV) build publish-metadata .PHONY: clippy-fix clippy-fix: - ${MAYBE_ENVIRONMENT_EXEC} cargo vdev check rust --fix + ${MAYBE_ENVIRONMENT_EXEC} $(VDEV) check rust --fix .PHONY: fmt fmt: - ${MAYBE_ENVIRONMENT_EXEC} cargo vdev fmt + ${MAYBE_ENVIRONMENT_EXEC} $(VDEV) fmt .PHONY: build-licenses build-licenses: - ${MAYBE_ENVIRONMENT_EXEC} cargo vdev build licenses + ${MAYBE_ENVIRONMENT_EXEC} $(VDEV) build licenses diff --git a/README.md b/README.md index bf4db29273e00..1e5b13adf3ce2 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ [![Nightly](https://github.com/vectordotdev/vector/actions/workflows/nightly.yml/badge.svg)](https://github.com/vectordotdev/vector/actions/workflows/nightly.yml) -[![E2E Test Suite](https://github.com/vectordotdev/vector/actions/workflows/e2e.yml/badge.svg)](https://github.com/vectordotdev/vector/actions/workflows/e2e.yml) +[![Integration/E2E Test Suite](https://github.com/vectordotdev/vector/actions/workflows/integration.yml/badge.svg)](https://github.com/vectordotdev/vector/actions/workflows/integration.yml/badge.svg?event=merge_group) [![Component Features](https://github.com/vectordotdev/vector/actions/workflows/component_features.yml/badge.svg)](https://github.com/vectordotdev/vector/actions/workflows/component_features.yml)

diff --git a/aqua/aqua.yaml b/aqua/aqua.yaml index 37c8ae623bbe8..e55f3daaddcad 100644 --- a/aqua/aqua.yaml +++ b/aqua/aqua.yaml @@ -5,10 +5,13 @@ registries: - type: standard ref: v4.268.0 # renovate: depName=aquaproj/aqua-registry packages: - - name: rustwasm/wasm-pack@v0.13.1 - - name: crates.io/cargo-deb@2.9.3 - - name: cross-rs/cross@v0.2.5 - - name: nextest-rs/nextest/cargo-nextest@cargo-nextest-0.9.47 - - name: EmbarkStudios/cargo-deny@0.16.2 - - name: foresterre/cargo-msrv@v0.15.1 - - name: crates.io/dd-rust-license-tool@1.0.1 + - name: rustwasm/wasm-pack@v0.13.1 # Build and package WebAssembly + - name: crates.io/cargo-deb@2.9.3 # Create Debian packages + - name: cross-rs/cross@v0.2.5 # Cross-compile Rust projects + - name: nextest-rs/nextest/cargo-nextest@cargo-nextest-0.9.47 # Rust test runner + - name: EmbarkStudios/cargo-deny@0.16.2 # Lint dependencies for security and licenses + - name: foresterre/cargo-msrv@v0.15.1 # Find minimum supported Rust version + - name: crates.io/dd-rust-license-tool@1.0.4 # Generate license information + - name: bufbuild/buf@v1.50.0 # Protobuf linting and breaking change detection + - name: cue-lang/cue@v0.11.2 # Validate and generate CUE configuration schemas + - name: nodejs/node@v22.12.0 # JavaScript runtime for website builds diff --git a/benches/http.rs b/benches/http.rs index e8ed333af5be5..0bc99dec551cf 100644 --- a/benches/http.rs +++ b/benches/http.rs @@ -15,7 +15,7 @@ use vector::{ }, sources, template::Template, - test_util::{next_addr, random_lines, runtime, send_lines, start_topology, wait_for_tcp}, + test_util::{addr::next_addr, random_lines, runtime, send_lines, start_topology, wait_for_tcp}, }; use vector_lib::codecs::{TextSerializerConfig, encoding::FramingConfig}; @@ -23,8 +23,8 @@ fn benchmark_http(c: &mut Criterion) { let num_lines: usize = 1_000; let line_size: usize = 100; - let in_addr = next_addr(); - let out_addr = next_addr(); + let (_guard_0, in_addr) = next_addr(); + let (_guard_1, out_addr) = next_addr(); let _srv = serve(out_addr); diff --git a/benches/languages.rs b/benches/languages.rs index ae99eb81cd45d..5294742e1b114 100644 --- a/benches/languages.rs +++ b/benches/languages.rs @@ -2,7 +2,9 @@ use criterion::{BatchSize, Criterion, SamplingMode, Throughput, criterion_group, use indoc::indoc; use vector::{ config, - test_util::{CountReceiver, next_addr, runtime, send_lines, start_topology, wait_for_tcp}, + test_util::{ + CountReceiver, addr::next_addr, runtime, send_lines, start_topology, wait_for_tcp, + }, }; criterion_group!( @@ -253,8 +255,8 @@ fn benchmark_configs( let _ = output; let num_lines = 10_000; - let in_addr = next_addr(); - let out_addr = next_addr(); + let (_guard_0, in_addr) = next_addr(); + let (_guard_1, out_addr) = next_addr(); let lines: Vec<_> = std::iter::repeat_n(input.to_string(), num_lines).collect(); diff --git a/benches/transform/reduce.rs b/benches/transform/reduce.rs index ac71f113249db..898ef4f88be40 100644 --- a/benches/transform/reduce.rs +++ b/benches/transform/reduce.rs @@ -54,7 +54,12 @@ fn reduce(c: &mut Criterion) { .iter_batched( || { let reduce = Transform::event_task( - Reduce::new(¶m.reduce_config, &Default::default()).unwrap(), + Reduce::new( + ¶m.reduce_config, + &Default::default(), + &Default::default(), + ) + .unwrap(), ) .into_task(); (Box::new(reduce), Box::pin(param.input.clone())) diff --git a/changelog.d/23671_file_sink_truncate.feature.md b/changelog.d/23671_file_sink_truncate.feature.md deleted file mode 100644 index a7b5367aca34a..0000000000000 --- a/changelog.d/23671_file_sink_truncate.feature.md +++ /dev/null @@ -1,3 +0,0 @@ -Added `truncate` options to `file` sink to truncate output files after some time. - -authors: esensar Quad9DNS diff --git a/changelog.d/23717_vector_test_color.feature.md b/changelog.d/23717_vector_test_color.feature.md deleted file mode 100644 index 5b694d9619407..0000000000000 --- a/changelog.d/23717_vector_test_color.feature.md +++ /dev/null @@ -1,3 +0,0 @@ -Disable ANSI color for `vector test` when running non-interactively. Honor `--color {auto|always|never}` and `VECTOR_COLOR`; VRL diagnostics no longer include ANSI sequences when color is disabled. - -authors: VanjaRo diff --git a/changelog.d/23748_add_indexer_ack_compression.feature.md b/changelog.d/23748_add_indexer_ack_compression.feature.md deleted file mode 100644 index 5b9068623a09a..0000000000000 --- a/changelog.d/23748_add_indexer_ack_compression.feature.md +++ /dev/null @@ -1,3 +0,0 @@ -Adds proper support for compression of HEC indexer ack queries, using the sink's configured `compression` setting. - -authors: sbalmos diff --git a/changelog.d/23815_memory_enrichment_expired_output.feature.md b/changelog.d/23815_memory_enrichment_expired_output.feature.md deleted file mode 100644 index 6bd44c05abceb..0000000000000 --- a/changelog.d/23815_memory_enrichment_expired_output.feature.md +++ /dev/null @@ -1,3 +0,0 @@ -Added `expired` output to the memory enrichment table source, to export items as they expire in the cache. - -authors: esensar Quad9DNS diff --git a/changelog.d/23820_sighup_reload_transforms.feature.md b/changelog.d/23820_sighup_reload_transforms.feature.md deleted file mode 100644 index 3efef1b42162d..0000000000000 --- a/changelog.d/23820_sighup_reload_transforms.feature.md +++ /dev/null @@ -1,3 +0,0 @@ -On receiving SIGHUP Vector now also reloads transform components with external VRL files. - -authors: nekorro diff --git a/changelog.d/23820_watch_config_handle_events.fix.md b/changelog.d/23820_watch_config_handle_events.fix.md deleted file mode 100644 index 24c92dbbf2277..0000000000000 --- a/changelog.d/23820_watch_config_handle_events.fix.md +++ /dev/null @@ -1,3 +0,0 @@ -The configuration watcher now collects event paths even during the delay period. These were previously ignored and prevented components from reloading. - -authors: nekorro diff --git a/changelog.d/23838_mqtt_sink_client_cert_auth.fix.md b/changelog.d/23838_mqtt_sink_client_cert_auth.fix.md deleted file mode 100644 index 4273f5deaf714..0000000000000 --- a/changelog.d/23838_mqtt_sink_client_cert_auth.fix.md +++ /dev/null @@ -1,3 +0,0 @@ -Enable unused TLS settings for perform client authentication by SSL certificate in `mqtt` sink. - -authors: ValentinChernovNTQ diff --git a/changelog.d/23863_memory_tables_tap.fix.md b/changelog.d/23863_memory_tables_tap.fix.md deleted file mode 100644 index e5a171a065cfc..0000000000000 --- a/changelog.d/23863_memory_tables_tap.fix.md +++ /dev/null @@ -1,3 +0,0 @@ -Memory enrichment table's outputs are now visible to the `vector tap` command. - -authors: esensar Quad9DNS diff --git a/changelog.d/23865_azure_blob_use_correct_feature.fix.md b/changelog.d/23865_azure_blob_use_correct_feature.fix.md deleted file mode 100644 index ec0b2ae7cdf85..0000000000000 --- a/changelog.d/23865_azure_blob_use_correct_feature.fix.md +++ /dev/null @@ -1,3 +0,0 @@ -Fixed a panic in the `azure_blob` sink by enabling a missing required crate feature. - -authors: thomasqueirozb diff --git a/changelog.d/23874_file_source_remove_legacy_checkpoint.breaking.md b/changelog.d/23874_file_source_remove_legacy_checkpoint.breaking.md deleted file mode 100644 index fabb3a8bf852d..0000000000000 --- a/changelog.d/23874_file_source_remove_legacy_checkpoint.breaking.md +++ /dev/null @@ -1,34 +0,0 @@ -* Dropped support for `file` source legacy checkpoints stored in the `checkpoints` folder (Vector `< 0.11`) which is located inside the `data_dir`. -* Removed the legacy checkpoint checksum format (Vector `< 0.15`). -* The intentionally hidden `fingerprint.bytes` option was also removed. - -### How to upgrade - -You can stop reading if you - -* have started using the `file` source on or after version `0.15`, or -* have cleared your `data_dir` on or after version `0.15`, or -* don't care about the file positions and don't care about current state of your checkpoints, meaning you accept that files could be read from the beginning again after the upgrade. - * Vector will re-read all files from the beginning if/when any `checkpoints.json` files nested inside `data_dir` fail to load due to legacy/corrupted data. - -You are only affected if your Vector version is: - -1. `>= 0.11` and `< 0.15`, then your checkpoints are using the legacy checkpoint checksum CRC format. -2. `>= 0.11` and `< 0.15`, then the `checksum` key is present under `checkpoints.fingerprint` in your `checkpoints.json` (instead of `first_lines_checksum`). -3. **or ever was** `< 0.11` and you are using the legacy `checkpoints` folder and/or the `unknown` key is present under `checkpoints.fingerprint` in any `checkpoints.json` files nested inside `data_dir`. - -#### If you are affected by `#1` or `#2` - -Run the `file` source with any version of Vector `>= 0.15`, but strictly before `0.51` and the checkpoints should be automatically updated. -For example, if you’re on Vector `0.10` and want to upgrade, keep upgrading Vector until `0.14` and Vector will automatically convert your checkpoints. -When upgrading, we recommend stepping through minor versions as these can each contain breaking changes while Vector is pre-1.0. These breaking changes are noted in their respective upgrade guides. - -Odds are the `file` source automatically converted checkpoints to the new format if you are using a recent version and you are not affected by this at all. - -#### If you are affected by `#3` - -You should manually delete the `unknown` checkpoint records from all `checkpoints.json` files nested inside `data_dir` -and then follow the upgrade guide for `#1` and `#2`. If you were using a recent version of Vector and `unknown` -was present it wasn't being used anyways. - -authors: thomasqueirozb diff --git a/changelog.d/23941_improve_journald_source_debug.feature.md b/changelog.d/23941_improve_journald_source_debug.feature.md deleted file mode 100644 index f996a21295c92..0000000000000 --- a/changelog.d/23941_improve_journald_source_debug.feature.md +++ /dev/null @@ -1,3 +0,0 @@ -The `journald` source now provides better error visibility by capturing and displaying stderr output from the underlying `journalctl` process as warning messages. - -authors: titaneric diff --git a/changelog.d/23986_datadog_agent_split_metric_namespace.enhancement.md b/changelog.d/23986_datadog_agent_split_metric_namespace.enhancement.md deleted file mode 100644 index dbd70462c4bc0..0000000000000 --- a/changelog.d/23986_datadog_agent_split_metric_namespace.enhancement.md +++ /dev/null @@ -1,4 +0,0 @@ -Added a new `split_metric_namespace` option to the `datadog_agent` source to -optionally disable the existing default metric name split behavior. - -authors: bruceg diff --git a/changelog.d/24014_memory_table_source_reload.fix.md b/changelog.d/24014_memory_table_source_reload.fix.md deleted file mode 100644 index 56b0403da230e..0000000000000 --- a/changelog.d/24014_memory_table_source_reload.fix.md +++ /dev/null @@ -1,3 +0,0 @@ -Fixed a crash on configuration reload when memory enrichment tables are configured to be used as a source. - -authors: esensar Quad9DNS diff --git a/changelog.d/24026_fix_docker_logs_socket_path.fix.md b/changelog.d/24026_fix_docker_logs_socket_path.fix.md deleted file mode 100644 index a6ef009511bd6..0000000000000 --- a/changelog.d/24026_fix_docker_logs_socket_path.fix.md +++ /dev/null @@ -1,3 +0,0 @@ -Fixed an issue in the `docker_logs` source where the `docker_host` option and `DOCKER_HOST` environment variable were ignored if they started with `unix://` or `npipe://`. In those cases the default location for the Docker socket was used - -authors: titaneric diff --git a/changelog.d/24073_utilization_bounds.fix.md b/changelog.d/24073_utilization_bounds.fix.md deleted file mode 100644 index 2d84b0cb109b4..0000000000000 --- a/changelog.d/24073_utilization_bounds.fix.md +++ /dev/null @@ -1,3 +0,0 @@ -Fixed an issue where utilization could report negative values. This could happen if messages from components were processed too late and were accounted for wrong utilization measurement period. These messages are now moved to the current utilization period, meaning there might be some inaccuracy in the resulting utilization metric, but it was never meant to be precise. - -authors: esensar Quad9DNS diff --git a/changelog.d/24074_clickhouse_arrow_complex_types.enhancement.md b/changelog.d/24074_clickhouse_arrow_complex_types.enhancement.md new file mode 100644 index 0000000000000..5a5d6c65ec212 --- /dev/null +++ b/changelog.d/24074_clickhouse_arrow_complex_types.enhancement.md @@ -0,0 +1,3 @@ +The `clickhouse` sink now supports complex data types (Array, Map, and Tuple) when using the `arrow_stream` format. + +authors: benjamin-awd diff --git a/changelog.d/24080_utilization_on_reload.fix.md b/changelog.d/24080_utilization_on_reload.fix.md deleted file mode 100644 index de09ebce3752f..0000000000000 --- a/changelog.d/24080_utilization_on_reload.fix.md +++ /dev/null @@ -1,3 +0,0 @@ -Fixed a bug where utilization metric could be lost for changed components on configuration reload. - -authors: esensar Quad9DNS diff --git a/changelog.d/24393_log_to_metric_histogram_summary.fix.md b/changelog.d/24393_log_to_metric_histogram_summary.fix.md new file mode 100644 index 0000000000000..1dbe834ec9a70 --- /dev/null +++ b/changelog.d/24393_log_to_metric_histogram_summary.fix.md @@ -0,0 +1,3 @@ +The `log_to_metric` transform now correctly handles aggregated histogram and aggregated summary metrics. + +authors: jblazquez diff --git a/changelog.d/add_metric_vector_config_reload_rejected.feature.md b/changelog.d/add_metric_vector_config_reload_rejected.feature.md deleted file mode 100644 index 2d2d74e5113ef..0000000000000 --- a/changelog.d/add_metric_vector_config_reload_rejected.feature.md +++ /dev/null @@ -1,3 +0,0 @@ -Vector now emits `config_reload_rejected` and `config_reloaded` counters. - -authors: suikammd diff --git a/changelog.d/aws_s3_source_exponential_backoff.enhancement.md b/changelog.d/aws_s3_source_exponential_backoff.enhancement.md deleted file mode 100644 index fa2b139dde6ba..0000000000000 --- a/changelog.d/aws_s3_source_exponential_backoff.enhancement.md +++ /dev/null @@ -1,5 +0,0 @@ -The `aws_s3` source now uses exponential backoff when retrying failed SQS `receive_message` operations. Previously, the source used a fixed 500ms delay between retries. - -The new behavior starts at 500ms and doubles with each consecutive failure, capping at 30 seconds. This prevents excessive API calls during prolonged AWS SQS outages, invalid IAM permissions, or throttling scenarios, while still being responsive when the service recovers. - -authors: medzin pront diff --git a/changelog.d/env_var_multiline_rejection.breaking.md b/changelog.d/env_var_multiline_rejection.breaking.md deleted file mode 100644 index 8c6ae4608178e..0000000000000 --- a/changelog.d/env_var_multiline_rejection.breaking.md +++ /dev/null @@ -1,5 +0,0 @@ -Environment variable interpolation in configuration files now rejects values containing newline characters. This prevents configuration -injection attacks where environment variables could inject malicious multi-line configurations. If you need to inject multi-line -configuration blocks, use a config pre-processing tool like `envsubst` instead. - -authors: pront diff --git a/changelog.d/fix_fluent_received_events_count.fix.md b/changelog.d/fix_fluent_received_events_count.fix.md deleted file mode 100644 index 37d830be6982f..0000000000000 --- a/changelog.d/fix_fluent_received_events_count.fix.md +++ /dev/null @@ -1,3 +0,0 @@ -Fixed duplicate reporting of received event count in the `fluent` source. - -authors: gwenaskell diff --git a/changelog.d/incremental_to_absolute_size_tracking.fix.md b/changelog.d/incremental_to_absolute_size_tracking.fix.md new file mode 100644 index 0000000000000..f5562ec4a721f --- /dev/null +++ b/changelog.d/incremental_to_absolute_size_tracking.fix.md @@ -0,0 +1,4 @@ +1. Fix memory tracking on MetricSet in incremental_to_absolute transform by accurately calculating metric sizes. Previously, all sizes were being calculated as 0, resulting in no actual tracking. +2. Add metrics for incremental_to_absolute to track events, internally tracked cache size, and evictions. + +authors: GreyLilac09 diff --git a/changelog.d/internal_log_component_id_field.breaking.md b/changelog.d/internal_log_component_id_field.breaking.md deleted file mode 100644 index 5237aa144c86c..0000000000000 --- a/changelog.d/internal_log_component_id_field.breaking.md +++ /dev/null @@ -1,4 +0,0 @@ -Vector's internal topology logs now use the `component_id` field name instead of `component` or `key`. -If you are monitoring or filtering Vector's internal logs based on these field names, update your queries to use `component_id`. - -authors: pront diff --git a/changelog.d/opentelemetry_source_http_decompression.fix.md b/changelog.d/opentelemetry_source_http_decompression.fix.md deleted file mode 100644 index ebf4bbd922e96..0000000000000 --- a/changelog.d/opentelemetry_source_http_decompression.fix.md +++ /dev/null @@ -1,4 +0,0 @@ -Fixed a `opentelemetry` source bug where HTTP payloads were not decompressed according to the request headers. -This only applied when `use_otlp_decoding` (recently added) was set to `true`. - -authors: pront diff --git a/changelog.d/optimize-websocket-source.enhancement.md b/changelog.d/optimize-websocket-source.enhancement.md new file mode 100644 index 0000000000000..9745ed1ebc329 --- /dev/null +++ b/changelog.d/optimize-websocket-source.enhancement.md @@ -0,0 +1,3 @@ +Small optimization to the `websocket` source performance by avoiding getting a new time for every event in an array. + +authors: bruceg diff --git a/changelog.d/otlp_decoding.feature.md b/changelog.d/otlp_decoding.feature.md deleted file mode 100644 index c13f55dee313d..0000000000000 --- a/changelog.d/otlp_decoding.feature.md +++ /dev/null @@ -1,3 +0,0 @@ -Added `otlp` codec for decoding OTLP format to Vector events, complementing the existing OTLP encoder. - -authors: pront diff --git a/changelog.d/otlp_encoding.feature.md b/changelog.d/otlp_encoding.feature.md deleted file mode 100644 index 513bcf4379d93..0000000000000 --- a/changelog.d/otlp_encoding.feature.md +++ /dev/null @@ -1,4 +0,0 @@ -Added `otlp` codec for encoding Vector events to OTLP format. -The codec can be used with sinks that support encoding configuration. - -authors: pront diff --git a/changelog.d/prometheus_remote_write_metadata_conflicts.fix.md b/changelog.d/prometheus_remote_write_metadata_conflicts.fix.md deleted file mode 100644 index eeb457c61028d..0000000000000 --- a/changelog.d/prometheus_remote_write_metadata_conflicts.fix.md +++ /dev/null @@ -1,3 +0,0 @@ -The `prometheus_remote_write` source now has a `metadata_conflict_strategy` option so you can determine how to handle conflicting metric metadata. By default, the source continues to reject requests with conflicting metadata (HTTP 400 error) to maintain backwards compatibility. Set `metadata_conflict_strategy` to `ignore` to align with Prometheus/Thanos behavior, which silently ignores metadata conflicts. - -authors: elohmeier diff --git a/changelog.d/prometheus_remote_write_path.feature.md b/changelog.d/prometheus_remote_write_path.feature.md deleted file mode 100644 index 0d8c2522e5b3c..0000000000000 --- a/changelog.d/prometheus_remote_write_path.feature.md +++ /dev/null @@ -1,3 +0,0 @@ -Added `path` configuration option to `prometheus_remote_write` source to allow accepting metrics on custom URL paths instead of only the root path. This enables configuration of endpoints like `/api/v1/write` to match standard Prometheus remote write conventions. - -authors: elohmeier diff --git a/changelog.d/protobuf_use_json_names.enhancement.md b/changelog.d/protobuf_use_json_names.enhancement.md deleted file mode 100644 index 2e3a3d30e2b88..0000000000000 --- a/changelog.d/protobuf_use_json_names.enhancement.md +++ /dev/null @@ -1,5 +0,0 @@ -Added `use_json_names` option to protobuf encoding and decoding. -When enabled, the codec uses JSON field names (camelCase) instead of protobuf field names (snake_case). -This is useful when working with data that uses JSON naming conventions. - -authors: pront diff --git a/changelog.d/utilization.breaking.md b/changelog.d/utilization.breaking.md deleted file mode 100644 index e07082da222e1..0000000000000 --- a/changelog.d/utilization.breaking.md +++ /dev/null @@ -1,3 +0,0 @@ -The `utilization` metric is now capped at 4 decimal digit precision. - -authors: pront diff --git a/changelog.d/x86_64_apple_builds.breaking.md b/changelog.d/x86_64_apple_builds.breaking.md deleted file mode 100644 index 275df179c0974..0000000000000 --- a/changelog.d/x86_64_apple_builds.breaking.md +++ /dev/null @@ -1,4 +0,0 @@ -Following [this announcement](https://blog.rust-lang.org/2025/09/18/Rust-1.90.0/#demoting-x86-64-apple-darwin-to-tier-2-with-host-tools), we will no longer publish `x86_64-apple-darwin` builds. -This means we will not be validating if Vector builds and works correctly on that platform. - -authors: pront diff --git a/clippy.toml b/clippy.toml index 20274f646b1d7..cbd8c640fdb71 100644 --- a/clippy.toml +++ b/clippy.toml @@ -1,9 +1,11 @@ large-error-threshold = 256 # in bytes +allow-unwrap-in-tests = true # for `disallowed_method`: # https://rust-lang.github.io/rust-clippy/master/index.html#disallowed_method disallowed-methods = [ { path = "std::io::Write::write", reason = "This doesn't handle short writes, use `write_all` instead." }, + { path = "vrl::stdlib::all", reason = "Use `vector_vrl_functions::all()` instead for consistency across all Vector VRL functions." }, ] disallowed-types = [ diff --git a/cue.mod/module.cue b/cue.mod/module.cue deleted file mode 100644 index caf275bea6bcf..0000000000000 --- a/cue.mod/module.cue +++ /dev/null @@ -1,10 +0,0 @@ -// This file establishes the Vector repo as a CUE module that can be imported by -// other CUE libraries. This is here largely so that the CUE team can use the -// the Vector docs as an integration test case. See -// https://github.com/vectordotdev/vector/pull/6593. This currently has no effect -// on the Vector docs build. - -module: "vector.dev" -language: { - version: "v0.9.0" -} diff --git a/deny.toml b/deny.toml index 7e200969f6e01..32a8862fd30ec 100644 --- a/deny.toml +++ b/deny.toml @@ -6,6 +6,7 @@ allow = [ "BSD-3-Clause", "BSL-1.0", "CC0-1.0", + "CDLA-Permissive-2.0", "ISC", "MIT", "MIT-0", @@ -22,6 +23,7 @@ exceptions = [ # compliance we cannot be modifying the source files. { allow = ["MPL-2.0"], name = "colored", version = "*" }, { allow = ["MPL-2.0"], name = "webpki-roots", version = "*" }, + { allow = ["MPL-2.0"], name = "vector-common-macros", version = "*" }, { allow = ["MPL-2.0"], name = "vector-config-common", version = "*" }, { allow = ["MPL-2.0"], name = "vector-config-macros", version = "*" }, { allow = ["MPL-2.0"], name = "vrl", version = "*" }, @@ -41,24 +43,13 @@ ignore = [ # There is not fix available yet. # https://github.com/vectordotdev/vector/issues/19262 "RUSTSEC-2023-0071", - # Vulnerability in `tonic` crate: https://rustsec.org/advisories/RUSTSEC-2024-0376 - # There is a fixed version (v0.12.3) but we are blocked from upgrading to `http` v1, which - # `tonic` v0.12 depends on. See https://github.com/vectordotdev/vector/issues/19179 - "RUSTSEC-2024-0376", - # Advisory in rustls crate: https://rustsec.org/advisories/RUSTSEC-2024-0336 If a `close_notify` - # alert is received during a handshake, `complete_io` does not terminate. - # Vulnerable version only used in dev-dependencies - "RUSTSEC-2024-0336", - # idna accepts Punycode labels that do not produce any non-ASCII when decoded - # Need to update some direct dependencies before we can upgrade idna to fix - "RUSTSEC-2024-0421", { id = "RUSTSEC-2021-0139", reason = " ansi_term is unmaintained" }, { id = "RUSTSEC-2024-0388", reason = "derivative is unmaintained" }, { id = "RUSTSEC-2024-0384", reason = "instant is unmaintained" }, { id = "RUSTSEC-2020-0168", reason = "mach is unmaintained" }, - { id = "RUSTSEC-2024-0370", reason = "proc-macro-error is unmaintained" }, - { id = "RUSTSEC-2024-0320", reason = "yaml-rust is unmaintained" }, { id = "RUSTSEC-2024-0436", reason = "paste is unmaintained" }, { id = "RUSTSEC-2025-0012", reason = "backoff is unmaintained" }, - { id = "RUSTSEC-2025-0014", reason = "humantime is unmaintained" }, + # rustls-pemfile is unmaintained. Blocked by both async-nats and http 1.0.0 upgrade. + { id = "RUSTSEC-2025-0134", reason = "rustls-pemfile is unmaintained" }, + { id = "RUSTSEC-2026-0002", reason = "latest aws-sdk-s3 (v1.119.0) is still using lru v0.12.5" }, ] diff --git a/distribution/docker/alpine/Dockerfile b/distribution/docker/alpine/Dockerfile index 0358bd9bdc882..9d40daff5ce8e 100644 --- a/distribution/docker/alpine/Dockerfile +++ b/distribution/docker/alpine/Dockerfile @@ -1,4 +1,4 @@ -FROM docker.io/alpine:3.22 AS builder +FROM docker.io/alpine:3.23 AS builder WORKDIR /vector @@ -12,7 +12,7 @@ RUN ARCH=$(if [ "$TARGETPLATFORM" = "linux/arm/v6" ]; then echo "arm"; else cat RUN mkdir -p /var/lib/vector -FROM docker.io/alpine:3.22 +FROM docker.io/alpine:3.23 # https://github.com/opencontainers/image-spec/blob/main/annotations.md LABEL org.opencontainers.image.url="https://vector.dev" diff --git a/distribution/docker/distroless-static/Dockerfile b/distribution/docker/distroless-static/Dockerfile index c369cff031f94..d692f48321614 100644 --- a/distribution/docker/distroless-static/Dockerfile +++ b/distribution/docker/distroless-static/Dockerfile @@ -1,4 +1,4 @@ -FROM docker.io/alpine:3.22 AS builder +FROM docker.io/alpine:3.23 AS builder WORKDIR /vector diff --git a/distribution/install.sh b/distribution/install.sh index b400b41d26862..39591daba2945 100755 --- a/distribution/install.sh +++ b/distribution/install.sh @@ -13,7 +13,7 @@ set -u # If PACKAGE_ROOT is unset or empty, default it. PACKAGE_ROOT="${PACKAGE_ROOT:-"https://packages.timber.io/vector"}" # If VECTOR_VERSION is unset or empty, default it. -VECTOR_VERSION="${VECTOR_VERSION:-"0.50.0"}" +VECTOR_VERSION="${VECTOR_VERSION:-"0.53.0"}" _divider="--------------------------------------------------------------------------------" _prompt=">>>" _indent=" " diff --git a/distribution/kubernetes/vector-agent/README.md b/distribution/kubernetes/vector-agent/README.md index f1e50a64b0a42..a3724e12329c9 100644 --- a/distribution/kubernetes/vector-agent/README.md +++ b/distribution/kubernetes/vector-agent/README.md @@ -1,6 +1,6 @@ The kubernetes manifests found in this directory have been automatically generated from the [helm chart `vector/vector`](https://github.com/vectordotdev/helm-charts/tree/master/charts/vector) -version 0.46.0 with the following `values.yaml`: +version 0.50.0 with the following `values.yaml`: ```yaml role: Agent diff --git a/distribution/kubernetes/vector-agent/configmap.yaml b/distribution/kubernetes/vector-agent/configmap.yaml index 88a30526cac48..822f9a2f46b09 100644 --- a/distribution/kubernetes/vector-agent/configmap.yaml +++ b/distribution/kubernetes/vector-agent/configmap.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Agent - app.kubernetes.io/version: "0.50.0-distroless-libc" + app.kubernetes.io/version: "0.53.0-distroless-libc" data: agent.yaml: | data_dir: /vector-data-dir diff --git a/distribution/kubernetes/vector-agent/daemonset.yaml b/distribution/kubernetes/vector-agent/daemonset.yaml index 6e6d3059aa173..4269320475e5e 100644 --- a/distribution/kubernetes/vector-agent/daemonset.yaml +++ b/distribution/kubernetes/vector-agent/daemonset.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Agent - app.kubernetes.io/version: "0.50.0-distroless-libc" + app.kubernetes.io/version: "0.53.0-distroless-libc" spec: selector: matchLabels: @@ -30,7 +30,7 @@ spec: dnsPolicy: ClusterFirst containers: - name: vector - image: "timberio/vector:0.50.0-distroless-libc" + image: "timberio/vector:0.53.0-distroless-libc" imagePullPolicy: IfNotPresent args: - --config-dir diff --git a/distribution/kubernetes/vector-agent/rbac.yaml b/distribution/kubernetes/vector-agent/rbac.yaml index 61e2d56f8778e..c2a122a73d05b 100644 --- a/distribution/kubernetes/vector-agent/rbac.yaml +++ b/distribution/kubernetes/vector-agent/rbac.yaml @@ -10,7 +10,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Agent - app.kubernetes.io/version: "0.50.0-distroless-libc" + app.kubernetes.io/version: "0.53.0-distroless-libc" rules: - apiGroups: - "" @@ -31,7 +31,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Agent - app.kubernetes.io/version: "0.50.0-distroless-libc" + app.kubernetes.io/version: "0.53.0-distroless-libc" roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole diff --git a/distribution/kubernetes/vector-agent/service-headless.yaml b/distribution/kubernetes/vector-agent/service-headless.yaml index f7353c743e6a2..50ffc5628ee46 100644 --- a/distribution/kubernetes/vector-agent/service-headless.yaml +++ b/distribution/kubernetes/vector-agent/service-headless.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Agent - app.kubernetes.io/version: "0.50.0-distroless-libc" + app.kubernetes.io/version: "0.53.0-distroless-libc" annotations: spec: clusterIP: None diff --git a/distribution/kubernetes/vector-agent/serviceaccount.yaml b/distribution/kubernetes/vector-agent/serviceaccount.yaml index eb95e58c49229..2814a485ca4e2 100644 --- a/distribution/kubernetes/vector-agent/serviceaccount.yaml +++ b/distribution/kubernetes/vector-agent/serviceaccount.yaml @@ -9,5 +9,5 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Agent - app.kubernetes.io/version: "0.50.0-distroless-libc" + app.kubernetes.io/version: "0.53.0-distroless-libc" automountServiceAccountToken: true diff --git a/distribution/kubernetes/vector-aggregator/README.md b/distribution/kubernetes/vector-aggregator/README.md index 80dc74083d340..505fc2138c0df 100644 --- a/distribution/kubernetes/vector-aggregator/README.md +++ b/distribution/kubernetes/vector-aggregator/README.md @@ -1,6 +1,6 @@ The kubernetes manifests found in this directory have been automatically generated from the [helm chart `vector/vector`](https://github.com/vectordotdev/helm-charts/tree/master/charts/vector) -version 0.46.0 with the following `values.yaml`: +version 0.50.0 with the following `values.yaml`: ```yaml diff --git a/distribution/kubernetes/vector-aggregator/configmap.yaml b/distribution/kubernetes/vector-aggregator/configmap.yaml index 84f37d7728ea5..a6a1f82010b58 100644 --- a/distribution/kubernetes/vector-aggregator/configmap.yaml +++ b/distribution/kubernetes/vector-aggregator/configmap.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Aggregator - app.kubernetes.io/version: "0.50.0-distroless-libc" + app.kubernetes.io/version: "0.53.0-distroless-libc" data: aggregator.yaml: | data_dir: /vector-data-dir diff --git a/distribution/kubernetes/vector-aggregator/service-headless.yaml b/distribution/kubernetes/vector-aggregator/service-headless.yaml index 25e1c5039a6e7..ecc8494e0256a 100644 --- a/distribution/kubernetes/vector-aggregator/service-headless.yaml +++ b/distribution/kubernetes/vector-aggregator/service-headless.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Aggregator - app.kubernetes.io/version: "0.50.0-distroless-libc" + app.kubernetes.io/version: "0.53.0-distroless-libc" annotations: spec: clusterIP: None diff --git a/distribution/kubernetes/vector-aggregator/service.yaml b/distribution/kubernetes/vector-aggregator/service.yaml index b5d84bf45c413..6a1438db5d1b8 100644 --- a/distribution/kubernetes/vector-aggregator/service.yaml +++ b/distribution/kubernetes/vector-aggregator/service.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Aggregator - app.kubernetes.io/version: "0.50.0-distroless-libc" + app.kubernetes.io/version: "0.53.0-distroless-libc" annotations: spec: ports: diff --git a/distribution/kubernetes/vector-aggregator/serviceaccount.yaml b/distribution/kubernetes/vector-aggregator/serviceaccount.yaml index ba114d904dc1f..4ddd4c0774715 100644 --- a/distribution/kubernetes/vector-aggregator/serviceaccount.yaml +++ b/distribution/kubernetes/vector-aggregator/serviceaccount.yaml @@ -9,5 +9,5 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Aggregator - app.kubernetes.io/version: "0.50.0-distroless-libc" + app.kubernetes.io/version: "0.53.0-distroless-libc" automountServiceAccountToken: true diff --git a/distribution/kubernetes/vector-aggregator/statefulset.yaml b/distribution/kubernetes/vector-aggregator/statefulset.yaml index b5a9c2944fe53..f163a10a8d41c 100644 --- a/distribution/kubernetes/vector-aggregator/statefulset.yaml +++ b/distribution/kubernetes/vector-aggregator/statefulset.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Aggregator - app.kubernetes.io/version: "0.50.0-distroless-libc" + app.kubernetes.io/version: "0.53.0-distroless-libc" annotations: {} spec: replicas: 1 @@ -34,7 +34,7 @@ spec: dnsPolicy: ClusterFirst containers: - name: vector - image: "timberio/vector:0.50.0-distroless-libc" + image: "timberio/vector:0.53.0-distroless-libc" imagePullPolicy: IfNotPresent args: - --config-dir diff --git a/distribution/kubernetes/vector-stateless-aggregator/README.md b/distribution/kubernetes/vector-stateless-aggregator/README.md index 735fa8cc8333d..3eaea97af7e20 100644 --- a/distribution/kubernetes/vector-stateless-aggregator/README.md +++ b/distribution/kubernetes/vector-stateless-aggregator/README.md @@ -1,6 +1,6 @@ The kubernetes manifests found in this directory have been automatically generated from the [helm chart `vector/vector`](https://github.com/vectordotdev/helm-charts/tree/master/charts/vector) -version 0.46.0 with the following `values.yaml`: +version 0.50.0 with the following `values.yaml`: ```yaml role: Stateless-Aggregator diff --git a/distribution/kubernetes/vector-stateless-aggregator/configmap.yaml b/distribution/kubernetes/vector-stateless-aggregator/configmap.yaml index d9246ba371dcb..bce12915342a8 100644 --- a/distribution/kubernetes/vector-stateless-aggregator/configmap.yaml +++ b/distribution/kubernetes/vector-stateless-aggregator/configmap.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Stateless-Aggregator - app.kubernetes.io/version: "0.50.0-distroless-libc" + app.kubernetes.io/version: "0.53.0-distroless-libc" data: aggregator.yaml: | data_dir: /vector-data-dir diff --git a/distribution/kubernetes/vector-stateless-aggregator/deployment.yaml b/distribution/kubernetes/vector-stateless-aggregator/deployment.yaml index ad9d539987417..7385a4b8e2151 100644 --- a/distribution/kubernetes/vector-stateless-aggregator/deployment.yaml +++ b/distribution/kubernetes/vector-stateless-aggregator/deployment.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Stateless-Aggregator - app.kubernetes.io/version: "0.50.0-distroless-libc" + app.kubernetes.io/version: "0.53.0-distroless-libc" annotations: {} spec: replicas: 1 @@ -32,7 +32,7 @@ spec: dnsPolicy: ClusterFirst containers: - name: vector - image: "timberio/vector:0.50.0-distroless-libc" + image: "timberio/vector:0.53.0-distroless-libc" imagePullPolicy: IfNotPresent args: - --config-dir diff --git a/distribution/kubernetes/vector-stateless-aggregator/service-headless.yaml b/distribution/kubernetes/vector-stateless-aggregator/service-headless.yaml index cb759eac742d4..ecea5e6e62213 100644 --- a/distribution/kubernetes/vector-stateless-aggregator/service-headless.yaml +++ b/distribution/kubernetes/vector-stateless-aggregator/service-headless.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Stateless-Aggregator - app.kubernetes.io/version: "0.50.0-distroless-libc" + app.kubernetes.io/version: "0.53.0-distroless-libc" annotations: spec: clusterIP: None diff --git a/distribution/kubernetes/vector-stateless-aggregator/service.yaml b/distribution/kubernetes/vector-stateless-aggregator/service.yaml index 49fb889b8a10d..bc394012f1681 100644 --- a/distribution/kubernetes/vector-stateless-aggregator/service.yaml +++ b/distribution/kubernetes/vector-stateless-aggregator/service.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Stateless-Aggregator - app.kubernetes.io/version: "0.50.0-distroless-libc" + app.kubernetes.io/version: "0.53.0-distroless-libc" annotations: spec: ports: diff --git a/distribution/kubernetes/vector-stateless-aggregator/serviceaccount.yaml b/distribution/kubernetes/vector-stateless-aggregator/serviceaccount.yaml index 10aaa571c1b9c..7205cb183ed13 100644 --- a/distribution/kubernetes/vector-stateless-aggregator/serviceaccount.yaml +++ b/distribution/kubernetes/vector-stateless-aggregator/serviceaccount.yaml @@ -9,5 +9,5 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Stateless-Aggregator - app.kubernetes.io/version: "0.50.0-distroless-libc" + app.kubernetes.io/version: "0.53.0-distroless-libc" automountServiceAccountToken: true diff --git a/docs/DEPRECATION.md b/docs/DEPRECATION.md index 3b4a5d8c576c5..23ef97ec2c475 100644 --- a/docs/DEPRECATION.md +++ b/docs/DEPRECATION.md @@ -68,8 +68,8 @@ When possible, Vector will error at start-up when a removed configuration option When introducing a deprecation into Vector, the pull request introducing the deprecation should: -- Add a note to the Deprecations section of the upgrade guide for the next release with a description and - directions for transitioning if applicable. +- Add a note to the Deprecations section of the upgrade guide in `website/content/en/highlights` for + the next release with a description and directions for transitioning if applicable. - Copy the same note from the previous step, to a changelog fragment, with type="deprecation". See the changelog fragment [README.md](../changelog.d/README.md) for details. - Add a deprecation note to the docs. Typically, this means adding `deprecation: "description of the deprecation"` @@ -80,7 +80,7 @@ When introducing a deprecation into Vector, the pull request introducing the dep the new name will be appended with the text `(formerly OldName)`. - Add a log message to Vector that is logged at the `WARN` level starting with the word `DEPRECATION` if Vector detects the deprecated configuration or feature being used (when possible). -- Add the deprecation to [DEPRECATIONS.md](DEPRECATIONS.md) to track migration (if applicable) and removal +- Add the deprecation to [docs/DEPRECATIONS.md](../docs/DEPRECATIONS.md) to track migration (if applicable) and removal When removing a deprecation in a subsequent release, the pull request should: @@ -90,4 +90,4 @@ When removing a deprecation in a subsequent release, the pull request should: for transitioning if applicable. - Copy the same note from the previous step, to a changelog fragment, with type="breaking". See the changelog fragment [README.md](../changelog.d/README.md) for details. -- Remove the deprecation from [DEPRECATIONS.md](DEPRECATIONS.md) +- Remove the deprecation from [docs/DEPRECATIONS.md](../docs/DEPRECATIONS.md) diff --git a/docs/DEPRECATIONS.md b/docs/DEPRECATIONS.md index 7c9042da1a70a..e43d70d493d3d 100644 --- a/docs/DEPRECATIONS.md +++ b/docs/DEPRECATIONS.md @@ -15,6 +15,7 @@ For example: ## To be deprecated - `v0.50.0` | `http-server-encoding` | The `encoding` field will be removed. Use `decoding` and `framing` instead. +- `v0.53.0` | `buffer-bytes-events-metrics` | The `buffer_byte_size` and `buffer_events` gauges are deprecated in favor of the `buffer_size_bytes`/`buffer_size_events` metrics described in `docs/specs/buffer.md`. ## To be migrated diff --git a/docs/specs/buffer.md b/docs/specs/buffer.md index 152f14ad56215..05d343a4bc0bc 100644 --- a/docs/specs/buffer.md +++ b/docs/specs/buffer.md @@ -42,8 +42,8 @@ _All buffers_ MUST emit a `BufferCreated` event upon creation. To avoid stale me - `max_size_bytes` - the max size of the buffer in bytes if relevant - `max_size_events` - the max size of the buffer in number of events if relevant - Metric - - MUST emit the `buffer_max_event_size` gauge (in-memory buffers) if the defined `max_size_events` value is present - - MUST emit the `buffer_max_byte_size` gauge (disk buffers) if the defined `max_size_bytes` value is present + - MUST emit the `buffer_max_size_events` gauge (in-memory buffers) if the defined `max_size_events` value is present, and emit `buffer_max_event_size` for backward compatibility + - MUST emit the `buffer_max_size_bytes` gauge (disk buffers) if the defined `max_size_bytes` value is present, and emit `buffer_max_byte_size` for backward compatibility #### BufferEventsReceived @@ -58,8 +58,8 @@ _All buffers_ MUST emit a `BufferEventsReceived` event: - Metric - MUST increment the `buffer_received_events_total` counter by the defined `count` - MUST increment the `buffer_received_bytes_total` counter by the defined `byte_size` - - MUST increment the `buffer_events` gauge by the defined `count` - - MUST increment the `buffer_byte_size` gauge by the defined `byte_size` + - MUST increment the `buffer_size_events` gauge by the defined `count`, and emit `buffer_events` for backward compatibility + - MUST increment the `buffer_size_bytes` gauge by the defined `byte_size`, and emit `buffer_byte_size` for backward compatibility #### BufferEventsSent @@ -71,8 +71,8 @@ _All buffers_ MUST emit a `BufferEventsSent` event after sending one or more Vec - Metric - MUST increment the `buffer_sent_events_total` counter by the defined `count` - MUST increment the `buffer_sent_bytes_total` counter by the defined `byte_size` - - MUST decrement the `buffer_events` gauge by the defined `count` - - MUST decrement the `buffer_byte_size` gauge by the defined `byte_size` + - MUST decrement the `buffer_size_events` gauge by the defined `count`, and emit `buffer_events` for backward compatibility + - MUST decrement the `buffer_size_bytes` gauge by the defined `byte_size`, and emit `buffer_byte_size` for backward compatibility #### BufferError diff --git a/lib/codecs/Cargo.toml b/lib/codecs/Cargo.toml index 8e4ce532c54b6..8b3cbac1b2f28 100644 --- a/lib/codecs/Cargo.toml +++ b/lib/codecs/Cargo.toml @@ -5,21 +5,30 @@ authors = ["Vector Contributors "] edition = "2024" publish = false +[lints.clippy] +unwrap-used = "deny" + [[bin]] name = "generate-avro-fixtures" path = "tests/bin/generate-avro-fixtures.rs" [dependencies] -apache-avro = { version = "0.16.0", default-features = false } +apache-avro = { version = "0.20.0", default-features = false } +arrow = { version = "56.2.0", default-features = false, features = ["ipc"], optional = true } +async-trait.workspace = true bytes.workspace = true chrono.workspace = true +rust_decimal.workspace = true csv-core = { version = "0.1.12", default-features = false } derivative.workspace = true +derive_more = { version = "2.0.1", optional = true, features = ["from", "display"] } dyn-clone = { version = "1", default-features = false } flate2.workspace = true +futures.workspace = true influxdb-line-protocol = { version = "2", default-features = false } lookup = { package = "vector-lookup", path = "../vector-lookup", default-features = false, features = ["test"] } memchr = { version = "2", default-features = false } +metrics.workspace = true opentelemetry-proto = { path = "../opentelemetry-proto", optional = true } ordered-float.workspace = true prost.workspace = true @@ -29,17 +38,23 @@ regex.workspace = true serde.workspace = true serde_with = { version = "3.14.0", default-features = false, features = ["std", "macros", "chrono_0_4"] } serde_json.workspace = true +serde_arrow = { version = "0.13", features = ["arrow-56"], optional = true } +serde-aux = { version = "4.5", optional = true } smallvec = { version = "1", default-features = false, features = ["union"] } snafu.workspace = true +strum = { version = "0.26.3", features = ["derive"], optional = true } syslog_loose = { version = "0.23", default-features = false, optional = true } tokio-util = { version = "0.7", default-features = false, features = ["codec"] } tokio = { workspace = true, features = ["full"] } tracing.workspace = true vrl.workspace = true vector-common = { path = "../vector-common", default-features = false } +vector-common-macros.workspace = true vector-config = { path = "../vector-config", default-features = false } vector-config-macros = { path = "../vector-config-macros", default-features = false } vector-core = { path = "../vector-core", default-features = false, features = ["vrl"] } +vector-vrl-functions.workspace = true +toml = { version = "0.9.8", optional = true } [dev-dependencies] futures.workspace = true @@ -53,5 +68,7 @@ uuid.workspace = true vrl.workspace = true [features] -syslog = ["dep:syslog_loose"] +arrow = ["dep:arrow", "dep:serde_arrow"] opentelemetry = ["dep:opentelemetry-proto"] +syslog = ["dep:syslog_loose", "dep:strum", "dep:derive_more", "dep:serde-aux", "dep:toml"] +test = [] diff --git a/src/codecs/decoding/config.rs b/lib/codecs/src/decoding/config.rs similarity index 87% rename from src/codecs/decoding/config.rs rename to lib/codecs/src/decoding/config.rs index 2670b76c977dc..6ed15f8243867 100644 --- a/src/codecs/decoding/config.rs +++ b/lib/codecs/src/decoding/config.rs @@ -1,10 +1,7 @@ use serde::{Deserialize, Serialize}; -use vector_lib::{ - codecs::decoding::{DeserializerConfig, FramingConfig}, - config::LogNamespace, -}; +use vector_core::config::LogNamespace; -use crate::codecs::Decoder; +use crate::decoding::{Decoder, DeserializerConfig, FramingConfig}; /// Config used to build a `Decoder`. #[derive(Debug, Clone, Deserialize, Serialize)] @@ -43,7 +40,7 @@ impl DecodingConfig { } /// Builds a `Decoder` from the provided configuration. - pub fn build(&self) -> vector_lib::Result { + pub fn build(&self) -> vector_common::Result { // Build the framer. let framer = self.framing.build(); diff --git a/src/codecs/decoding/decoder.rs b/lib/codecs/src/decoding/decoder.rs similarity index 89% rename from src/codecs/decoding/decoder.rs rename to lib/codecs/src/decoding/decoder.rs index 499dedb10e9e6..0796e61ff82f3 100644 --- a/src/codecs/decoding/decoder.rs +++ b/lib/codecs/src/decoding/decoder.rs @@ -1,18 +1,18 @@ use bytes::{Bytes, BytesMut}; use smallvec::SmallVec; -use vector_lib::{ - codecs::decoding::{ - BoxedFramingError, BytesDeserializer, Deserializer, Error, Framer, NewlineDelimitedDecoder, - format::Deserializer as _, - }, - config::LogNamespace, -}; +use vector_common::internal_event::emit; +use vector_core::{config::LogNamespace, event::Event}; use crate::{ - event::Event, + decoding::format::Deserializer as _, + decoding::{ + BoxedFramingError, BytesDeserializer, Deserializer, Error, Framer, NewlineDelimitedDecoder, + }, internal_events::{DecoderDeserializeError, DecoderFramingError}, }; +type DecodedFrame = (SmallVec<[Event; 1]>, usize); + /// A decoder that can decode structured events from a byte stream / byte /// messages. #[derive(Clone)] @@ -60,9 +60,9 @@ impl Decoder { fn handle_framing_result( &mut self, frame: Result, BoxedFramingError>, - ) -> Result, usize)>, Error> { + ) -> Result, Error> { let frame = frame.map_err(|error| { - emit!(DecoderFramingError { error: &error }); + emit(DecoderFramingError { error: &error }); Error::FramingError(error) })?; @@ -72,7 +72,7 @@ impl Decoder { } /// Parses a frame using the included deserializer, and handles any errors by logging. - pub fn deserializer_parse(&self, frame: Bytes) -> Result<(SmallVec<[Event; 1]>, usize), Error> { + pub fn deserializer_parse(&self, frame: Bytes) -> Result { let byte_size = frame.len(); // Parse structured events from the byte frame. @@ -80,14 +80,14 @@ impl Decoder { .parse(frame, self.log_namespace) .map(|events| (events, byte_size)) .map_err(|error| { - emit!(DecoderDeserializeError { error: &error }); + emit(DecoderDeserializeError { error: &error }); Error::ParsingError(error) }) } } impl tokio_util::codec::Decoder for Decoder { - type Item = (SmallVec<[Event; 1]>, usize); + type Item = DecodedFrame; type Error = Error; fn decode(&mut self, buf: &mut BytesMut) -> Result, Self::Error> { @@ -106,13 +106,13 @@ mod tests { use bytes::Bytes; use futures::{StreamExt, stream}; use tokio_util::{codec::FramedRead, io::StreamReader}; - use vector_lib::codecs::{ - JsonDeserializer, NewlineDelimitedDecoder, StreamDecodingError, - decoding::{Deserializer, Framer}, - }; use vrl::value::Value; use super::Decoder; + use crate::{ + JsonDeserializer, NewlineDelimitedDecoder, StreamDecodingError, + decoding::{Deserializer, Framer}, + }; #[tokio::test] async fn framed_read_recover_from_error() { diff --git a/lib/codecs/src/decoding/format/avro.rs b/lib/codecs/src/decoding/format/avro.rs index 9a0121a09b495..c21c09de21cfb 100644 --- a/lib/codecs/src/decoding/format/avro.rs +++ b/lib/codecs/src/decoding/format/avro.rs @@ -39,14 +39,14 @@ impl AvroDeserializerConfig { } /// Build the `AvroDeserializer` from this configuration. - pub fn build(&self) -> AvroDeserializer { + pub fn build(&self) -> vector_common::Result { let schema = apache_avro::Schema::parse_str(&self.avro_options.schema) - .map_err(|error| format!("Failed building Avro serializer: {error}")) - .unwrap(); - AvroDeserializer { + .map_err(|error| format!("Failed building Avro serializer: {error}"))?; + + Ok(AvroDeserializer { schema, strip_schema_id_prefix: self.avro_options.strip_schema_id_prefix, - } + }) } /// The data type of events that are accepted by `AvroDeserializer`. @@ -230,6 +230,15 @@ pub fn try_from(value: AvroValue) -> vector_common::Result { AvroValue::Uuid(uuid) => Ok(VrlValue::from(uuid.as_hyphenated().to_string())), AvroValue::LocalTimestampMillis(ts_millis) => Ok(VrlValue::from(ts_millis)), AvroValue::LocalTimestampMicros(ts_micros) => Ok(VrlValue::from(ts_micros)), + AvroValue::BigDecimal(_) => Err(vector_common::Error::from( + "AvroValue::BigDecimal is not supported", + )), + AvroValue::TimestampNanos(_) => Err(vector_common::Error::from( + "AvroValue::TimestampNanos is not supported", + )), + AvroValue::LocalTimestampNanos(_) => Err(vector_common::Error::from( + "AvroValue::LocalTimestampNanos is not supported", + )), } } diff --git a/lib/codecs/src/decoding/format/gelf.rs b/lib/codecs/src/decoding/format/gelf.rs index 544dd4d169937..377d437104cc4 100644 --- a/lib/codecs/src/decoding/format/gelf.rs +++ b/lib/codecs/src/decoding/format/gelf.rs @@ -36,6 +36,22 @@ pub struct GelfDeserializerConfig { pub gelf: GelfDeserializerOptions, } +/// Configures the decoding validation mode. +#[configurable_component] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +#[serde(rename_all = "snake_case")] +pub enum ValidationMode { + /// Uses strict validation that closely follows the GELF spec. + #[default] + Strict, + + /// Uses more relaxed validation that skips strict GELF specification checks. + /// + /// This mode will not treat specification violations as errors, allowing the decoder + /// to accept messages from sources that don't strictly follow the GELF spec. + Relaxed, +} + impl GelfDeserializerConfig { /// Creates a new `GelfDeserializerConfig`. pub fn new(options: GelfDeserializerOptions) -> Self { @@ -46,6 +62,7 @@ impl GelfDeserializerConfig { pub fn build(&self) -> GelfDeserializer { GelfDeserializer { lossy: self.gelf.lossy, + validation: self.gelf.validation, } } @@ -92,6 +109,10 @@ pub struct GelfDeserializerOptions { )] #[derivative(Default(value = "default_lossy()"))] pub lossy: bool, + + /// Configures the decoding validation mode. + #[serde(default, skip_serializing_if = "vector_core::serde::is_default")] + pub validation: ValidationMode, } /// Deserializer that builds an `Event` from a byte frame containing a GELF log message. @@ -100,12 +121,14 @@ pub struct GelfDeserializerOptions { pub struct GelfDeserializer { #[derivative(Default(value = "default_lossy()"))] lossy: bool, + + validation: ValidationMode, } impl GelfDeserializer { /// Create a new `GelfDeserializer`. - pub fn new(lossy: bool) -> GelfDeserializer { - GelfDeserializer { lossy } + pub fn new(lossy: bool, validation: ValidationMode) -> GelfDeserializer { + GelfDeserializer { lossy, validation } } /// Builds a LogEvent from the parsed GelfMessage. @@ -114,7 +137,7 @@ impl GelfDeserializer { let mut log = LogEvent::from_str_legacy(parsed.short_message.to_string()); // GELF spec defines the version as 1.1 which has not changed since 2013 - if parsed.version != GELF_VERSION { + if self.validation == ValidationMode::Strict && parsed.version != GELF_VERSION { return Err( format!("{VERSION} does not match GELF spec version ({GELF_VERSION})").into(), ); @@ -159,7 +182,7 @@ impl GelfDeserializer { continue; } // per GELF spec, Additional field names must be prefixed with an underscore - if !key.starts_with('_') { + if self.validation == ValidationMode::Strict && !key.starts_with('_') { return Err(format!( "'{key}' field is invalid. \ Additional field names must be prefixed with an underscore." @@ -167,7 +190,7 @@ impl GelfDeserializer { .into()); } // per GELF spec, Additional field names must be characters dashes or dots - if !VALID_FIELD_REGEX.is_match(key) { + if self.validation == ValidationMode::Strict && !VALID_FIELD_REGEX.is_match(key) { return Err(format!( "'{key}' field contains invalid characters. Field names may \ contain only letters, numbers, underscores, dashes and dots." @@ -176,7 +199,7 @@ impl GelfDeserializer { } // per GELF spec, Additional field values must be either strings or numbers - if val.is_string() || val.is_number() { + if self.validation != ValidationMode::Strict || val.is_string() || val.is_number() { let vector_val: Value = val.into(); log.insert(event_path!(key.as_str()), vector_val); } else { @@ -244,8 +267,9 @@ mod tests { fn deserialize_gelf_input( input: &serde_json::Value, + options: GelfDeserializerOptions, ) -> vector_common::Result> { - let config = GelfDeserializerConfig::default(); + let config = GelfDeserializerConfig::new(options); let deserializer = config.build(); let buffer = Bytes::from(serde_json::to_vec(&input).unwrap()); deserializer.parse(buffer, LogNamespace::Legacy) @@ -272,7 +296,7 @@ mod tests { }); // Ensure that we can parse the gelf json successfully - let events = deserialize_gelf_input(&input).unwrap(); + let events = deserialize_gelf_input(&input, GelfDeserializerOptions::default()).unwrap(); assert_eq!(events.len(), 1); let log = events[0].as_log(); @@ -334,7 +358,8 @@ mod tests { SHORT_MESSAGE: "foobar", VERSION: "1.1", }); - let events = deserialize_gelf_input(&input).unwrap(); + let events = + deserialize_gelf_input(&input, GelfDeserializerOptions::default()).unwrap(); assert_eq!(events.len(), 1); let log = events[0].as_log(); assert!(log.contains(log_schema().message_key_target_path().unwrap())); @@ -348,7 +373,8 @@ mod tests { VERSION: "1.1", "_id": "S3creTz", }); - let events = deserialize_gelf_input(&input).unwrap(); + let events = + deserialize_gelf_input(&input, GelfDeserializerOptions::default()).unwrap(); assert_eq!(events.len(), 1); let log = events[0].as_log(); assert!(!log.contains(event_path!("_id"))); @@ -359,7 +385,7 @@ mod tests { #[test] fn gelf_deserializing_err() { fn validate_err(input: &serde_json::Value) { - assert!(deserialize_gelf_input(input).is_err()); + assert!(deserialize_gelf_input(input, GelfDeserializerOptions::default()).is_err()); } // invalid character in field name validate_err(&json!({ @@ -404,4 +430,55 @@ mod tests { LEVEL: "baz", })); } + + /// Validates the relaxed validation mode + #[test] + fn gelf_deserialize_relaxed() { + let incorrect_extra_field = "incorrect^_extra_field"; + let input = json!({ + VERSION: "1.0", + HOST: "example.org", + SHORT_MESSAGE: "A short message that helps you identify what is going on", + FULL_MESSAGE: "Backtrace here\n\nmore stuff", + TIMESTAMP: 1385053862.3072, + LEVEL: 1, + FACILITY: "foo", + LINE: 42, + FILE: "/tmp/bar", + incorrect_extra_field: null, + }); + + assert!( + deserialize_gelf_input( + &input, + GelfDeserializerOptions { + validation: ValidationMode::Strict, + ..Default::default() + } + ) + .is_err() + ); + + let events = deserialize_gelf_input( + &input, + GelfDeserializerOptions { + validation: ValidationMode::Relaxed, + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(events.len(), 1); + + let log = events[0].as_log(); + + assert_eq!( + log.get(VERSION), + Some(&Value::Bytes(Bytes::from_static(b"1.0"))) + ); + + assert_eq!( + log.get(event_path!(incorrect_extra_field)), + Some(&Value::Null) + ); + } } diff --git a/lib/codecs/src/decoding/format/vrl.rs b/lib/codecs/src/decoding/format/vrl.rs index 7ed5244768b5b..c67f60fdd8952 100644 --- a/lib/codecs/src/decoding/format/vrl.rs +++ b/lib/codecs/src/decoding/format/vrl.rs @@ -59,7 +59,7 @@ impl VrlDeserializerConfig { match compile_vrl( &self.vrl.source, - &vrl::stdlib::all(), + &vector_vrl_functions::all(), &state, CompileConfig::default(), ) { diff --git a/lib/codecs/src/decoding/mod.rs b/lib/codecs/src/decoding/mod.rs index 1cf799360ffc2..c87337856454a 100644 --- a/lib/codecs/src/decoding/mod.rs +++ b/lib/codecs/src/decoding/mod.rs @@ -1,6 +1,8 @@ //! A collection of support structures that are used in the process of decoding //! bytes into events. +mod config; +mod decoder; mod error; pub mod format; pub mod framing; @@ -8,6 +10,8 @@ pub mod framing; use std::fmt::Debug; use bytes::{Bytes, BytesMut}; +pub use config::DecodingConfig; +pub use decoder::Decoder; pub use error::StreamDecodingError; pub use format::{ BoxedDeserializer, BytesDeserializer, BytesDeserializerConfig, GelfDeserializer, @@ -372,7 +376,7 @@ impl DeserializerConfig { AvroDeserializerConfig { avro_options: avro.clone(), } - .build(), + .build()?, )), DeserializerConfig::Bytes => Ok(Deserializer::Bytes(BytesDeserializerConfig.build())), DeserializerConfig::Json(config) => Ok(Deserializer::Json(config.build())), diff --git a/src/codecs/encoding/config.rs b/lib/codecs/src/encoding/config.rs similarity index 86% rename from src/codecs/encoding/config.rs rename to lib/codecs/src/encoding/config.rs index a04f44315047a..32d18c9e83d00 100644 --- a/src/codecs/encoding/config.rs +++ b/lib/codecs/src/encoding/config.rs @@ -1,14 +1,13 @@ -use crate::codecs::Transformer; -use vector_lib::{ - codecs::{ - CharacterDelimitedEncoder, LengthDelimitedEncoder, NewlineDelimitedEncoder, - encoding::{Framer, FramingConfig, Serializer, SerializerConfig}, - }, - configurable::configurable_component, +use vector_config::configurable_component; + +use super::{Encoder, EncoderKind, Transformer}; +use crate::encoding::{ + CharacterDelimitedEncoder, Framer, FramingConfig, LengthDelimitedEncoder, + NewlineDelimitedEncoder, Serializer, SerializerConfig, }; -#[cfg(feature = "codecs-opentelemetry")] -use vector_lib::codecs::BytesEncoder; +#[cfg(feature = "opentelemetry")] +use crate::encoding::BytesEncoder; /// Encoding configuration. #[configurable_component] @@ -43,7 +42,7 @@ impl EncodingConfig { } /// Build the `Serializer` for this config. - pub fn build(&self) -> crate::Result { + pub fn build(&self) -> vector_common::Result { self.encoding.build() } } @@ -100,7 +99,7 @@ impl EncodingConfigWithFraming { } /// Build the `Framer` and `Serializer` for this config. - pub fn build(&self, sink_type: SinkType) -> crate::Result<(Framer, Serializer)> { + pub fn build(&self, sink_type: SinkType) -> vector_common::Result<(Framer, Serializer)> { let framer = self.framing.as_ref().map(|framing| framing.build()); let serializer = self.encoding.build()?; @@ -132,12 +131,24 @@ impl EncodingConfigWithFraming { | Serializer::RawMessage(_) | Serializer::Text(_), ) => NewlineDelimitedEncoder::default().into(), - #[cfg(feature = "codecs-opentelemetry")] + #[cfg(feature = "syslog")] + (None, Serializer::Syslog(_)) => NewlineDelimitedEncoder::default().into(), + #[cfg(feature = "opentelemetry")] (None, Serializer::Otlp(_)) => BytesEncoder.into(), }; Ok((framer, serializer)) } + + /// Build the `Transformer` and `EncoderKind` for this config. + pub fn build_encoder( + &self, + sink_type: SinkType, + ) -> vector_common::Result<(Transformer, EncoderKind)> { + let (framer, serializer) = self.build(sink_type)?; + let encoder = EncoderKind::Framed(Box::new(Encoder::::new(framer, serializer))); + Ok((self.transformer(), encoder)) + } } /// The way a sink processes outgoing events. @@ -163,10 +174,10 @@ where #[cfg(test)] mod test { - use vector_lib::lookup::lookup_v2::{ConfigValuePath, parse_value_path}; + use lookup::lookup_v2::{ConfigValuePath, parse_value_path}; use super::*; - use crate::codecs::encoding::TimestampFormat; + use crate::encoding::TimestampFormat; #[test] fn deserialize_encoding_config() { diff --git a/src/codecs/encoding/encoder.rs b/lib/codecs/src/encoding/encoder.rs similarity index 71% rename from src/codecs/encoding/encoder.rs rename to lib/codecs/src/encoding/encoder.rs index f1d0741bb669c..4924dd05447b1 100644 --- a/src/codecs/encoding/encoder.rs +++ b/lib/codecs/src/encoding/encoder.rs @@ -1,15 +1,83 @@ use bytes::BytesMut; use tokio_util::codec::Encoder as _; -use vector_lib::codecs::{ - CharacterDelimitedEncoder, NewlineDelimitedEncoder, TextSerializerConfig, - encoding::{Error, Framer, Serializer}, -}; +use vector_common::internal_event::emit; +use vector_core::event::Event; +#[cfg(feature = "arrow")] +use crate::encoding::ArrowStreamSerializer; use crate::{ - event::Event, + encoding::{Error, Framer, Serializer}, internal_events::{EncoderFramingError, EncoderSerializeError}, }; +/// Serializers that support batch encoding (encoding all events at once). +#[derive(Debug, Clone)] +pub enum BatchSerializer { + /// Arrow IPC stream format serializer. + #[cfg(feature = "arrow")] + Arrow(ArrowStreamSerializer), +} + +/// An encoder that encodes batches of events. +#[derive(Debug, Clone)] +pub struct BatchEncoder { + serializer: BatchSerializer, +} + +impl BatchEncoder { + /// Creates a new `BatchEncoder` with the specified batch serializer. + pub const fn new(serializer: BatchSerializer) -> Self { + Self { serializer } + } + + /// Get the batch serializer. + pub const fn serializer(&self) -> &BatchSerializer { + &self.serializer + } + + /// Get the HTTP content type. + #[cfg(feature = "arrow")] + pub const fn content_type(&self) -> &'static str { + match &self.serializer { + BatchSerializer::Arrow(_) => "application/vnd.apache.arrow.stream", + } + } +} + +impl tokio_util::codec::Encoder> for BatchEncoder { + type Error = Error; + + #[allow(unused_variables)] + fn encode(&mut self, events: Vec, buffer: &mut BytesMut) -> Result<(), Self::Error> { + #[allow(unreachable_patterns)] + match &mut self.serializer { + #[cfg(feature = "arrow")] + BatchSerializer::Arrow(serializer) => { + serializer.encode(events, buffer).map_err(|err| { + use crate::encoding::ArrowEncodingError; + match err { + ArrowEncodingError::NullConstraint { .. } => { + Error::SchemaConstraintViolation(Box::new(err)) + } + _ => Error::SerializingError(Box::new(err)), + } + }) + } + _ => unreachable!("BatchSerializer cannot be constructed without encode()"), + } + } +} + +/// An wrapper that supports both framed and batch encoding modes. +#[derive(Debug, Clone)] +pub enum EncoderKind { + /// Uses framing to encode individual events + Framed(Box>), + /// Encodes events in batches without framing + #[cfg(feature = "arrow")] + Batch(BatchEncoder), +} + #[derive(Debug, Clone)] /// An encoder that can encode structured events into byte frames. pub struct Encoder @@ -22,6 +90,8 @@ where impl Default for Encoder { fn default() -> Self { + use crate::encoding::{NewlineDelimitedEncoder, TextSerializerConfig}; + Self { framer: NewlineDelimitedEncoder::default().into(), serializer: TextSerializerConfig::default().build().into(), @@ -31,6 +101,8 @@ impl Default for Encoder { impl Default for Encoder<()> { fn default() -> Self { + use crate::encoding::TextSerializerConfig; + Self { framer: (), serializer: TextSerializerConfig::default().build().into(), @@ -57,7 +129,7 @@ where /// Serialize the event without applying framing, at the start of the provided buffer. fn serialize_at_start(&mut self, event: Event, buffer: &mut BytesMut) -> Result<(), Error> { self.serializer.encode(event, buffer).map_err(|error| { - emit!(EncoderSerializeError { error: &error }); + emit(EncoderSerializeError { error: &error }); Error::SerializingError(error) }) } @@ -85,7 +157,9 @@ impl Encoder { pub const fn batch_prefix(&self) -> &[u8] { match (&self.framer, &self.serializer) { ( - Framer::CharacterDelimited(CharacterDelimitedEncoder { delimiter: b',' }), + Framer::CharacterDelimited(crate::encoding::CharacterDelimitedEncoder { + delimiter: b',', + }), Serializer::Json(_) | Serializer::NativeJson(_), ) => b"[", _ => &[], @@ -96,7 +170,9 @@ impl Encoder { pub const fn batch_suffix(&self, empty: bool) -> &[u8] { match (&self.framer, &self.serializer, empty) { ( - Framer::CharacterDelimited(CharacterDelimitedEncoder { delimiter: b',' }), + Framer::CharacterDelimited(crate::encoding::CharacterDelimitedEncoder { + delimiter: b',', + }), Serializer::Json(_) | Serializer::NativeJson(_), _, ) => b"]", @@ -113,7 +189,9 @@ impl Encoder { } ( Serializer::Gelf(_) | Serializer::Json(_) | Serializer::NativeJson(_), - Framer::CharacterDelimited(CharacterDelimitedEncoder { delimiter: b',' }), + Framer::CharacterDelimited(crate::encoding::CharacterDelimitedEncoder { + delimiter: b',', + }), ) => "application/json", (Serializer::Native(_), _) | (Serializer::Protobuf(_), _) => "application/octet-stream", ( @@ -128,7 +206,9 @@ impl Encoder { | Serializer::Text(_), _, ) => "text/plain", - #[cfg(feature = "codecs-opentelemetry")] + #[cfg(feature = "syslog")] + (Serializer::Syslog(_), _) => "text/plain", + #[cfg(feature = "opentelemetry")] (Serializer::Otlp(_), _) => "application/x-protobuf", } } @@ -161,7 +241,7 @@ impl tokio_util::codec::Encoder for Encoder { // Frame the serialized event. self.framer.encode((), &mut payload).map_err(|error| { - emit!(EncoderFramingError { error: &error }); + emit(EncoderFramingError { error: &error }); Error::FramingError(error) })?; @@ -189,11 +269,12 @@ impl tokio_util::codec::Encoder for Encoder<()> { #[cfg(test)] mod tests { use bytes::BufMut; - use futures_util::{SinkExt, StreamExt}; + use futures::{SinkExt, StreamExt}; use tokio_util::codec::FramedWrite; - use vector_lib::{codecs::encoding::BoxedFramingError, event::LogEvent}; + use vector_core::event::LogEvent; use super::*; + use crate::encoding::BoxedFramingError; #[derive(Debug, Clone)] struct ParenEncoder; @@ -253,7 +334,9 @@ mod tests { async fn test_encode_events_sink_empty() { let encoder = Encoder::::new( Framer::Boxed(Box::new(ParenEncoder::new())), - TextSerializerConfig::default().build().into(), + crate::encoding::TextSerializerConfig::default() + .build() + .into(), ); let source = futures::stream::iter(vec![ Event::Log(LogEvent::from("foo")), @@ -272,7 +355,9 @@ mod tests { async fn test_encode_events_sink_non_empty() { let encoder = Encoder::::new( Framer::Boxed(Box::new(ParenEncoder::new())), - TextSerializerConfig::default().build().into(), + crate::encoding::TextSerializerConfig::default() + .build() + .into(), ); let source = futures::stream::iter(vec![ Event::Log(LogEvent::from("bar")), @@ -291,7 +376,9 @@ mod tests { async fn test_encode_events_sink_empty_handle_framing_error() { let encoder = Encoder::::new( Framer::Boxed(Box::new(ErrorNthEncoder::new(ParenEncoder::new(), 1))), - TextSerializerConfig::default().build().into(), + crate::encoding::TextSerializerConfig::default() + .build() + .into(), ); let source = futures::stream::iter(vec![ Event::Log(LogEvent::from("foo")), @@ -311,7 +398,9 @@ mod tests { async fn test_encode_events_sink_non_empty_handle_framing_error() { let encoder = Encoder::::new( Framer::Boxed(Box::new(ErrorNthEncoder::new(ParenEncoder::new(), 1))), - TextSerializerConfig::default().build().into(), + crate::encoding::TextSerializerConfig::default() + .build() + .into(), ); let source = futures::stream::iter(vec![ Event::Log(LogEvent::from("bar")), @@ -330,8 +419,10 @@ mod tests { #[tokio::test] async fn test_encode_batch_newline() { let encoder = Encoder::::new( - Framer::NewlineDelimited(NewlineDelimitedEncoder::default()), - TextSerializerConfig::default().build().into(), + Framer::NewlineDelimited(crate::encoding::NewlineDelimitedEncoder::default()), + crate::encoding::TextSerializerConfig::default() + .build() + .into(), ); let source = futures::stream::iter(vec![ Event::Log(LogEvent::from("bar")), diff --git a/lib/codecs/src/encoding/format/arrow.rs b/lib/codecs/src/encoding/format/arrow.rs new file mode 100644 index 0000000000000..4812b4f573c1e --- /dev/null +++ b/lib/codecs/src/encoding/format/arrow.rs @@ -0,0 +1,979 @@ +//! Arrow IPC streaming format codec for batched event encoding +//! +//! Provides Apache Arrow IPC stream format encoding with static schema support. +//! This implements the streaming variant of the Arrow IPC protocol, which writes +//! a continuous stream of record batches without a file footer. + +use arrow::{ + array::ArrayRef, + compute::{CastOptions, cast_with_options}, + datatypes::{DataType, Field, Fields, Schema, SchemaRef, TimeUnit}, + ipc::writer::StreamWriter, + record_batch::RecordBatch, +}; +use async_trait::async_trait; +use bytes::{BufMut, Bytes, BytesMut}; +use chrono::{DateTime, Utc}; +use snafu::Snafu; +use std::sync::Arc; +use vector_config::configurable_component; +use vector_core::event::{Event, LogEvent, Value}; + +/// Provides Arrow schema for encoding. +/// +/// Sinks can implement this trait to provide custom schema fetching logic. +#[async_trait] +pub trait SchemaProvider: Send + Sync + std::fmt::Debug { + /// Fetch the Arrow schema from the data store. + /// + /// This is called during sink configuration build phase to fetch + /// the schema once at startup, rather than at runtime. + async fn get_schema(&self) -> Result; +} + +/// Configuration for Arrow IPC stream serialization +#[configurable_component] +#[derive(Clone, Default)] +pub struct ArrowStreamSerializerConfig { + /// The Arrow schema to use for encoding + #[serde(skip)] + #[configurable(derived)] + pub schema: Option, + + /// Allow null values for non-nullable fields in the schema. + /// + /// When enabled, missing or incompatible values will be encoded as null even for fields + /// marked as non-nullable in the Arrow schema. This is useful when working with downstream + /// systems that can handle null values through defaults, computed columns, or other mechanisms. + /// + /// When disabled (default), missing values for non-nullable fields will cause encoding errors, + /// ensuring all required data is present before sending to the sink. + #[serde(default)] + #[configurable(derived)] + pub allow_nullable_fields: bool, +} + +impl std::fmt::Debug for ArrowStreamSerializerConfig { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ArrowStreamSerializerConfig") + .field( + "schema", + &self + .schema + .as_ref() + .map(|s| format!("{} fields", s.fields().len())), + ) + .field("allow_nullable_fields", &self.allow_nullable_fields) + .finish() + } +} + +impl ArrowStreamSerializerConfig { + /// Create a new ArrowStreamSerializerConfig with a schema + pub fn new(schema: arrow::datatypes::Schema) -> Self { + Self { + schema: Some(schema), + allow_nullable_fields: false, + } + } + + /// The data type of events that are accepted by `ArrowStreamEncoder`. + pub fn input_type(&self) -> vector_core::config::DataType { + vector_core::config::DataType::Log + } + + /// The schema required by the serializer. + pub fn schema_requirement(&self) -> vector_core::schema::Requirement { + vector_core::schema::Requirement::empty() + } +} + +/// Arrow IPC stream batch serializer that holds the schema +#[derive(Clone, Debug)] +pub struct ArrowStreamSerializer { + schema: SchemaRef, +} + +impl ArrowStreamSerializer { + /// Create a new ArrowStreamSerializer with the given configuration + pub fn new(config: ArrowStreamSerializerConfig) -> Result { + let schema = config + .schema + .ok_or_else(|| vector_common::Error::from("Arrow serializer requires a schema."))?; + + // If allow_nullable_fields is enabled, transform the schema once here + // instead of on every batch encoding + let schema = if config.allow_nullable_fields { + let nullable_fields: Fields = schema + .fields() + .iter() + .map(|f| make_field_nullable(f)) + .collect::, _>>() + .map_err(|e| vector_common::Error::from(e.to_string()))? + .into(); + Schema::new_with_metadata(nullable_fields, schema.metadata().clone()) + } else { + schema + }; + + Ok(Self { + schema: SchemaRef::new(schema), + }) + } +} + +impl tokio_util::codec::Encoder> for ArrowStreamSerializer { + type Error = ArrowEncodingError; + + fn encode(&mut self, events: Vec, buffer: &mut BytesMut) -> Result<(), Self::Error> { + if events.is_empty() { + return Err(ArrowEncodingError::NoEvents); + } + + let bytes = encode_events_to_arrow_ipc_stream(&events, Some(Arc::clone(&self.schema)))?; + + buffer.extend_from_slice(&bytes); + Ok(()) + } +} + +/// Errors that can occur during Arrow encoding +#[derive(Debug, Snafu)] +pub enum ArrowEncodingError { + /// Failed to create Arrow record batch + #[snafu(display("Failed to create Arrow record batch: {}", source))] + RecordBatchCreation { + /// The underlying Arrow error + source: arrow::error::ArrowError, + }, + + /// Failed to write Arrow IPC data + #[snafu(display("Failed to write Arrow IPC data: {}", source))] + IpcWrite { + /// The underlying Arrow error + source: arrow::error::ArrowError, + }, + + /// No events provided for encoding + #[snafu(display("No events provided for encoding"))] + NoEvents, + + /// Schema must be provided before encoding + #[snafu(display("Schema must be provided before encoding"))] + NoSchemaProvided, + + /// Failed to fetch schema from provider + #[snafu(display("Failed to fetch schema from provider: {}", message))] + SchemaFetchError { + /// Error message from the provider + message: String, + }, + + /// Null value encountered for non-nullable field + #[snafu(display("Null value for non-nullable field '{}'", field_name))] + NullConstraint { + /// The field name + field_name: String, + }, + + /// IO error during encoding + #[snafu(display("IO error: {}", source))] + Io { + /// The underlying IO error + source: std::io::Error, + }, + + /// Serde Arrow serialization error + #[snafu(display("Serde Arrow error: {}", source))] + SerdeArrow { + /// The underlying serde_arrow error + source: serde_arrow::Error, + }, + + /// Timestamp value overflows the representable range + #[snafu(display( + "Timestamp overflow for field '{}': value '{}' cannot be represented as i64 nanoseconds", + field_name, + timestamp + ))] + TimestampOverflow { + /// The field name + field_name: String, + /// The timestamp value that overflowed + timestamp: String, + }, + + /// Invalid Map schema structure + #[snafu(display("Invalid Map schema for field '{}': {}", field_name, reason))] + InvalidMapSchema { + /// The field name + field_name: String, + /// Description of the schema violation + reason: String, + }, +} + +impl From for ArrowEncodingError { + fn from(error: std::io::Error) -> Self { + Self::Io { source: error } + } +} + +/// Encodes a batch of events into Arrow IPC streaming format +pub fn encode_events_to_arrow_ipc_stream( + events: &[Event], + schema: Option, +) -> Result { + if events.is_empty() { + return Err(ArrowEncodingError::NoEvents); + } + + let schema_ref = schema.ok_or(ArrowEncodingError::NoSchemaProvided)?; + + let record_batch = build_record_batch(schema_ref, events)?; + + let ipc_err = |source| ArrowEncodingError::IpcWrite { source }; + + let mut buffer = BytesMut::new().writer(); + let mut writer = + StreamWriter::try_new(&mut buffer, record_batch.schema_ref()).map_err(ipc_err)?; + writer.write(&record_batch).map_err(ipc_err)?; + writer.finish().map_err(ipc_err)?; + + Ok(buffer.into_inner().freeze()) +} + +/// Recursively makes a Field and all its nested fields nullable +fn make_field_nullable(field: &Field) -> Result { + let new_data_type = match field.data_type() { + DataType::List(inner_field) => DataType::List(make_field_nullable(inner_field)?.into()), + DataType::Struct(fields) => DataType::Struct( + fields + .iter() + .map(|f| make_field_nullable(f)) + .collect::, _>>()? + .into(), + ), + DataType::Map(inner, sorted) => { + // A Map's inner field is a "entries" Struct + let DataType::Struct(fields) = inner.data_type() else { + return Err(ArrowEncodingError::InvalidMapSchema { + field_name: field.name().to_string(), + reason: format!("inner type must be Struct, found {:?}", inner.data_type()), + }); + }; + + if fields.len() != 2 { + return Err(ArrowEncodingError::InvalidMapSchema { + field_name: field.name().to_string(), + reason: format!("expected 2 fields (key, value), found {}", fields.len()), + }); + } + let key_field = &fields[0]; + let value_field = &fields[1]; + + let new_struct_fields: Fields = + [key_field.clone(), make_field_nullable(value_field)?.into()].into(); + + // Reconstruct the inner "entries" field + // The inner field itself must be non-nullable (only the Map wrapper is nullable) + let new_inner_field = inner + .as_ref() + .clone() + .with_data_type(DataType::Struct(new_struct_fields)) + .with_nullable(false); + + DataType::Map(new_inner_field.into(), *sorted) + } + other => other.clone(), + }; + + Ok(field + .clone() + .with_data_type(new_data_type) + .with_nullable(true)) +} + +/// Build an Arrow RecordBatch from a slice of events using the provided schema. +fn build_record_batch( + schema: SchemaRef, + events: &[Event], +) -> Result { + let log_events: Vec = events + .iter() + .filter_map(Event::maybe_as_log) + .map(|log| convert_timestamps(log, &schema)) + .collect::, _>>()?; + + let batch = serde_arrow::to_record_batch(schema.fields(), &log_events).map_err(|source| { + // serde_arrow doesn't expose structured error variants (see + // https://docs.rs/serde_arrow/latest/serde_arrow/enum.Error.html), so we string-match on + // the message to detect null constraint violations, then find the actual field ourselves. + if source.message().contains("non-nullable") + && let Some(field_name) = find_null_field(&log_events, &schema) + { + return ArrowEncodingError::NullConstraint { field_name }; + } + ArrowEncodingError::SerdeArrow { source } + })?; + + // Post-process: use Arrow's cast for any remaining type mismatches. + // serde_arrow serializes Vector's Value types using fixed Arrow types (e.g., Int64 + // for all integers, Float64 for floats, LargeUtf8 for strings), but the target schema + // may specify narrower types. Arrow's cast handles these conversions safely. + let columns: Result, _> = batch + .columns() + .iter() + .zip(schema.fields()) + .map(|(col, field)| { + if col.data_type() == field.data_type() { + Ok(col.clone()) + } else { + cast_with_options(col, field.data_type(), &CastOptions::default()) + .map_err(|source| ArrowEncodingError::RecordBatchCreation { source }) + } + }) + .collect(); + + RecordBatch::try_new(schema, columns?) + .map_err(|source| ArrowEncodingError::RecordBatchCreation { source }) +} + +/// Find which non-nullable field has a missing value (called only on error). +fn find_null_field(events: &[LogEvent], schema: &SchemaRef) -> Option { + for field in schema.fields() { + if !field.is_nullable() { + let name = field.name(); + if events + .iter() + .any(|e| e.get(lookup::event_path!(name)).is_none()) + { + return Some(name.to_string()); + } + } + } + None +} + +/// Convert Value::Timestamp to Value::Integer for timestamp columns. +/// +/// This is necessary because serde_arrow's string parsing expects specific formats +/// based on the timezone setting, but Vector's timestamps always serialize as RFC 3339 +/// with 'Z' suffix. Converting to i64 directly avoids this format mismatch. +fn convert_timestamps( + event: &LogEvent, + schema: &SchemaRef, +) -> Result { + let mut result = event.clone(); + + for field in schema.fields() { + if let DataType::Timestamp(unit, _) = field.data_type() { + let field_name = field.name().as_str(); + + if let Some(Value::Timestamp(ts)) = event.get(lookup::event_path!(field_name)) { + let val = timestamp_to_unit(ts, unit).ok_or_else(|| { + ArrowEncodingError::TimestampOverflow { + field_name: field_name.to_string(), + timestamp: ts.to_rfc3339(), + } + })?; + result.insert(field_name, Value::Integer(val)); + } + } + } + + Ok(result) +} + +/// Convert a DateTime to i64 in the specified Arrow TimeUnit. +/// Returns None if the value would overflow (only possible for nanoseconds). +fn timestamp_to_unit(ts: &DateTime, unit: &TimeUnit) -> Option { + match unit { + TimeUnit::Second => Some(ts.timestamp()), + TimeUnit::Millisecond => Some(ts.timestamp_millis()), + TimeUnit::Microsecond => Some(ts.timestamp_micros()), + TimeUnit::Nanosecond => ts.timestamp_nanos_opt(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow::{ + array::{ + Array, BinaryArray, BooleanArray, Float64Array, Int64Array, StringArray, + TimestampMicrosecondArray, TimestampMillisecondArray, TimestampNanosecondArray, + TimestampSecondArray, + }, + ipc::reader::StreamReader, + }; + use std::io::Cursor; + + /// Helper to encode events and return the decoded RecordBatch + fn encode_and_decode( + events: Vec, + schema: SchemaRef, + ) -> Result> { + let bytes = encode_events_to_arrow_ipc_stream(&events, Some(Arc::clone(&schema)))?; + let cursor = Cursor::new(bytes); + let mut reader = StreamReader::try_new(cursor, None)?; + Ok(reader.next().unwrap()?) + } + + /// Create a simple event from key-value pairs + fn create_event(fields: Vec<(&str, V)>) -> Event + where + V: Into, + { + let mut log = LogEvent::default(); + for (key, value) in fields { + log.insert(key, value.into()); + } + Event::Log(log) + } + + /// Assert a primitive value at a specific column and row + macro_rules! assert_primitive_value { + ($batch:expr, $col:expr, $row:expr, $array_type:ty, $expected:expr) => { + assert_eq!( + $batch + .column($col) + .as_any() + .downcast_ref::<$array_type>() + .unwrap() + .value($row), + $expected + ) + }; + } + + mod comprehensive { + use super::*; + + #[test] + fn test_encode_all_types() { + use arrow::array::{ + Decimal128Array, ListArray, MapArray, UInt8Array, UInt16Array, UInt32Array, + UInt64Array, + }; + use vrl::value::ObjectMap; + + let now = Utc::now(); + + // Create a struct (tuple) value with unnamed fields + let mut tuple_value = ObjectMap::new(); + tuple_value.insert("f0".into(), Value::Bytes("nested_str".into())); + tuple_value.insert("f1".into(), Value::Integer(999)); + + // Create a named struct (named tuple) value + let mut named_tuple_value = ObjectMap::new(); + named_tuple_value.insert("category".into(), Value::Bytes("test_category".into())); + named_tuple_value.insert("tag".into(), Value::Bytes("test_tag".into())); + + // Create a list value + let list_value = Value::Array(vec![ + Value::Integer(1), + Value::Integer(2), + Value::Integer(3), + ]); + + // Create a map value + let mut map_value = ObjectMap::new(); + map_value.insert("key1".into(), Value::Integer(100)); + map_value.insert("key2".into(), Value::Integer(200)); + + let mut log = LogEvent::default(); + // Primitive types + log.insert("string_field", "test"); + log.insert("int8_field", 127); + log.insert("int16_field", 32000); + log.insert("int32_field", 1000000); + log.insert("int64_field", 42); + log.insert("uint8_field", 255); + log.insert("uint16_field", 65535); + log.insert("uint32_field", 4000000); + log.insert("uint64_field", 9000000000_i64); + log.insert("float32_field", 3.15); + log.insert("float64_field", 3.15); + log.insert("bool_field", true); + log.insert("bytes_field", bytes::Bytes::from("binary")); + log.insert("timestamp_field", now); + log.insert("decimal_field", 99.99); + // Complex types + log.insert("list_field", list_value); + log.insert("struct_field", Value::Object(tuple_value)); + log.insert("named_struct_field", Value::Object(named_tuple_value)); + log.insert("map_field", Value::Object(map_value)); + + let events = vec![Event::Log(log)]; + + // Build schema with all supported types + let struct_fields = arrow::datatypes::Fields::from(vec![ + Field::new("f0", DataType::Utf8, true), + Field::new("f1", DataType::Int64, true), + ]); + + let named_struct_fields = arrow::datatypes::Fields::from(vec![ + Field::new("category", DataType::Utf8, true), + Field::new("tag", DataType::Utf8, true), + ]); + + let map_entries = Field::new( + "entries", + DataType::Struct(arrow::datatypes::Fields::from(vec![ + Field::new("keys", DataType::Utf8, false), + Field::new("values", DataType::Int64, true), + ])), + false, + ); + + let schema = SchemaRef::new(Schema::new(vec![ + Field::new("string_field", DataType::Utf8, true), + Field::new("int8_field", DataType::Int8, true), + Field::new("int16_field", DataType::Int16, true), + Field::new("int32_field", DataType::Int32, true), + Field::new("int64_field", DataType::Int64, true), + Field::new("uint8_field", DataType::UInt8, true), + Field::new("uint16_field", DataType::UInt16, true), + Field::new("uint32_field", DataType::UInt32, true), + Field::new("uint64_field", DataType::UInt64, true), + Field::new("float32_field", DataType::Float32, true), + Field::new("float64_field", DataType::Float64, true), + Field::new("bool_field", DataType::Boolean, true), + Field::new("bytes_field", DataType::Binary, true), + Field::new( + "timestamp_field", + DataType::Timestamp(TimeUnit::Millisecond, None), + true, + ), + Field::new("decimal_field", DataType::Decimal128(10, 2), true), + Field::new( + "list_field", + DataType::List(Field::new("item", DataType::Int64, true).into()), + true, + ), + Field::new("struct_field", DataType::Struct(struct_fields), true), + Field::new( + "named_struct_field", + DataType::Struct(named_struct_fields), + true, + ), + Field::new("map_field", DataType::Map(map_entries.into(), false), true), + ])); + + let batch = encode_and_decode(events, schema).expect("Failed to encode"); + + assert_eq!(batch.num_rows(), 1); + assert_eq!(batch.num_columns(), 19); + + // Verify all primitive types + assert_eq!( + batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + "test" + ); + assert_primitive_value!(batch, 1, 0, arrow::array::Int8Array, 127); + assert_primitive_value!(batch, 2, 0, arrow::array::Int16Array, 32000); + assert_primitive_value!(batch, 3, 0, arrow::array::Int32Array, 1000000); + assert_primitive_value!(batch, 4, 0, Int64Array, 42); + assert_primitive_value!(batch, 5, 0, UInt8Array, 255); + assert_primitive_value!(batch, 6, 0, UInt16Array, 65535); + assert_primitive_value!(batch, 7, 0, UInt32Array, 4000000); + assert_primitive_value!(batch, 8, 0, UInt64Array, 9000000000); + assert!( + (batch + .column(9) + .as_any() + .downcast_ref::() + .unwrap() + .value(0) + - 3.15) + .abs() + < 0.001 + ); + assert!( + (batch + .column(10) + .as_any() + .downcast_ref::() + .unwrap() + .value(0) + - 3.15) + .abs() + < 0.001 + ); + assert!( + batch + .column(11) + .as_any() + .downcast_ref::() + .unwrap() + .value(0) + ); + assert_primitive_value!(batch, 12, 0, BinaryArray, b"binary"); + assert_primitive_value!( + batch, + 13, + 0, + TimestampMillisecondArray, + now.timestamp_millis() + ); + assert_primitive_value!(batch, 14, 0, Decimal128Array, 9999); + + let list_array = batch + .column(15) + .as_any() + .downcast_ref::() + .unwrap(); + assert!(!list_array.is_null(0)); + let list_value = list_array.value(0); + assert_eq!(list_value.len(), 3); + let int_array = list_value.as_any().downcast_ref::().unwrap(); + assert_eq!(int_array.value(0), 1); + assert_eq!(int_array.value(1), 2); + assert_eq!(int_array.value(2), 3); + + // Verify struct field (unnamed) + let struct_array = batch + .column(16) + .as_any() + .downcast_ref::() + .unwrap(); + assert!(!struct_array.is_null(0)); + assert_primitive_value!(struct_array, 0, 0, StringArray, "nested_str"); + assert_primitive_value!(struct_array, 1, 0, Int64Array, 999); + + // Verify named struct field (named tuple) + let named_struct_array = batch + .column(17) + .as_any() + .downcast_ref::() + .unwrap(); + assert!(!named_struct_array.is_null(0)); + assert_primitive_value!(named_struct_array, 0, 0, StringArray, "test_category"); + assert_primitive_value!(named_struct_array, 1, 0, StringArray, "test_tag"); + + // Verify map field + let map_array = batch + .column(18) + .as_any() + .downcast_ref::() + .unwrap(); + assert!(!map_array.is_null(0)); + let map_value = map_array.value(0); + assert_eq!(map_value.len(), 2); + } + } + + mod error_handling { + use super::*; + + #[test] + fn test_encode_without_schema_fails() { + let events = vec![create_event(vec![("message", "hello")])]; + + let result = encode_events_to_arrow_ipc_stream(&events, None); + assert!(result.is_err()); + assert!(matches!( + result.unwrap_err(), + ArrowEncodingError::NoSchemaProvided + )); + } + + #[test] + fn test_encode_empty_events() { + let events: Vec = vec![]; + let result = encode_events_to_arrow_ipc_stream(&events, None); + assert!(result.is_err()); + assert!(matches!(result.unwrap_err(), ArrowEncodingError::NoEvents)); + } + + #[test] + fn test_null_constraint_error() { + let events = vec![create_event(vec![("other_field", "value")])]; + + let schema = SchemaRef::new(Schema::new(vec![Field::new( + "required_field", + DataType::Utf8, + false, // non-nullable + )])); + + let result = encode_events_to_arrow_ipc_stream(&events, Some(schema)); + assert!(result.is_err()); + match result.unwrap_err() { + ArrowEncodingError::NullConstraint { field_name } => { + assert_eq!(field_name, "required_field"); + } + other => panic!("Expected NullConstraint error, got: {:?}", other), + } + } + } + + mod temporal_types { + use super::*; + + #[test] + fn test_encode_timestamp_precisions() { + let now = Utc::now(); + let mut log = LogEvent::default(); + log.insert("ts_second", now); + log.insert("ts_milli", now); + log.insert("ts_micro", now); + log.insert("ts_nano", now); + + let events = vec![Event::Log(log)]; + + let schema = SchemaRef::new(Schema::new(vec![ + Field::new( + "ts_second", + DataType::Timestamp(TimeUnit::Second, None), + true, + ), + Field::new( + "ts_milli", + DataType::Timestamp(TimeUnit::Millisecond, None), + true, + ), + Field::new( + "ts_micro", + DataType::Timestamp(TimeUnit::Microsecond, None), + true, + ), + Field::new( + "ts_nano", + DataType::Timestamp(TimeUnit::Nanosecond, None), + true, + ), + ])); + + let batch = encode_and_decode(events, schema).unwrap(); + + assert_eq!(batch.num_rows(), 1); + assert_eq!(batch.num_columns(), 4); + + let ts_second = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert!(!ts_second.is_null(0)); + assert_eq!(ts_second.value(0), now.timestamp()); + + let ts_milli = batch + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + assert!(!ts_milli.is_null(0)); + assert_eq!(ts_milli.value(0), now.timestamp_millis()); + + let ts_micro = batch + .column(2) + .as_any() + .downcast_ref::() + .unwrap(); + assert!(!ts_micro.is_null(0)); + assert_eq!(ts_micro.value(0), now.timestamp_micros()); + + let ts_nano = batch + .column(3) + .as_any() + .downcast_ref::() + .unwrap(); + assert!(!ts_nano.is_null(0)); + assert_eq!(ts_nano.value(0), now.timestamp_nanos_opt().unwrap()); + } + + #[test] + fn test_encode_mixed_timestamp_string_native_and_integer() { + // Test mixing RFC3339 string timestamps, native Timestamp values, and integers. + // Note: String timestamps require the schema to have Some("UTC") timezone for + // serde_arrow to parse them correctly. Native Value::Timestamp values are + // converted to integers internally, so they work with any timezone setting. + let now = Utc::now(); + + let mut log1 = LogEvent::default(); + log1.insert("ts", "2025-10-22T10:18:44.256Z"); // RFC3339 String + + let mut log2 = LogEvent::default(); + log2.insert("ts", now); // Native Timestamp + + let mut log3 = LogEvent::default(); + log3.insert("ts", 1729594724256000000_i64); // Integer (nanoseconds) + + let events = vec![Event::Log(log1), Event::Log(log2), Event::Log(log3)]; + + // Use Some("UTC") to enable serde_arrow's RFC3339 string parsing + let schema = SchemaRef::new(Schema::new(vec![Field::new( + "ts", + DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into())), + true, + )])); + + let batch = encode_and_decode(events, schema).unwrap(); + + assert_eq!(batch.num_rows(), 3); + + let ts_array = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + + // All three should be non-null + assert!(!ts_array.is_null(0)); + assert!(!ts_array.is_null(1)); + assert!(!ts_array.is_null(2)); + + // First one should match the parsed RFC3339 string + let expected = chrono::DateTime::parse_from_rfc3339("2025-10-22T10:18:44.256Z") + .unwrap() + .timestamp_nanos_opt() + .unwrap(); + assert_eq!(ts_array.value(0), expected); + + // Second one should match the native timestamp + assert_eq!(ts_array.value(1), now.timestamp_nanos_opt().unwrap()); + + // Third one should match the integer + assert_eq!(ts_array.value(2), 1729594724256000000_i64); + } + } + + mod config_tests { + use super::*; + use tokio_util::codec::Encoder; + + #[test] + fn test_config_allow_nullable_fields_overrides_schema() { + let mut log1 = LogEvent::default(); + log1.insert("strict_field", 42); + let log2 = LogEvent::default(); + let events = vec![Event::Log(log1), Event::Log(log2)]; + + let schema = Schema::new(vec![Field::new("strict_field", DataType::Int64, false)]); + + let mut config = ArrowStreamSerializerConfig::new(schema); + config.allow_nullable_fields = true; + + let mut serializer = + ArrowStreamSerializer::new(config).expect("Failed to create serializer"); + + let mut buffer = BytesMut::new(); + serializer + .encode(events, &mut buffer) + .expect("Encoding should succeed when allow_nullable_fields is true"); + + let cursor = Cursor::new(buffer); + let mut reader = StreamReader::try_new(cursor, None).expect("Failed to create reader"); + let batch = reader.next().unwrap().expect("Failed to read batch"); + + assert_eq!(batch.num_rows(), 2); + + let binding = batch.schema(); + let output_field = binding.field(0); + assert!( + output_field.is_nullable(), + "The output schema field should have been transformed to nullable=true" + ); + + let array = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + + assert_eq!(array.value(0), 42); + assert!(!array.is_null(0)); + assert!( + array.is_null(1), + "The missing value should be encoded as null" + ); + } + + #[test] + fn test_make_field_nullable_with_nested_types() { + let inner_struct_field = Field::new("nested_field", DataType::Int64, false); + let inner_struct = + DataType::Struct(arrow::datatypes::Fields::from(vec![inner_struct_field])); + let list_field = Field::new("item", inner_struct, false); + let list_type = DataType::List(list_field.into()); + let outer_field = Field::new("inner_list", list_type, false); + let outer_struct = DataType::Struct(arrow::datatypes::Fields::from(vec![outer_field])); + + let original_field = Field::new("root", outer_struct, false); + let nullable_field = make_field_nullable(&original_field).unwrap(); + + assert!( + nullable_field.is_nullable(), + "Root field should be nullable" + ); + + if let DataType::Struct(root_fields) = nullable_field.data_type() { + let inner_list_field = &root_fields[0]; + assert!(inner_list_field.is_nullable()); + + if let DataType::List(list_item_field) = inner_list_field.data_type() { + assert!(list_item_field.is_nullable()); + + if let DataType::Struct(inner_struct_fields) = list_item_field.data_type() { + let nested_field = &inner_struct_fields[0]; + assert!(nested_field.is_nullable()); + } else { + panic!("Expected Struct type for list items"); + } + } else { + panic!("Expected List type for inner_list"); + } + } else { + panic!("Expected Struct type for root field"); + } + } + + #[test] + fn test_make_field_nullable_with_map_type() { + let key_field = Field::new("key", DataType::Utf8, false); + let value_field = Field::new("value", DataType::Int64, false); + let entries_struct = + DataType::Struct(arrow::datatypes::Fields::from(vec![key_field, value_field])); + let entries_field = Field::new("entries", entries_struct, false); + let map_type = DataType::Map(entries_field.into(), false); + + let original_field = Field::new("my_map", map_type, false); + let nullable_field = make_field_nullable(&original_field).unwrap(); + + assert!( + nullable_field.is_nullable(), + "Root map field should be nullable" + ); + + if let DataType::Map(entries_field, _sorted) = nullable_field.data_type() { + assert!( + !entries_field.is_nullable(), + "Map entries field should be non-nullable" + ); + + if let DataType::Struct(struct_fields) = entries_field.data_type() { + let key_field = &struct_fields[0]; + let value_field = &struct_fields[1]; + assert!( + !key_field.is_nullable(), + "Map key field should be non-nullable" + ); + assert!( + value_field.is_nullable(), + "Map value field should be nullable" + ); + } else { + panic!("Expected Struct type for map entries"); + } + } else { + panic!("Expected Map type for my_map field"); + } + } + } +} diff --git a/lib/codecs/src/encoding/format/gelf.rs b/lib/codecs/src/encoding/format/gelf.rs index aedb6d20678b7..966126df4f895 100644 --- a/lib/codecs/src/encoding/format/gelf.rs +++ b/lib/codecs/src/encoding/format/gelf.rs @@ -206,7 +206,11 @@ fn coerce_field_names_and_values( if let Value::Timestamp(ts) = value { let ts_millis = ts.timestamp_millis(); if ts_millis % 1000 != 0 { - *value = Value::Float(NotNan::new(ts_millis as f64 / 1000.0).unwrap()); + // i64 to f64 / 1000.0 will never be NaN + *value = Value::Float( + NotNan::new(ts_millis as f64 / 1000.0) + .expect("i64 -> f64 produced NaN"), + ); } else { // keep full range of representable time if no milliseconds are set // but still convert to numeric according to GELF protocol diff --git a/lib/codecs/src/encoding/format/mod.rs b/lib/codecs/src/encoding/format/mod.rs index 9377cdca5d906..5026dda422d9e 100644 --- a/lib/codecs/src/encoding/format/mod.rs +++ b/lib/codecs/src/encoding/format/mod.rs @@ -3,6 +3,8 @@ #![deny(missing_docs)] +#[cfg(feature = "arrow")] +mod arrow; mod avro; mod cef; mod common; @@ -16,10 +18,16 @@ mod native_json; mod otlp; mod protobuf; mod raw_message; +#[cfg(feature = "syslog")] +mod syslog; mod text; use std::fmt::Debug; +#[cfg(feature = "arrow")] +pub use arrow::{ + ArrowEncodingError, ArrowStreamSerializer, ArrowStreamSerializerConfig, SchemaProvider, +}; pub use avro::{AvroSerializer, AvroSerializerConfig, AvroSerializerOptions}; pub use cef::{CefSerializer, CefSerializerConfig}; use dyn_clone::DynClone; @@ -32,6 +40,8 @@ pub use native_json::{NativeJsonSerializer, NativeJsonSerializerConfig}; pub use otlp::{OtlpSerializer, OtlpSerializerConfig}; pub use protobuf::{ProtobufSerializer, ProtobufSerializerConfig, ProtobufSerializerOptions}; pub use raw_message::{RawMessageSerializer, RawMessageSerializerConfig}; +#[cfg(feature = "syslog")] +pub use syslog::{SyslogSerializer, SyslogSerializerConfig}; pub use text::{TextSerializer, TextSerializerConfig}; use vector_core::event::Event; diff --git a/lib/codecs/src/encoding/format/syslog.rs b/lib/codecs/src/encoding/format/syslog.rs new file mode 100644 index 0000000000000..601d4d3c04617 --- /dev/null +++ b/lib/codecs/src/encoding/format/syslog.rs @@ -0,0 +1,850 @@ +use bytes::{BufMut, BytesMut}; +use chrono::{DateTime, SecondsFormat, SubsecRound, Utc}; +use lookup::lookup_v2::ConfigTargetPath; +use std::collections::HashMap; +use std::fmt::Write; +use std::str::FromStr; +use strum::{EnumString, FromRepr, VariantNames}; +use tokio_util::codec::Encoder; +use vector_config::configurable_component; +use vector_core::{ + config::DataType, + event::{Event, LogEvent, Value}, + schema, +}; +use vrl::value::ObjectMap; + +/// Config used to build a `SyslogSerializer`. +#[configurable_component] +#[derive(Clone, Debug, Default)] +#[serde(default)] +pub struct SyslogSerializerConfig { + /// Options for the Syslog serializer. + pub syslog: SyslogSerializerOptions, +} + +impl SyslogSerializerConfig { + /// Build the `SyslogSerializer` from this configuration. + pub fn build(&self) -> SyslogSerializer { + SyslogSerializer::new(self) + } + + /// The data type of events that are accepted by `SyslogSerializer`. + pub fn input_type(&self) -> DataType { + DataType::Log + } + + /// The schema required by the serializer. + pub fn schema_requirement(&self) -> schema::Requirement { + schema::Requirement::empty() + } +} + +/// Syslog serializer options. +#[configurable_component] +#[derive(Clone, Debug, Default)] +#[serde(default, deny_unknown_fields)] +pub struct SyslogSerializerOptions { + /// RFC to use for formatting. + rfc: SyslogRFC, + /// Path to a field in the event to use for the facility. Defaults to "user". + facility: Option, + /// Path to a field in the event to use for the severity. Defaults to "informational". + severity: Option, + /// Path to a field in the event to use for the app name. + /// + /// If not provided, the encoder checks for a semantic "service" field. + /// If that is also missing, it defaults to "vector". + app_name: Option, + /// Path to a field in the event to use for the proc ID. + proc_id: Option, + /// Path to a field in the event to use for the msg ID. + msg_id: Option, +} + +/// Serializer that converts an `Event` to bytes using the Syslog format. +#[derive(Debug, Clone)] +pub struct SyslogSerializer { + config: SyslogSerializerConfig, +} + +impl SyslogSerializer { + /// Creates a new `SyslogSerializer`. + pub fn new(conf: &SyslogSerializerConfig) -> Self { + Self { + config: conf.clone(), + } + } +} + +impl Encoder for SyslogSerializer { + type Error = vector_common::Error; + + fn encode(&mut self, event: Event, buffer: &mut BytesMut) -> Result<(), Self::Error> { + if let Event::Log(log_event) = event { + let syslog_message = ConfigDecanter::new(&log_event).decant_config(&self.config.syslog); + let vec = syslog_message + .encode(&self.config.syslog.rfc) + .as_bytes() + .to_vec(); + buffer.put_slice(&vec); + } + + Ok(()) + } +} + +struct ConfigDecanter<'a> { + log: &'a LogEvent, +} + +impl<'a> ConfigDecanter<'a> { + fn new(log: &'a LogEvent) -> Self { + Self { log } + } + + fn decant_config(&self, config: &SyslogSerializerOptions) -> SyslogMessage { + let mut app_name = self + .get_value(&config.app_name) // P1: Configured path + .unwrap_or_else(|| { + // P2: Semantic Fallback: Check for the field designated as "service" in the schema + self.log + .get_by_meaning("service") + .map(|v| v.to_string_lossy().to_string()) + // P3: Hardcoded default + .unwrap_or_else(|| "vector".to_owned()) + }); + let mut proc_id = self.get_value(&config.proc_id); + let mut msg_id = self.get_value(&config.msg_id); + if config.rfc == SyslogRFC::Rfc5424 { + if app_name.len() > 48 { + app_name.truncate(48); + } + if let Some(pid) = &mut proc_id + && pid.len() > 128 + { + pid.truncate(128); + } + if let Some(mid) = &mut msg_id + && mid.len() > 32 + { + mid.truncate(32); + } + } + + SyslogMessage { + pri: Pri { + facility: self.get_facility(config), + severity: self.get_severity(config), + }, + timestamp: self.get_timestamp(), + hostname: self.log.get_host().map(|v| v.to_string_lossy().to_string()), + tag: Tag { + app_name, + proc_id, + msg_id, + }, + structured_data: self.get_structured_data(), + message: self.get_payload(), + } + } + + fn get_value(&self, path: &Option) -> Option { + path.as_ref() + .and_then(|p| self.log.get(p).cloned()) + .map(|v| v.to_string_lossy().to_string()) + } + + fn get_structured_data(&self) -> Option { + self.log + .get("structured_data") + .and_then(|v| v.clone().into_object()) + .map(StructuredData::from) + } + + fn get_timestamp(&self) -> DateTime { + if let Some(Value::Timestamp(timestamp)) = self.log.get_timestamp() { + return *timestamp; + } + Utc::now() + } + + fn get_payload(&self) -> String { + self.log + .get_message() + .map(|v| v.to_string_lossy().to_string()) + .unwrap_or_default() + } + + fn get_facility(&self, config: &SyslogSerializerOptions) -> Facility { + config.facility.as_ref().map_or(Facility::User, |path| { + self.get_syslog_code(path, Facility::from_repr, Facility::User) + }) + } + + fn get_severity(&self, config: &SyslogSerializerOptions) -> Severity { + config + .severity + .as_ref() + .map_or(Severity::Informational, |path| { + self.get_syslog_code(path, Severity::from_repr, Severity::Informational) + }) + } + + fn get_syslog_code( + &self, + path: &ConfigTargetPath, + from_repr_fn: fn(usize) -> Option, + default_value: T, + ) -> T + where + T: Copy + FromStr, + { + if let Some(value) = self.log.get(path).cloned() { + let s = value.to_string_lossy(); + if let Ok(val_from_name) = s.to_ascii_lowercase().parse::() { + return val_from_name; + } + if let Value::Integer(n) = value + && let Some(val_from_num) = from_repr_fn(n as usize) + { + return val_from_num; + } + } + default_value + } +} + +const NIL_VALUE: &str = "-"; +const SYSLOG_V1: &str = "1"; +const RFC3164_TAG_MAX_LENGTH: usize = 32; + +/// The syslog RFC standard to use for formatting. +#[configurable_component] +#[derive(PartialEq, Clone, Debug, Default)] +#[serde(rename_all = "snake_case")] +pub enum SyslogRFC { + /// The legacy RFC3164 syslog format. + Rfc3164, + /// The modern RFC5424 syslog format. + #[default] + Rfc5424, +} + +#[derive(Default, Debug)] +struct SyslogMessage { + pri: Pri, + timestamp: DateTime, + hostname: Option, + tag: Tag, + structured_data: Option, + message: String, +} + +impl SyslogMessage { + fn encode(&self, rfc: &SyslogRFC) -> String { + let pri_header = self.pri.encode(); + + let mut parts = Vec::new(); + + let timestamp_str = match rfc { + SyslogRFC::Rfc3164 => self.timestamp.format("%b %e %H:%M:%S").to_string(), + SyslogRFC::Rfc5424 => self + .timestamp + .round_subsecs(6) + .to_rfc3339_opts(SecondsFormat::Micros, true), + }; + parts.push(timestamp_str); + parts.push(self.hostname.as_deref().unwrap_or(NIL_VALUE).to_string()); + + let tag_str = match rfc { + SyslogRFC::Rfc3164 => self.tag.encode_rfc_3164(), + SyslogRFC::Rfc5424 => self.tag.encode_rfc_5424(), + }; + parts.push(tag_str); + + let mut message_part = self.message.clone(); + if *rfc == SyslogRFC::Rfc3164 { + message_part = Self::sanitize_rfc3164_message(&message_part); + } + + if let Some(sd) = &self.structured_data { + let sd_string = sd.encode(); + if *rfc == SyslogRFC::Rfc3164 { + if !sd.elements.is_empty() { + if !message_part.is_empty() { + message_part = format!("{sd_string} {message_part}"); + } else { + message_part = sd_string; + } + } + } else { + parts.push(sd_string); + } + } else if *rfc == SyslogRFC::Rfc5424 { + parts.push(NIL_VALUE.to_string()); + } + + if !message_part.is_empty() { + parts.push(message_part); + } + + let main_message = parts.join(" "); + + if *rfc == SyslogRFC::Rfc5424 { + format!("{pri_header}{SYSLOG_V1} {main_message}") + } else { + format!("{pri_header}{main_message}") + } + } + + fn sanitize_rfc3164_message(message: &str) -> String { + message + .chars() + .map(|ch| if (' '..='~').contains(&ch) { ch } else { ' ' }) + .collect() + } +} + +#[derive(Default, Debug)] +struct Tag { + app_name: String, + proc_id: Option, + msg_id: Option, +} + +impl Tag { + fn encode_rfc_3164(&self) -> String { + let mut tag = if let Some(proc_id) = self.proc_id.as_deref() { + format!("{}[{}]:", self.app_name, proc_id) + } else { + format!("{}:", self.app_name) + }; + if tag.len() > RFC3164_TAG_MAX_LENGTH { + tag.truncate(RFC3164_TAG_MAX_LENGTH); + if !tag.ends_with(':') { + tag.pop(); + tag.push(':'); + } + } + tag + } + + fn encode_rfc_5424(&self) -> String { + let proc_id_str = self.proc_id.as_deref().unwrap_or(NIL_VALUE); + let msg_id_str = self.msg_id.as_deref().unwrap_or(NIL_VALUE); + format!("{} {} {}", self.app_name, proc_id_str, msg_id_str) + } +} + +type StructuredDataMap = HashMap>; +#[derive(Debug, Default)] +struct StructuredData { + elements: StructuredDataMap, +} + +impl StructuredData { + fn encode(&self) -> String { + if self.elements.is_empty() { + NIL_VALUE.to_string() + } else { + self.elements + .iter() + .fold(String::new(), |mut acc, (sd_id, sd_params)| { + let _ = write!(acc, "[{sd_id}"); + for (key, value) in sd_params { + let esc_val = Self::escape_sd(value); + let _ = write!(acc, " {key}=\"{esc_val}\""); + } + let _ = write!(acc, "]"); + acc + }) + } + } + + fn escape_sd(s: &str) -> String { + s.replace('\\', "\\\\") + .replace('"', "\\\"") + .replace(']', "\\]") + } +} + +impl From for StructuredData { + fn from(fields: ObjectMap) -> Self { + let elements = fields + .into_iter() + .flat_map(|(sd_id, value)| { + let sd_params = value + .into_object()? + .into_iter() + .map(|(k, v)| (k.into(), v.to_string_lossy().to_string())) + .collect(); + Some((sd_id.into(), sd_params)) + }) + .collect(); + Self { elements } + } +} + +#[derive(Default, Debug)] +struct Pri { + facility: Facility, + severity: Severity, +} + +impl Pri { + // The last paragraph describes how to compose the enums into `PRIVAL`: + // https://datatracker.ietf.org/doc/html/rfc5424#section-6.2.1 + fn encode(&self) -> String { + let pri_val = (self.facility as u8 * 8) + self.severity as u8; + format!("<{pri_val}>") + } +} + +/// Syslog facility +#[derive(Default, Debug, EnumString, FromRepr, VariantNames, Copy, Clone, PartialEq, Eq)] +#[strum(serialize_all = "kebab-case")] +#[configurable_component] +pub enum Facility { + /// Kern + Kern = 0, + /// User + #[default] + User = 1, + /// Mail + Mail = 2, + /// Daemon + Daemon = 3, + /// Auth + Auth = 4, + /// Syslog + Syslog = 5, + /// Lpr + Lpr = 6, + /// News + News = 7, + /// Uucp + Uucp = 8, + /// Cron + Cron = 9, + /// Authpriv + Authpriv = 10, + /// Ftp + Ftp = 11, + /// Ntp + Ntp = 12, + /// Security + Security = 13, + /// Console + Console = 14, + /// SolarisCron + SolarisCron = 15, + /// Local0 + Local0 = 16, + /// Local1 + Local1 = 17, + /// Local2 + Local2 = 18, + /// Local3 + Local3 = 19, + /// Local4 + Local4 = 20, + /// Local5 + Local5 = 21, + /// Local6 + Local6 = 22, + /// Local7 + Local7 = 23, +} + +/// Syslog severity +#[derive(Default, Debug, EnumString, FromRepr, VariantNames, Copy, Clone, PartialEq, Eq)] +#[strum(serialize_all = "kebab-case")] +#[configurable_component] +pub enum Severity { + /// Emergency + Emergency = 0, + /// Alert + Alert = 1, + /// Critical + Critical = 2, + /// Error + Error = 3, + /// Warning + Warning = 4, + /// Notice + Notice = 5, + /// Informational + #[default] + Informational = 6, + /// Debug + Debug = 7, +} + +#[cfg(test)] +mod tests { + use super::*; + use bytes::BytesMut; + use chrono::NaiveDate; + use std::sync::Arc; + use vector_core::config::LogNamespace; + use vector_core::event::Event::Metric; + use vector_core::event::{Event, MetricKind, MetricValue, StatisticKind}; + use vrl::path::parse_target_path; + use vrl::prelude::Kind; + use vrl::{btreemap, event_path, value}; + + fn run_encode(config: SyslogSerializerConfig, event: Event) -> String { + let mut serializer = SyslogSerializer::new(&config); + let mut buffer = BytesMut::new(); + serializer.encode(event, &mut buffer).unwrap(); + String::from_utf8(buffer.to_vec()).unwrap() + } + + fn create_simple_log() -> LogEvent { + let mut log = LogEvent::from("original message"); + log.insert( + event_path!("timestamp"), + NaiveDate::from_ymd_opt(2025, 8, 28) + .unwrap() + .and_hms_micro_opt(18, 30, 00, 123456) + .unwrap() + .and_local_timezone(Utc) + .unwrap(), + ); + log.insert(event_path!("host"), "test-host.com"); + log + } + + fn create_test_log() -> LogEvent { + let mut log = create_simple_log(); + log.insert(event_path!("app"), "my-app"); + log.insert(event_path!("pid"), "12345"); + log.insert(event_path!("mid"), "req-abc-789"); + log.insert(event_path!("fac"), "daemon"); //3 + log.insert(event_path!("sev"), Value::from(2u8)); // Critical + log.insert( + event_path!("structured_data"), + value!({"metrics": {"retries": 3}}), + ); + log + } + + #[test] + fn test_rfc5424_defaults() { + let config = toml::from_str::( + r#" + [syslog] + rfc = "rfc5424" + "#, + ) + .unwrap(); + let log = create_simple_log(); + let output = run_encode(config, Event::Log(log)); + let expected = + "<14>1 2025-08-28T18:30:00.123456Z test-host.com vector - - - original message"; + assert_eq!(output, expected); + } + + #[test] + fn test_rfc5424_all_fields() { + let config = toml::from_str::( + r#" + [syslog] + app_name = ".app" + proc_id = ".pid" + msg_id = ".mid" + facility = ".fac" + severity = ".sev" + "#, + ) + .unwrap(); + let log = create_test_log(); + let output = run_encode(config, Event::Log(log)); + let expected = "<26>1 2025-08-28T18:30:00.123456Z test-host.com my-app 12345 req-abc-789 [metrics retries=\"3\"] original message"; + assert_eq!(output, expected); + } + + #[test] + fn test_rfc3164_all_fields() { + let config = toml::from_str::( + r#" + [syslog] + rfc = "rfc3164" + facility = ".fac" + severity = ".sev" + app_name = ".app" + proc_id = ".pid" + "#, + ) + .unwrap(); + let log = create_test_log(); + let output = run_encode(config, Event::Log(log)); + let expected = "<26>Aug 28 18:30:00 test-host.com my-app[12345]: [metrics retries=\"3\"] original message"; + assert_eq!(output, expected); + } + + #[test] + fn test_parsing_logic() { + let mut log = LogEvent::from("test message"); + let config_fac = + toml::from_str::(r#"facility = ".syslog_facility""#).unwrap(); + let config_sev = + toml::from_str::(r#"severity = ".syslog_severity""#).unwrap(); + //check lowercase and digit + log.insert(event_path!("syslog_facility"), "daemon"); + log.insert(event_path!("syslog_severity"), "critical"); + let decanter = ConfigDecanter::new(&log); + let facility = decanter.get_facility(&config_fac); + let severity = decanter.get_severity(&config_sev); + assert_eq!(facility, Facility::Daemon); + assert_eq!(severity, Severity::Critical); + + //check uppercase + log.insert(event_path!("syslog_facility"), "DAEMON"); + log.insert(event_path!("syslog_severity"), "CRITICAL"); + let decanter = ConfigDecanter::new(&log); + let facility = decanter.get_facility(&config_fac); + let severity = decanter.get_severity(&config_sev); + assert_eq!(facility, Facility::Daemon); + assert_eq!(severity, Severity::Critical); + + //check digit + log.insert(event_path!("syslog_facility"), Value::from(3u8)); + log.insert(event_path!("syslog_severity"), Value::from(2u8)); + let decanter = ConfigDecanter::new(&log); + let facility = decanter.get_facility(&config_fac); + let severity = decanter.get_severity(&config_sev); + assert_eq!(facility, Facility::Daemon); + assert_eq!(severity, Severity::Critical); + + //check defaults with empty config + let empty_config = + toml::from_str::(r#"facility = ".missing_field""#).unwrap(); + let default_facility = decanter.get_facility(&empty_config); + let default_severity = decanter.get_severity(&empty_config); + assert_eq!(default_facility, Facility::User); + assert_eq!(default_severity, Severity::Informational); + } + + #[test] + fn test_rfc3164_sanitization() { + let config = toml::from_str::( + r#" + [syslog] + rfc = "rfc3164" + "#, + ) + .unwrap(); + + let mut log = create_simple_log(); + log.insert( + event_path!("message"), + "A\nB\tC, Привіт D, E\u{0007}F", //newline, tab, unicode + ); + + let output = run_encode(config, Event::Log(log)); + let expected_message = "A B C, D, E F"; + assert!(output.ends_with(expected_message)); + } + + #[test] + fn test_rfc5424_field_truncation() { + let long_string = "vector".repeat(50); + + let mut log = create_simple_log(); + log.insert(event_path!("long_app_name"), long_string.clone()); + log.insert(event_path!("long_proc_id"), long_string.clone()); + log.insert(event_path!("long_msg_id"), long_string.clone()); + + let config = toml::from_str::( + r#" + [syslog] + rfc = "rfc5424" + app_name = ".long_app_name" + proc_id = ".long_proc_id" + msg_id = ".long_msg_id" + "#, + ) + .unwrap(); + + let decanter = ConfigDecanter::new(&log); + let message = decanter.decant_config(&config.syslog); + + assert_eq!(message.tag.app_name.len(), 48); + assert_eq!(message.tag.proc_id.unwrap().len(), 128); + assert_eq!(message.tag.msg_id.unwrap().len(), 32); + } + + #[test] + fn test_rfc3164_tag_truncation() { + let config = toml::from_str::( + r#" + [syslog] + rfc = "rfc3164" + facility = "user" + severity = "notice" + app_name = ".app_name" + proc_id = ".proc_id" + "#, + ) + .unwrap(); + + let mut log = create_simple_log(); + log.insert( + event_path!("app_name"), + "this-is-a-very-very-long-application-name", + ); + log.insert(event_path!("proc_id"), "1234567890"); + + let output = run_encode(config, Event::Log(log)); + let expected_tag = "this-is-a-very-very-long-applic:"; + assert!(output.contains(expected_tag)); + } + + #[test] + fn test_rfc5424_missing_fields() { + let config = toml::from_str::( + r#" + [syslog] + rfc = "rfc5424" + app_name = ".app" # configured path, but not in log + proc_id = ".pid" # configured path, but not in log + msg_id = ".mid" # configured path, but not in log + "#, + ) + .unwrap(); + + let log = create_simple_log(); + let output = run_encode(config, Event::Log(log)); + + let expected = + "<14>1 2025-08-28T18:30:00.123456Z test-host.com vector - - - original message"; + assert_eq!(output, expected); + } + + #[test] + fn test_invalid_parsing_fallback() { + let config = toml::from_str::( + r#" + [syslog] + rfc = "rfc5424" + facility = ".fac" + severity = ".sev" + "#, + ) + .unwrap(); + + let mut log = create_simple_log(); + + log.insert(event_path!("fac"), ""); + log.insert(event_path!("sev"), "invalid_severity_name"); + + let output = run_encode(config, Event::Log(log)); + + let expected_pri = "<14>"; + assert!(output.starts_with(expected_pri)); + + let expected_suffix = "vector - - - original message"; + assert!(output.ends_with(expected_suffix)); + } + + #[test] + fn test_rfc5424_empty_message_and_sd() { + let config = toml::from_str::( + r#" + [syslog] + rfc = "rfc5424" + app_name = ".app" + proc_id = ".pid" + msg_id = ".mid" + "#, + ) + .unwrap(); + + let mut log = create_simple_log(); + log.insert(event_path!("message"), ""); + log.insert(event_path!("structured_data"), value!({})); + + let output = run_encode(config, Event::Log(log)); + let expected = "<14>1 2025-08-28T18:30:00.123456Z test-host.com vector - - -"; + assert_eq!(output, expected); + } + + #[test] + fn test_non_log_event_filtering() { + let config = toml::from_str::( + r#" + [syslog] + rfc = "rfc5424" + "#, + ) + .unwrap(); + + let metric_event = Metric(vector_core::event::Metric::new( + "metric1", + MetricKind::Incremental, + MetricValue::Distribution { + samples: vector_core::samples![10.0 => 1], + statistic: StatisticKind::Histogram, + }, + )); + + let mut serializer = SyslogSerializer::new(&config); + let mut buffer = BytesMut::new(); + + let result = serializer.encode(metric_event, &mut buffer); + + assert!(result.is_ok()); + assert!(buffer.is_empty()); + } + + #[test] + fn test_minimal_event() { + let config = toml::from_str::( + r#" + [syslog] + "#, + ) + .unwrap(); + let log = LogEvent::from(""); + + let output = run_encode(config, Event::Log(log)); + let expected_suffix = "vector - - -"; + assert!(output.starts_with("<14>1")); + assert!(output.ends_with(expected_suffix)); + } + + #[test] + fn test_app_name_meaning_fallback() { + let config = toml::from_str::( + r#" + [syslog] + rfc = "rfc5424" + severity = ".sev" + app_name = ".nonexistent" + "#, + ) + .unwrap(); + + let mut log = LogEvent::default(); + log.insert("syslog.service", "meaning-app"); + + let schema = schema::Definition::new_with_default_metadata( + Kind::object(btreemap! { + "syslog" => Kind::object(btreemap! { + "service" => Kind::bytes(), + }) + }), + [LogNamespace::Vector], + ); + let schema = schema.with_meaning(parse_target_path("syslog.service").unwrap(), "service"); + let mut event = Event::from(log); + event + .metadata_mut() + .set_schema_definition(&Arc::new(schema)); + + let output = run_encode(config, event); + assert!(output.contains("meaning-app - -")); + } +} diff --git a/lib/codecs/src/encoding/mod.rs b/lib/codecs/src/encoding/mod.rs index 8dd2c4ddc79a5..5e3885d2752e4 100644 --- a/lib/codecs/src/encoding/mod.rs +++ b/lib/codecs/src/encoding/mod.rs @@ -2,10 +2,19 @@ //! events into bytes. pub mod chunking; +mod config; +mod encoder; pub mod format; pub mod framing; pub mod serializer; +mod transformer; pub use chunking::{Chunker, Chunking, GelfChunker}; +pub use config::{EncodingConfig, EncodingConfigWithFraming, SinkType}; +pub use encoder::{BatchEncoder, BatchSerializer, Encoder, EncoderKind}; +#[cfg(feature = "arrow")] +pub use format::{ + ArrowEncodingError, ArrowStreamSerializer, ArrowStreamSerializerConfig, SchemaProvider, +}; pub use format::{ AvroSerializer, AvroSerializerConfig, AvroSerializerOptions, CefSerializer, CefSerializerConfig, CsvSerializer, CsvSerializerConfig, GelfSerializer, GelfSerializerConfig, @@ -17,6 +26,8 @@ pub use format::{ }; #[cfg(feature = "opentelemetry")] pub use format::{OtlpSerializer, OtlpSerializerConfig}; +#[cfg(feature = "syslog")] +pub use format::{SyslogSerializer, SyslogSerializerConfig}; pub use framing::{ BoxedFramer, BoxedFramingError, BytesEncoder, BytesEncoderConfig, CharacterDelimitedEncoder, CharacterDelimitedEncoderConfig, CharacterDelimitedEncoderOptions, Framer, FramingConfig, @@ -24,18 +35,23 @@ pub use framing::{ NewlineDelimitedEncoderConfig, VarintLengthDelimitedEncoder, VarintLengthDelimitedEncoderConfig, }; +#[cfg(feature = "arrow")] +pub use serializer::BatchSerializerConfig; pub use serializer::{Serializer, SerializerConfig}; +pub use transformer::{TimestampFormat, Transformer}; /// An error that occurred while building an encoder. pub type BuildError = Box; -/// An error that occurred while encoding structured events into byte frames. +/// An error that occurred while encoding structured events. #[derive(Debug)] pub enum Error { /// The error occurred while encoding the byte frame boundaries. FramingError(BoxedFramingError), /// The error occurred while serializing a structured event into bytes. SerializingError(vector_common::Error), + /// A schema constraint was violated during encoding (e.g., null value for non-nullable field). + SchemaConstraintViolation(vector_common::Error), } impl std::fmt::Display for Error { @@ -43,6 +59,9 @@ impl std::fmt::Display for Error { match self { Self::FramingError(error) => write!(formatter, "FramingError({error})"), Self::SerializingError(error) => write!(formatter, "SerializingError({error})"), + Self::SchemaConstraintViolation(error) => { + write!(formatter, "SchemaConstraintViolation({error})") + } } } } diff --git a/lib/codecs/src/encoding/serializer.rs b/lib/codecs/src/encoding/serializer.rs index eef088fca4b72..8422587bddf28 100644 --- a/lib/codecs/src/encoding/serializer.rs +++ b/lib/codecs/src/encoding/serializer.rs @@ -4,20 +4,26 @@ use bytes::BytesMut; use vector_config::configurable_component; use vector_core::{config::DataType, event::Event, schema}; -use super::chunking::Chunker; -use super::format::{ - AvroSerializer, AvroSerializerConfig, AvroSerializerOptions, CefSerializer, - CefSerializerConfig, CsvSerializer, CsvSerializerConfig, GelfSerializer, GelfSerializerConfig, - JsonSerializer, JsonSerializerConfig, LogfmtSerializer, LogfmtSerializerConfig, - NativeJsonSerializer, NativeJsonSerializerConfig, NativeSerializer, NativeSerializerConfig, - ProtobufSerializer, ProtobufSerializerConfig, RawMessageSerializer, RawMessageSerializerConfig, - TextSerializer, TextSerializerConfig, -}; +#[cfg(feature = "arrow")] +use super::format::{ArrowStreamSerializer, ArrowStreamSerializerConfig}; #[cfg(feature = "opentelemetry")] use super::format::{OtlpSerializer, OtlpSerializerConfig}; -use super::framing::{ - CharacterDelimitedEncoderConfig, FramingConfig, LengthDelimitedEncoderConfig, - VarintLengthDelimitedEncoderConfig, +#[cfg(feature = "syslog")] +use super::format::{SyslogSerializer, SyslogSerializerConfig}; +use super::{ + chunking::Chunker, + format::{ + AvroSerializer, AvroSerializerConfig, AvroSerializerOptions, CefSerializer, + CefSerializerConfig, CsvSerializer, CsvSerializerConfig, GelfSerializer, + GelfSerializerConfig, JsonSerializer, JsonSerializerConfig, LogfmtSerializer, + LogfmtSerializerConfig, NativeJsonSerializer, NativeJsonSerializerConfig, NativeSerializer, + NativeSerializerConfig, ProtobufSerializer, ProtobufSerializerConfig, RawMessageSerializer, + RawMessageSerializerConfig, TextSerializer, TextSerializerConfig, + }, + framing::{ + CharacterDelimitedEncoderConfig, FramingConfig, LengthDelimitedEncoderConfig, + VarintLengthDelimitedEncoderConfig, + }, }; /// Serializer configuration. @@ -124,6 +130,11 @@ pub enum SerializerConfig { /// transform) and removing the message field while doing additional parsing on it, as this /// could lead to the encoding emitting empty strings for the given event. Text(TextSerializerConfig), + + /// Syslog encoding + /// RFC 3164 and 5424 are supported + #[cfg(feature = "syslog")] + Syslog(SyslogSerializerConfig), } impl Default for SerializerConfig { @@ -132,6 +143,53 @@ impl Default for SerializerConfig { } } +/// Batch serializer configuration. +#[configurable_component] +#[derive(Clone, Debug)] +#[serde(tag = "codec", rename_all = "snake_case")] +#[configurable(metadata( + docs::enum_tag_description = "The codec to use for batch encoding events." +))] +pub enum BatchSerializerConfig { + /// Encodes events in [Apache Arrow][apache_arrow] IPC streaming format. + /// + /// This is the streaming variant of the Arrow IPC format, which writes + /// a continuous stream of record batches. + /// + /// [apache_arrow]: https://arrow.apache.org/ + #[cfg(feature = "arrow")] + #[serde(rename = "arrow_stream")] + ArrowStream(ArrowStreamSerializerConfig), +} + +#[cfg(feature = "arrow")] +impl BatchSerializerConfig { + /// Build the `ArrowStreamSerializer` from this configuration. + pub fn build( + &self, + ) -> Result> { + match self { + BatchSerializerConfig::ArrowStream(arrow_config) => { + ArrowStreamSerializer::new(arrow_config.clone()) + } + } + } + + /// The data type of events that are accepted by this batch serializer. + pub fn input_type(&self) -> DataType { + match self { + BatchSerializerConfig::ArrowStream(arrow_config) => arrow_config.input_type(), + } + } + + /// The schema required by the batch serializer. + pub fn schema_requirement(&self) -> schema::Requirement { + match self { + BatchSerializerConfig::ArrowStream(arrow_config) => arrow_config.schema_requirement(), + } + } +} + impl From for SerializerConfig { fn from(config: AvroSerializerConfig) -> Self { Self::Avro { avro: config.avro } @@ -230,6 +288,8 @@ impl SerializerConfig { Ok(Serializer::RawMessage(RawMessageSerializerConfig.build())) } SerializerConfig::Text(config) => Ok(Serializer::Text(config.build())), + #[cfg(feature = "syslog")] + SerializerConfig::Syslog(config) => Ok(Serializer::Syslog(config.build())), } } @@ -262,6 +322,8 @@ impl SerializerConfig { | SerializerConfig::NativeJson | SerializerConfig::RawMessage | SerializerConfig::Text(_) => FramingConfig::NewlineDelimited, + #[cfg(feature = "syslog")] + SerializerConfig::Syslog(_) => FramingConfig::NewlineDelimited, SerializerConfig::Gelf(_) => { FramingConfig::CharacterDelimited(CharacterDelimitedEncoderConfig::new(0)) } @@ -286,6 +348,8 @@ impl SerializerConfig { SerializerConfig::Protobuf(config) => config.input_type(), SerializerConfig::RawMessage => RawMessageSerializerConfig.input_type(), SerializerConfig::Text(config) => config.input_type(), + #[cfg(feature = "syslog")] + SerializerConfig::Syslog(config) => config.input_type(), } } @@ -307,6 +371,8 @@ impl SerializerConfig { SerializerConfig::Protobuf(config) => config.schema_requirement(), SerializerConfig::RawMessage => RawMessageSerializerConfig.schema_requirement(), SerializerConfig::Text(config) => config.schema_requirement(), + #[cfg(feature = "syslog")] + SerializerConfig::Syslog(config) => config.schema_requirement(), } } } @@ -339,6 +405,9 @@ pub enum Serializer { RawMessage(RawMessageSerializer), /// Uses a `TextSerializer` for serialization. Text(TextSerializer), + /// Uses a `SyslogSerializer` for serialization. + #[cfg(feature = "syslog")] + Syslog(SyslogSerializer), } impl Serializer { @@ -354,6 +423,8 @@ impl Serializer { | Serializer::Native(_) | Serializer::Protobuf(_) | Serializer::RawMessage(_) => false, + #[cfg(feature = "syslog")] + Serializer::Syslog(_) => false, #[cfg(feature = "opentelemetry")] Serializer::Otlp(_) => false, } @@ -380,6 +451,10 @@ impl Serializer { | Serializer::RawMessage(_) => { panic!("Serializer does not support JSON") } + #[cfg(feature = "syslog")] + Serializer::Syslog(_) => { + panic!("Serializer does not support JSON") + } #[cfg(feature = "opentelemetry")] Serializer::Otlp(_) => { panic!("Serializer does not support JSON") @@ -407,6 +482,8 @@ impl Serializer { | Serializer::Protobuf(_) => true, #[cfg(feature = "opentelemetry")] Serializer::Otlp(_) => true, + #[cfg(feature = "syslog")] + Serializer::Syslog(_) => false, Serializer::Cef(_) | Serializer::Csv(_) | Serializer::Logfmt(_) @@ -490,6 +567,12 @@ impl From for Serializer { Self::Text(serializer) } } +#[cfg(feature = "syslog")] +impl From for Serializer { + fn from(serializer: SyslogSerializer) -> Self { + Self::Syslog(serializer) + } +} impl tokio_util::codec::Encoder for Serializer { type Error = vector_common::Error; @@ -509,6 +592,8 @@ impl tokio_util::codec::Encoder for Serializer { Serializer::Protobuf(serializer) => serializer.encode(event, buffer), Serializer::RawMessage(serializer) => serializer.encode(event, buffer), Serializer::Text(serializer) => serializer.encode(event, buffer), + #[cfg(feature = "syslog")] + Serializer::Syslog(serializer) => serializer.encode(event, buffer), } } } diff --git a/src/codecs/encoding/transformer.rs b/lib/codecs/src/encoding/transformer.rs similarity index 95% rename from src/codecs/encoding/transformer.rs rename to lib/codecs/src/encoding/transformer.rs index 65304989af291..2fb84f00e6fd1 100644 --- a/src/codecs/encoding/transformer.rs +++ b/lib/codecs/src/encoding/transformer.rs @@ -1,21 +1,19 @@ #![deny(missing_docs)] -use core::fmt::Debug; use std::collections::BTreeMap; use chrono::{DateTime, Utc}; +use lookup::{PathPrefix, event_path, lookup_v2::ConfigValuePath}; use ordered_float::NotNan; use serde::{Deserialize, Deserializer}; -use vector_lib::{ - configurable::configurable_component, - event::{LogEvent, MaybeAsLogMut}, - lookup::{PathPrefix, event_path, lookup_v2::ConfigValuePath}, +use vector_config::configurable_component; +use vector_core::{ + event::{Event, LogEvent, MaybeAsLogMut}, schema::meaning, + serde::is_default, }; use vrl::{path::OwnedValuePath, value::Value}; -use crate::{event::Event, serde::is_default}; - /// Transformations to prepare an event for serialization. #[configurable_component(no_deser)] #[derive(Clone, Debug, Default, PartialEq, Eq)] @@ -72,7 +70,7 @@ impl Transformer { only_fields: Option>, except_fields: Option>, timestamp_format: Option, - ) -> Result { + ) -> vector_common::Result { Self::validate_fields(only_fields.as_ref(), except_fields.as_ref())?; Ok(Self { @@ -83,7 +81,7 @@ impl Transformer { } /// Get the `Transformer`'s `only_fields`. - #[cfg(test)] + #[cfg(any(test, feature = "test"))] pub const fn only_fields(&self) -> &Option> { &self.only_fields } @@ -104,7 +102,7 @@ impl Transformer { fn validate_fields( only_fields: Option<&Vec>, except_fields: Option<&Vec>, - ) -> crate::Result<()> { + ) -> vector_common::Result<()> { if let (Some(only_fields), Some(except_fields)) = (only_fields, except_fields) && except_fields .iter() @@ -188,7 +186,8 @@ impl Transformer { } } for (k, v) in unix_timestamps { - log.parse_path_and_insert(k, v).unwrap(); + log.parse_path_and_insert(k, v) + .expect("timestamp fields must allow insertion"); } } else { // root is not an object @@ -213,7 +212,8 @@ impl Transformer { ts.timestamp_nanos_opt().expect("Timestamp out of range") }), TimestampFormat::UnixFloat => self.format_timestamps(log, |ts| { - NotNan::new(ts.timestamp_micros() as f64 / 1e6).unwrap() + NotNan::new(ts.timestamp_micros() as f64 / 1e6) + .expect("this division will never produce a NaN") }), // RFC3339 is the default serialization of a timestamp. TimestampFormat::Rfc3339 => (), @@ -225,11 +225,11 @@ impl Transformer { /// /// Returns `Err` if the new `except_fields` fail validation, i.e. are not mutually exclusive /// with `only_fields`. - #[cfg(test)] + #[cfg(any(test, feature = "test"))] pub fn set_except_fields( &mut self, except_fields: Option>, - ) -> crate::Result<()> { + ) -> vector_common::Result<()> { Self::validate_fields(self.only_fields.as_ref(), except_fields.as_ref())?; self.except_fields = except_fields; Ok(()) @@ -265,15 +265,14 @@ mod tests { use std::{collections::BTreeMap, sync::Arc}; use indoc::indoc; - use vector_lib::{ - btreemap, + use lookup::path::parse_target_path; + use vector_core::{ config::{LogNamespace, log_schema}, - lookup::path::parse_target_path, + schema, }; - use vrl::value::Kind; + use vrl::{btreemap, value::Kind}; use super::*; - use crate::config::schema; #[test] fn serialize() { @@ -450,7 +449,7 @@ mod tests { Kind::object(btreemap! { "thing" => Kind::object(btreemap! { "service" => Kind::bytes(), - }) + }), }), [LogNamespace::Vector], ); @@ -490,7 +489,7 @@ mod tests { Kind::object(btreemap! { "thing" => Kind::object(btreemap! { "service" => Kind::bytes(), - }) + }), }), [LogNamespace::Vector], ); diff --git a/lib/codecs/src/gelf.rs b/lib/codecs/src/gelf.rs index d4ddc79267359..f97d1c0863d2e 100644 --- a/lib/codecs/src/gelf.rs +++ b/lib/codecs/src/gelf.rs @@ -73,4 +73,4 @@ pub(crate) static GELF_TARGET_PATHS: LazyLock = /// As Graylog itself will produce GELF with any existing field names on the Graylog GELF Output, /// vector is more lenient, too, at least allowing the additional `@` character. pub static VALID_FIELD_REGEX: LazyLock = - LazyLock::new(|| Regex::new(r"^[\w\.\-@]*$").unwrap()); + LazyLock::new(|| Regex::new(r"^[\w\.\-@]*$").expect("valid regex pattern")); diff --git a/src/internal_events/codecs.rs b/lib/codecs/src/internal_events.rs similarity index 55% rename from src/internal_events/codecs.rs rename to lib/codecs/src/internal_events.rs index a2c41b7806be1..134fee16ecf8b 100644 --- a/src/internal_events/codecs.rs +++ b/lib/codecs/src/internal_events.rs @@ -1,10 +1,16 @@ +//! Internal events for codecs. + use metrics::counter; -use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, +use tracing::error; +use vector_common::internal_event::{ + ComponentEventsDropped, InternalEvent, UNINTENTIONAL, emit, error_stage, error_type, }; +use vector_common_macros::NamedInternalEvent; -#[derive(Debug)] +#[derive(Debug, NamedInternalEvent)] +/// Emitted when a decoder framing error occurs. pub struct DecoderFramingError { + /// The framing error that occurred. pub error: E, } @@ -27,9 +33,11 @@ impl InternalEvent for DecoderFramingError { } } -#[derive(Debug)] +#[derive(Debug, NamedInternalEvent)] +/// Emitted when a decoder fails to deserialize a frame. pub struct DecoderDeserializeError<'a> { - pub error: &'a crate::Error, + /// The deserialize error that occurred. + pub error: &'a vector_common::Error, } impl InternalEvent for DecoderDeserializeError<'_> { @@ -51,9 +59,11 @@ impl InternalEvent for DecoderDeserializeError<'_> { } } -#[derive(Debug)] +#[derive(Debug, NamedInternalEvent)] +/// Emitted when an encoder framing error occurs. pub struct EncoderFramingError<'a> { - pub error: &'a vector_lib::codecs::encoding::BoxedFramingError, + /// The framing error that occurred. + pub error: &'a crate::encoding::BoxedFramingError, } impl InternalEvent for EncoderFramingError<'_> { @@ -73,20 +83,22 @@ impl InternalEvent for EncoderFramingError<'_> { "stage" => error_stage::SENDING, ) .increment(1); - emit!(ComponentEventsDropped:: { count: 1, reason }); + emit(ComponentEventsDropped:: { count: 1, reason }); } } -#[derive(Debug)] +#[derive(Debug, NamedInternalEvent)] +/// Emitted when an encoder fails to serialize a frame. pub struct EncoderSerializeError<'a> { - pub error: &'a crate::Error, + /// The serialization error that occurred. + pub error: &'a vector_common::Error, } impl InternalEvent for EncoderSerializeError<'_> { fn emit(self) { - let reason = "Failed serializing frame."; + const SERIALIZE_REASON: &str = "Failed serializing frame."; error!( - message = reason, + message = SERIALIZE_REASON, error = %self.error, error_code = "encoder_serialize", error_type = error_type::ENCODER_FAILED, @@ -99,13 +111,19 @@ impl InternalEvent for EncoderSerializeError<'_> { "stage" => error_stage::SENDING, ) .increment(1); - emit!(ComponentEventsDropped:: { count: 1, reason }); + emit(ComponentEventsDropped:: { + count: 1, + reason: SERIALIZE_REASON, + }); } } -#[derive(Debug)] +#[derive(Debug, NamedInternalEvent)] +/// Emitted when writing encoded bytes fails. pub struct EncoderWriteError<'a, E> { + /// The write error that occurred. pub error: &'a E, + /// The number of events dropped by the failed write. pub count: usize, } @@ -125,10 +143,43 @@ impl InternalEvent for EncoderWriteError<'_, E> { ) .increment(1); if self.count > 0 { - emit!(ComponentEventsDropped:: { + emit(ComponentEventsDropped:: { count: self.count, reason, }); } } } + +#[cfg(feature = "arrow")] +#[derive(Debug, NamedInternalEvent)] +/// Emitted when encoding violates a schema constraint. +pub struct EncoderNullConstraintError<'a> { + /// The schema constraint error that occurred. + pub error: &'a vector_common::Error, +} + +#[cfg(feature = "arrow")] +impl InternalEvent for EncoderNullConstraintError<'_> { + fn emit(self) { + const CONSTRAINT_REASON: &str = "Schema constraint violation."; + error!( + message = CONSTRAINT_REASON, + error = %self.error, + error_code = "encoding_null_constraint", + error_type = error_type::ENCODER_FAILED, + stage = error_stage::SENDING, + ); + counter!( + "component_errors_total", + "error_code" => "encoding_null_constraint", + "error_type" => error_type::ENCODER_FAILED, + "stage" => error_stage::SENDING, + ) + .increment(1); + emit(ComponentEventsDropped:: { + count: 1, + reason: CONSTRAINT_REASON, + }); + } +} diff --git a/lib/codecs/src/lib.rs b/lib/codecs/src/lib.rs index e386bb30f378a..c88eb531dc1a7 100644 --- a/lib/codecs/src/lib.rs +++ b/lib/codecs/src/lib.rs @@ -8,27 +8,33 @@ mod common; pub mod decoding; pub mod encoding; pub mod gelf; +pub mod internal_events; +mod ready_frames; pub use decoding::{ BytesDecoder, BytesDecoderConfig, BytesDeserializer, BytesDeserializerConfig, - CharacterDelimitedDecoder, CharacterDelimitedDecoderConfig, GelfDeserializer, - GelfDeserializerConfig, JsonDeserializer, JsonDeserializerConfig, LengthDelimitedDecoder, - LengthDelimitedDecoderConfig, NativeDeserializer, NativeDeserializerConfig, - NativeJsonDeserializer, NativeJsonDeserializerConfig, NewlineDelimitedDecoder, - NewlineDelimitedDecoderConfig, OctetCountingDecoder, OctetCountingDecoderConfig, - StreamDecodingError, VarintLengthDelimitedDecoder, VarintLengthDelimitedDecoderConfig, + CharacterDelimitedDecoder, CharacterDelimitedDecoderConfig, Decoder, DecodingConfig, + GelfDeserializer, GelfDeserializerConfig, JsonDeserializer, JsonDeserializerConfig, + LengthDelimitedDecoder, LengthDelimitedDecoderConfig, NativeDeserializer, + NativeDeserializerConfig, NativeJsonDeserializer, NativeJsonDeserializerConfig, + NewlineDelimitedDecoder, NewlineDelimitedDecoderConfig, OctetCountingDecoder, + OctetCountingDecoderConfig, StreamDecodingError, VarintLengthDelimitedDecoder, + VarintLengthDelimitedDecoderConfig, }; #[cfg(feature = "syslog")] pub use decoding::{SyslogDeserializer, SyslogDeserializerConfig}; pub use encoding::{ - BytesEncoder, BytesEncoderConfig, CharacterDelimitedEncoder, CharacterDelimitedEncoderConfig, - CsvSerializer, CsvSerializerConfig, GelfSerializer, GelfSerializerConfig, JsonSerializer, - JsonSerializerConfig, LengthDelimitedEncoder, LengthDelimitedEncoderConfig, LogfmtSerializer, - LogfmtSerializerConfig, NativeJsonSerializer, NativeJsonSerializerConfig, NativeSerializer, - NativeSerializerConfig, NewlineDelimitedEncoder, NewlineDelimitedEncoderConfig, - RawMessageSerializer, RawMessageSerializerConfig, TextSerializer, TextSerializerConfig, + BatchEncoder, BatchSerializer, BytesEncoder, BytesEncoderConfig, CharacterDelimitedEncoder, + CharacterDelimitedEncoderConfig, CsvSerializer, CsvSerializerConfig, Encoder, EncoderKind, + EncodingConfig, EncodingConfigWithFraming, GelfSerializer, GelfSerializerConfig, + JsonSerializer, JsonSerializerConfig, LengthDelimitedEncoder, LengthDelimitedEncoderConfig, + LogfmtSerializer, LogfmtSerializerConfig, NativeJsonSerializer, NativeJsonSerializerConfig, + NativeSerializer, NativeSerializerConfig, NewlineDelimitedEncoder, + NewlineDelimitedEncoderConfig, RawMessageSerializer, RawMessageSerializerConfig, SinkType, + TextSerializer, TextSerializerConfig, TimestampFormat, Transformer, }; pub use gelf::{VALID_FIELD_REGEX, gelf_fields}; +pub use ready_frames::ReadyFrames; use vector_config_macros::configurable_component; /// The user configuration to choose the metric tag strategy. diff --git a/src/codecs/ready_frames.rs b/lib/codecs/src/ready_frames.rs similarity index 100% rename from src/codecs/ready_frames.rs rename to lib/codecs/src/ready_frames.rs diff --git a/lib/codecs/tests/avro.rs b/lib/codecs/tests/avro.rs index 4afa403c616de..fff274706d7d9 100644 --- a/lib/codecs/tests/avro.rs +++ b/lib/codecs/tests/avro.rs @@ -1,3 +1,5 @@ +#![allow(clippy::unwrap_used)] + use std::{ fs::File, io::Read, @@ -33,7 +35,9 @@ fn roundtrip_avro_fixtures( fn roundtrip_avro(data_path: PathBuf, schema_path: PathBuf, reserialize: bool) { let schema = load_file(&schema_path); let schema = from_utf8(&schema).unwrap().to_string(); - let deserializer = AvroDeserializerConfig::new(schema.clone(), false).build(); + let deserializer = AvroDeserializerConfig::new(schema.clone(), false) + .build() + .unwrap(); let mut serializer = AvroSerializerConfig::new(schema.clone()).build().unwrap(); let (buf, event) = load_deserialize(&data_path, &deserializer); diff --git a/lib/codecs/tests/bin/generate-avro-fixtures.rs b/lib/codecs/tests/bin/generate-avro-fixtures.rs index d8797cac43dcc..29f22ef0991c0 100644 --- a/lib/codecs/tests/bin/generate-avro-fixtures.rs +++ b/lib/codecs/tests/bin/generate-avro-fixtures.rs @@ -1,11 +1,12 @@ use std::{fs::File, io::Write, path::PathBuf}; +use vector_common::Result; use apache_avro::{Decimal, Schema, types::Value}; use serde::{Deserialize, Serialize}; const FIXTURES_PATH: &str = "lib/codecs/tests/data/avro/generated"; -fn generate_avro_test_case_boolean() { +fn generate_avro_test_case_boolean() -> Result<()> { let schema = r#" { "type": "record", @@ -20,10 +21,10 @@ fn generate_avro_test_case_boolean() { bool_field: bool, } let value = Test { bool_field: true }; - generate_test_case(schema, value, "boolean"); + generate_test_case(schema, value, "boolean") } -fn generate_avro_test_case_int() { +fn generate_avro_test_case_int() -> Result<()> { let schema = r#" { "type": "record", @@ -38,10 +39,10 @@ fn generate_avro_test_case_int() { int_field: i32, } let value = Test { int_field: 1234 }; - generate_test_case(schema, value, "int"); + generate_test_case(schema, value, "int") } -fn generate_avro_test_case_long() { +fn generate_avro_test_case_long() -> Result<()> { let schema = r#" { "type": "record", @@ -58,10 +59,10 @@ fn generate_avro_test_case_long() { let value = Test { long_field: 42949672960i64, }; - generate_test_case(schema, value, "long"); + generate_test_case(schema, value, "long") } -fn generate_avro_test_case_float() { +fn generate_avro_test_case_float() -> Result<()> { let schema = r#" { "type": "record", @@ -78,10 +79,10 @@ fn generate_avro_test_case_float() { let value = Test { float_field: 123.456, }; - generate_test_case(schema, value, "float"); + generate_test_case(schema, value, "float") } -fn generate_avro_test_case_double() { +fn generate_avro_test_case_double() -> Result<()> { let schema = r#" { "type": "record", @@ -98,10 +99,10 @@ fn generate_avro_test_case_double() { let value = Test { double_field: 123.456f64, }; - generate_test_case(schema, value, "double"); + generate_test_case(schema, value, "double") } -fn generate_avro_test_case_bytes() { +fn generate_avro_test_case_bytes() -> Result<()> { let schema = r#" { "type": "record", @@ -118,10 +119,10 @@ fn generate_avro_test_case_bytes() { let value = Test { bytes_field: vec![1, 2, 3, 4, 5, 6, 6, 7], }; - generate_test_case(schema, value, "bytes"); + generate_test_case(schema, value, "bytes") } -fn generate_avro_test_case_string() { +fn generate_avro_test_case_string() -> Result<()> { let schema = r#" { "type": "record", @@ -138,11 +139,11 @@ fn generate_avro_test_case_string() { let value = Test { string_field: "hello world!".to_string(), }; - generate_test_case(schema, value, "string"); + generate_test_case(schema, value, "string") } #[allow(unused)] -fn generate_avro_test_case_fixed() { +fn generate_avro_test_case_fixed() -> Result<()> { let schema = r#" { "type": "record", @@ -156,10 +157,10 @@ fn generate_avro_test_case_fixed() { "fixed_field".into(), Value::Fixed(16, b"1019181716151413".to_vec()), )]); - generate_test_case_from_value(schema, record, "fixed"); + generate_test_case_from_value(schema, record, "fixed") } -fn generate_avro_test_case_enum() { +fn generate_avro_test_case_enum() -> Result<()> { let schema = r#" { "type": "record", @@ -183,10 +184,10 @@ fn generate_avro_test_case_enum() { let value = Test { enum_field: Value::Hearts, }; - generate_test_case(schema, value, "enum"); + generate_test_case(schema, value, "enum") } -fn generate_avro_test_case_union() { +fn generate_avro_test_case_union() -> Result<()> { let schema = r#" { "type": "record", @@ -207,10 +208,10 @@ fn generate_avro_test_case_union() { let value = Test { union_field: 123456, }; - generate_test_case(schema, value, "union"); + generate_test_case(schema, value, "union") } -fn generate_avro_test_case_array() { +fn generate_avro_test_case_array() -> Result<()> { let schema = r#" { "type": "record", @@ -232,10 +233,10 @@ fn generate_avro_test_case_array() { "codec".to_string(), ], }; - generate_test_case(schema, value, "array"); + generate_test_case(schema, value, "array") } -fn generate_avro_test_case_map() { +fn generate_avro_test_case_map() -> Result<()> { let schema = r#" { "type": "record", @@ -253,10 +254,10 @@ fn generate_avro_test_case_map() { let mut scores = HashMap::new(); scores.insert(String::from("Blue"), 10i64); let value = Test { map_field: scores }; - generate_test_case(schema, value, "map"); + generate_test_case(schema, value, "map") } -fn generate_avro_test_case_record() { +fn generate_avro_test_case_record() -> Result<()> { let schema = r#" { "type": "record", @@ -276,11 +277,11 @@ fn generate_avro_test_case_record() { name: "John".to_string(), age: 23, }; - generate_test_case(schema, value, "record"); + generate_test_case(schema, value, "record") } #[allow(unused)] -fn generate_avro_test_case_date() { +fn generate_avro_test_case_date() -> Result<()> { let schema = r#" { "type": "record", @@ -295,11 +296,11 @@ fn generate_avro_test_case_date() { date_field: i32, } let value = Test { date_field: 19646 }; - generate_test_case(schema, value, "date"); + generate_test_case(schema, value, "date") } #[allow(unused)] -fn generate_avro_test_case_decimal_var() { +fn generate_avro_test_case_decimal_var() -> Result<()> { let schema = r#" { "type": "record", @@ -316,11 +317,11 @@ fn generate_avro_test_case_decimal_var() { 249, 33, 74, 206, 142, 64, 190, 170, 17, 153, ])), )]); - generate_test_case_from_value(schema, record, "decimal_var"); + generate_test_case_from_value(schema, record, "decimal_var") } #[allow(unused)] -fn generate_avro_test_case_time_millis() { +fn generate_avro_test_case_time_millis() -> Result<()> { let schema = r#" { "type": "record", @@ -337,10 +338,10 @@ fn generate_avro_test_case_time_millis() { let value = Test { time_millis_field: 59820123, }; - generate_test_case(schema, value, "time_millis"); + generate_test_case(schema, value, "time_millis") } -fn generate_avro_test_case_time_micros() { +fn generate_avro_test_case_time_micros() -> Result<()> { let schema = r#" { "type": "record", @@ -357,10 +358,10 @@ fn generate_avro_test_case_time_micros() { let value: Test = Test { time_micros_field: 59820123456i64, }; - generate_test_case(schema, value, "time_micros"); + generate_test_case(schema, value, "time_micros") } -fn generate_avro_test_case_timestamp_millis() { +fn generate_avro_test_case_timestamp_millis() -> Result<()> { let schema = r#" { "type": "record", @@ -377,10 +378,10 @@ fn generate_avro_test_case_timestamp_millis() { let value = Test { timestamp_millis_field: 1697445291056i64, }; - generate_test_case(schema, value, "timestamp_millis"); + generate_test_case(schema, value, "timestamp_millis") } -fn generate_avro_test_case_timestamp_micros() { +fn generate_avro_test_case_timestamp_micros() -> Result<()> { let schema = r#" { "type": "record", @@ -397,10 +398,10 @@ fn generate_avro_test_case_timestamp_micros() { let value = Test { timestamp_micros_field: 1697445291056567i64, }; - generate_test_case(schema, value, "timestamp_micros"); + generate_test_case(schema, value, "timestamp_micros") } -fn generate_avro_test_case_local_timestamp_millis() { +fn generate_avro_test_case_local_timestamp_millis() -> Result<()> { let schema = r#" { "type": "record", @@ -417,10 +418,10 @@ fn generate_avro_test_case_local_timestamp_millis() { let value = Test { local_timestamp_millis_field: 1697445291056i64, }; - generate_test_case(schema, value, "local-timestamp_millis"); + generate_test_case(schema, value, "local-timestamp_millis") } -fn generate_avro_test_case_local_timestamp_micros() { +fn generate_avro_test_case_local_timestamp_micros() -> Result<()> { let schema = r#" { "type": "record", @@ -437,10 +438,10 @@ fn generate_avro_test_case_local_timestamp_micros() { let value = Test { local_timestamp_micros_field: 1697445291056567i64, }; - generate_test_case(schema, value, "local-timestamp_micros"); + generate_test_case(schema, value, "local-timestamp_micros") } -fn generate_avro_test_case_uuid() { +fn generate_avro_test_case_uuid() -> Result<()> { let schema = r#" { "type": "record", @@ -459,48 +460,48 @@ fn generate_avro_test_case_uuid() { let value = Test { uuid_field: "550e8400-e29b-41d4-a716-446655440000".into(), }; - generate_test_case(schema, value, "uuid"); + generate_test_case(schema, value, "uuid") } -fn generate_test_case(schema: &str, value: S, filename: &str) { - let value = apache_avro::to_value(value).unwrap(); - generate_test_case_from_value(schema, value, filename); +fn generate_test_case(schema: &str, value: S, filename: &str) -> Result<()> { + let value = apache_avro::to_value(value)?; + generate_test_case_from_value(schema, value, filename) } -fn generate_test_case_from_value(schema: &str, value: Value, filename: &str) { - let schema = Schema::parse_str(schema).unwrap(); +fn generate_test_case_from_value(schema: &str, value: Value, filename: &str) -> Result<()> { + let schema = Schema::parse_str(schema)?; - let value = value.resolve(&schema).unwrap(); - let bytes = apache_avro::to_avro_datum(&schema, value).unwrap(); + let value = value.resolve(&schema)?; + let bytes = apache_avro::to_avro_datum(&schema, value)?; - let mut schema_file = File::create(format!("{FIXTURES_PATH}/{filename}.avsc")).unwrap(); - let mut avro_file = File::create(format!("{FIXTURES_PATH}/{filename}.avro")).unwrap(); - schema_file - .write_all(schema.canonical_form().as_bytes()) - .unwrap(); - avro_file.write_all(&bytes).unwrap(); + let mut schema_file = File::create(format!("{FIXTURES_PATH}/{filename}.avsc"))?; + let mut avro_file = File::create(format!("{FIXTURES_PATH}/{filename}.avro"))?; + schema_file.write_all(schema.canonical_form().as_bytes())?; + avro_file.write_all(&bytes)?; + Ok(()) } -fn main() { +fn main() -> Result<()> { if !PathBuf::from(FIXTURES_PATH).is_dir() { panic!("dir {FIXTURES_PATH} not exist\n"); } - generate_avro_test_case_array(); - generate_avro_test_case_boolean(); - generate_avro_test_case_bytes(); - generate_avro_test_case_double(); - generate_avro_test_case_enum(); - generate_avro_test_case_float(); - generate_avro_test_case_int(); - generate_avro_test_case_long(); - generate_avro_test_case_map(); - generate_avro_test_case_record(); - generate_avro_test_case_string(); - generate_avro_test_case_time_micros(); - generate_avro_test_case_timestamp_micros(); - generate_avro_test_case_timestamp_millis(); - generate_avro_test_case_local_timestamp_micros(); - generate_avro_test_case_local_timestamp_millis(); - generate_avro_test_case_union(); - generate_avro_test_case_uuid(); + generate_avro_test_case_array()?; + generate_avro_test_case_boolean()?; + generate_avro_test_case_bytes()?; + generate_avro_test_case_double()?; + generate_avro_test_case_enum()?; + generate_avro_test_case_float()?; + generate_avro_test_case_int()?; + generate_avro_test_case_long()?; + generate_avro_test_case_map()?; + generate_avro_test_case_record()?; + generate_avro_test_case_string()?; + generate_avro_test_case_time_micros()?; + generate_avro_test_case_timestamp_micros()?; + generate_avro_test_case_timestamp_millis()?; + generate_avro_test_case_local_timestamp_micros()?; + generate_avro_test_case_local_timestamp_millis()?; + generate_avro_test_case_union()?; + generate_avro_test_case_uuid()?; + Ok(()) } diff --git a/lib/codecs/tests/native.rs b/lib/codecs/tests/native.rs index 4f6d61049a38c..d0c6329c35090 100644 --- a/lib/codecs/tests/native.rs +++ b/lib/codecs/tests/native.rs @@ -1,3 +1,5 @@ +#![allow(clippy::unwrap_used)] + use std::{ fs::{self, File}, io::{Read, Write}, diff --git a/lib/codecs/tests/native_json.rs b/lib/codecs/tests/native_json.rs index 4bcf31982aee8..48db364f0a987 100644 --- a/lib/codecs/tests/native_json.rs +++ b/lib/codecs/tests/native_json.rs @@ -1,3 +1,5 @@ +#![allow(clippy::unwrap_used)] + use bytes::BytesMut; use codecs::{ NativeJsonDeserializerConfig, NativeJsonSerializerConfig, decoding::format::Deserializer, diff --git a/lib/codecs/tests/protobuf.rs b/lib/codecs/tests/protobuf.rs index 1f56e7695c6b4..87bc8ee394880 100644 --- a/lib/codecs/tests/protobuf.rs +++ b/lib/codecs/tests/protobuf.rs @@ -1,5 +1,7 @@ //! Tests for the behaviour of Protobuf serializer and deserializer (together). +#![allow(clippy::unwrap_used)] + use std::path::{Path, PathBuf}; use bytes::{Bytes, BytesMut}; diff --git a/lib/codecs/tests/varint_framing.rs b/lib/codecs/tests/varint_framing.rs index 5fd9cc333f2ca..3fd5fba70756a 100644 --- a/lib/codecs/tests/varint_framing.rs +++ b/lib/codecs/tests/varint_framing.rs @@ -1,3 +1,5 @@ +#![allow(clippy::unwrap_used)] + use bytes::BytesMut; use codecs::{ VarintLengthDelimitedDecoder, VarintLengthDelimitedDecoderConfig, diff --git a/lib/dnsmsg-parser/Cargo.toml b/lib/dnsmsg-parser/Cargo.toml index 5dd9051955b05..b34623c4352a0 100644 --- a/lib/dnsmsg-parser/Cargo.toml +++ b/lib/dnsmsg-parser/Cargo.toml @@ -6,6 +6,9 @@ edition = "2024" publish = false license = "MIT" +[lints.clippy] +unwrap-used = "forbid" + [dependencies] data-encoding = "2.9" hickory-proto.workspace = true diff --git a/lib/dnsmsg-parser/benches/benches.rs b/lib/dnsmsg-parser/benches/benches.rs index 29247accdea53..f307aef7e1463 100644 --- a/lib/dnsmsg-parser/benches/benches.rs +++ b/lib/dnsmsg-parser/benches/benches.rs @@ -5,14 +5,20 @@ use hickory_proto::rr::rdata::NULL; fn benchmark_parse_as_query_message(c: &mut Criterion) { let raw_dns_message = "szgAAAABAAAAAAAAAmg1B2V4YW1wbGUDY29tAAAGAAE="; - let raw_query_message = BASE64.decode(raw_dns_message.as_bytes()).unwrap(); + let raw_query_message = BASE64 + .decode(raw_dns_message.as_bytes()) + .expect("invalid base64"); let mut group = c.benchmark_group("dnstap"); group.throughput(Throughput::Bytes(raw_query_message.len() as u64)); group.bench_function("parse_as_query_message", |b| { b.iter_batched( || DnsMessageParser::new(raw_query_message.clone()), - |mut parser| parser.parse_as_query_message().unwrap(), + |mut parser| { + parser + .parse_as_query_message() + .expect("failed to parse as query") + }, BatchSize::SmallInput, ) }); @@ -22,14 +28,20 @@ fn benchmark_parse_as_query_message(c: &mut Criterion) { fn benchmark_parse_as_update_message(c: &mut Criterion) { let raw_dns_message = "xjUoAAABAAAAAQAAB2V4YW1wbGUDY29tAAAGAAECaDXADAD/AP8AAAAAAAA="; - let raw_update_message = BASE64.decode(raw_dns_message.as_bytes()).unwrap(); + let raw_update_message = BASE64 + .decode(raw_dns_message.as_bytes()) + .expect("invalid base64"); let mut group = c.benchmark_group("dnstap"); group.throughput(Throughput::Bytes(raw_update_message.len() as u64)); group.bench_function("parse_as_update_message", |b| { b.iter_batched( || DnsMessageParser::new(raw_update_message.clone()), - |mut parser| parser.parse_as_update_message().unwrap(), + |mut parser| { + parser + .parse_as_update_message() + .expect("failed to parse as update") + }, BatchSize::SmallInput, ) }); @@ -59,7 +71,7 @@ fn benchmark_parse_apl_rdata(c: &mut Criterion) { } fn benchmark_parse_rdata(c: &mut Criterion, data: &str, code: u16, id: &str) { - let raw_rdata = BASE64.decode(data.as_bytes()).unwrap(); + let raw_rdata = BASE64.decode(data.as_bytes()).expect("invalid base64"); let record_rdata = NULL::with(raw_rdata.clone()); @@ -73,7 +85,11 @@ fn benchmark_parse_rdata(c: &mut Criterion, data: &str, code: u16, id: &str) { DnsMessageParser::new(Vec::::new()), ) }, - |(record_rdata, mut parser)| parser.format_unknown_rdata(code, &record_rdata).unwrap(), + |(record_rdata, mut parser)| { + parser + .format_unknown_rdata(code, &record_rdata) + .expect("failed to parse rdata") + }, BatchSize::SmallInput, ) }); diff --git a/lib/dnsmsg-parser/src/dns_message_parser.rs b/lib/dnsmsg-parser/src/dns_message_parser.rs index cf57171b27b67..d0b271b2e7930 100644 --- a/lib/dnsmsg-parser/src/dns_message_parser.rs +++ b/lib/dnsmsg-parser/src/dns_message_parser.rs @@ -219,7 +219,12 @@ impl DnsMessageParser { }; self.raw_message_for_rdata_parsing = Some(raw_message_for_rdata_parsing_data); - BinDecoder::new(self.raw_message_for_rdata_parsing.as_ref().unwrap()).clone(index as u16) + BinDecoder::new( + self.raw_message_for_rdata_parsing + .as_ref() + .expect("None raw_message_for_rdata_parsing"), + ) + .clone(index as u16) } fn parse_wks_rdata( diff --git a/lib/dnstap-parser/Cargo.toml b/lib/dnstap-parser/Cargo.toml index cc5527f2e5814..8768274b3d3e0 100644 --- a/lib/dnstap-parser/Cargo.toml +++ b/lib/dnstap-parser/Cargo.toml @@ -15,7 +15,10 @@ hickory-proto.workspace = true prost.workspace = true snafu.workspace = true tracing.workspace = true -vector-lib = { path = "../vector-lib" } +vector-config.workspace = true +vector-common = { path = "../vector-common" } +vector-lookup = { path = "../vector-lookup", features = ["test"] } +vector-core = { path = "../vector-core" } vrl.workspace = true paste.workspace = true diff --git a/lib/dnstap-parser/src/internal_events.rs b/lib/dnstap-parser/src/internal_events.rs index 8510d1f54db1e..fc8f52e2b6952 100644 --- a/lib/dnstap-parser/src/internal_events.rs +++ b/lib/dnstap-parser/src/internal_events.rs @@ -1,7 +1,10 @@ use tracing::warn; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_common::{ + NamedInternalEvent, + internal_event::{InternalEvent, error_stage, error_type}, +}; -#[derive(Debug)] +#[derive(Debug, NamedInternalEvent)] pub(crate) struct DnstapParseWarning { pub error: E, } diff --git a/lib/dnstap-parser/src/parser.rs b/lib/dnstap-parser/src/parser.rs index b90589b8e890b..3876ff363bec6 100644 --- a/lib/dnstap-parser/src/parser.rs +++ b/lib/dnstap-parser/src/parser.rs @@ -16,10 +16,8 @@ use hickory_proto::{ }; use prost::Message; use snafu::Snafu; -use vector_lib::{ - Error, Result, emit, - event::{LogEvent, Value}, -}; +use vector_common::{Error, Result, internal_event::emit}; +use vector_core::event::{LogEvent, Value}; use vrl::{owned_value_path, path}; #[allow(warnings, clippy::all, clippy::pedantic, clippy::nursery)] @@ -38,10 +36,8 @@ use dnstap_proto::{ Dnstap, Message as DnstapMessage, SocketFamily, SocketProtocol, message::Type as DnstapMessageType, }; -use vector_lib::{ - config::log_schema, - lookup::{PathPrefix, lookup_v2::ValuePath}, -}; +use vector_core::config::log_schema; +use vector_lookup::{PathPrefix, lookup_v2::ValuePath}; use crate::{internal_events::DnstapParseWarning, schema::DNSTAP_VALUE_PATHS}; @@ -151,13 +147,13 @@ impl DnstapParser { && let Err(err) = DnstapParser::parse_dnstap_message(event, &root, message, parsing_options) { - emit!(DnstapParseWarning { error: &err }); + emit(DnstapParseWarning { error: &err }); need_raw_data = true; DnstapParser::insert(event, &root, &DNSTAP_VALUE_PATHS.error, err.to_string()); } } else { - emit!(DnstapParseWarning { - error: format!("Unknown dnstap data type: {dnstap_data_type_id}") + emit(DnstapParseWarning { + error: format!("Unknown dnstap data type: {dnstap_data_type_id}"), }); need_raw_data = true; } diff --git a/lib/dnstap-parser/src/schema.rs b/lib/dnstap-parser/src/schema.rs index e795cce9d9789..29a1b861f6e69 100644 --- a/lib/dnstap-parser/src/schema.rs +++ b/lib/dnstap-parser/src/schema.rs @@ -1,6 +1,6 @@ use std::{collections::BTreeMap, sync::LazyLock}; -use vector_lib::lookup::{OwnedValuePath, owned_value_path}; +use vector_lookup::{OwnedValuePath, owned_value_path}; use vrl::{ btreemap, value::{ @@ -135,8 +135,8 @@ impl DnstapEventSchema { /// Schema definition for fields stored in the root. fn root_schema_definition( &self, - schema: vector_lib::schema::Definition, - ) -> vector_lib::schema::Definition { + schema: vector_core::schema::Definition, + ) -> vector_core::schema::Definition { schema .optional_field(&DNSTAP_VALUE_PATHS.server_identity, Kind::bytes(), None) .optional_field(&DNSTAP_VALUE_PATHS.server_version, Kind::bytes(), None) @@ -152,8 +152,8 @@ impl DnstapEventSchema { /// Schema definition from the message. pub fn message_schema_definition( &self, - schema: vector_lib::schema::Definition, - ) -> vector_lib::schema::Definition { + schema: vector_core::schema::Definition, + ) -> vector_core::schema::Definition { schema .optional_field(&DNSTAP_VALUE_PATHS.socket_family, Kind::bytes(), None) .optional_field(&DNSTAP_VALUE_PATHS.socket_protocol, Kind::bytes(), None) @@ -179,8 +179,8 @@ impl DnstapEventSchema { /// The schema definition for a dns tap message. pub fn schema_definition( &self, - schema: vector_lib::schema::Definition, - ) -> vector_lib::schema::Definition { + schema: vector_core::schema::Definition, + ) -> vector_core::schema::Definition { self.root_schema_definition(self.message_schema_definition(schema)) } } diff --git a/lib/dnstap-parser/src/vrl_functions/parse_dnstap.rs b/lib/dnstap-parser/src/vrl_functions/parse_dnstap.rs index e9eecd61339b2..f61ef30c8f28a 100644 --- a/lib/dnstap-parser/src/vrl_functions/parse_dnstap.rs +++ b/lib/dnstap-parser/src/vrl_functions/parse_dnstap.rs @@ -1,6 +1,6 @@ use base64::prelude::{BASE64_STANDARD, Engine as _}; use dnsmsg_parser::dns_message_parser::DnsParserOptions; -use vector_lib::event::LogEvent; +use vector_core::event::LogEvent; use vrl::prelude::*; use crate::{parser::DnstapParser, schema::DnstapEventSchema}; @@ -13,6 +13,10 @@ impl Function for ParseDnstap { "parse_dnstap" } + fn usage(&self) -> &'static str { + "Parses the `value` as base64 encoded DNSTAP data." + } + fn parameters(&self) -> &'static [Parameter] { &[ Parameter { @@ -29,7 +33,7 @@ impl Function for ParseDnstap { } fn examples(&self) -> &'static [Example] { - &[Example { + &[example!( title: "Parse dnstap query message", source: r#"parse_dnstap!("ChVqYW1lcy1WaXJ0dWFsLU1hY2hpbmUSC0JJTkQgOS4xNi4zGgBy5wEIAxACGAEiEAAAAAAAAAAAAAAAAAAAAAAqECABBQJwlAAAAAAAAAAAADAw8+0CODVA7+zq9wVNMU3WNlI2kwIAAAABAAAAAAABCWZhY2Vib29rMQNjb20AAAEAAQAAKQIAAACAAAAMAAoACOxjCAG9zVgzWgUDY29tAGAAbQAAAAByZLM4AAAAAQAAAAAAAQJoNQdleGFtcGxlA2NvbQAABgABAAApBNABAUAAADkADwA1AAlubyBTRVAgbWF0Y2hpbmcgdGhlIERTIGZvdW5kIGZvciBkbnNzZWMtZmFpbGVkLm9yZy54AQ==")"#, result: Ok(indoc!( @@ -135,7 +139,7 @@ impl Function for ParseDnstap { "timestamp": "2020-06-30T03:50:07.920014129Z" }"# )), - }] + )] } fn compile( diff --git a/lib/enrichment/Cargo.toml b/lib/enrichment/Cargo.toml index 9e82fd16e0f95..1d2a67a2c26b6 100644 --- a/lib/enrichment/Cargo.toml +++ b/lib/enrichment/Cargo.toml @@ -6,7 +6,9 @@ edition = "2024" publish = false [dependencies] -arc-swap = { version = "1.7.1", default-features = false } +arc-swap.workspace = true chrono.workspace = true +const-str.workspace = true dyn-clone = { version = "1.0.20", default-features = false } +indoc.workspace = true vrl.workspace = true diff --git a/lib/enrichment/src/find_enrichment_table_records.rs b/lib/enrichment/src/find_enrichment_table_records.rs index 7420b18b26fbb..1b016aa901a14 100644 --- a/lib/enrichment/src/find_enrichment_table_records.rs +++ b/lib/enrichment/src/find_enrichment_table_records.rs @@ -51,6 +51,13 @@ impl Function for FindEnrichmentTableRecords { "find_enrichment_table_records" } + fn usage(&self) -> &'static str { + const_str::concat!( + "Searches an [enrichment table](/docs/reference/glossary/#enrichment-tables) for rows that match the provided condition.\n\n", + super::ENRICHMENT_TABLE_EXPLAINER + ) + } + fn parameters(&self) -> &'static [Parameter] { &[ Parameter { @@ -82,7 +89,7 @@ impl Function for FindEnrichmentTableRecords { } fn examples(&self) -> &'static [Example] { - &[Example { + &[example!( title: "find records", source: r#"find_enrichment_table_records!("test", {"surname": "Smith"})"#, result: Ok( @@ -90,7 +97,7 @@ impl Function for FindEnrichmentTableRecords { {"id": 2, "firstname": "Fred", "surname": "Smith"}]"#, }, ), - }] + )] } fn compile( diff --git a/lib/enrichment/src/get_enrichment_table_record.rs b/lib/enrichment/src/get_enrichment_table_record.rs index 49ada44993192..ef2103702f8ca 100644 --- a/lib/enrichment/src/get_enrichment_table_record.rs +++ b/lib/enrichment/src/get_enrichment_table_record.rs @@ -48,6 +48,14 @@ impl Function for GetEnrichmentTableRecord { "get_enrichment_table_record" } + fn usage(&self) -> &'static str { + const USAGE: &str = const_str::concat!( + "Searches an [enrichment table](/docs/reference/glossary/#enrichment-tables) for a row that matches the provided condition. A single row must be matched. If no rows are found or more than one row is found, an error is returned.\n\n", + super::ENRICHMENT_TABLE_EXPLAINER + ); + USAGE + } + fn parameters(&self) -> &'static [Parameter] { &[ Parameter { @@ -79,11 +87,11 @@ impl Function for GetEnrichmentTableRecord { } fn examples(&self) -> &'static [Example] { - &[Example { + &[example!( title: "find records", source: r#"get_enrichment_table_record!("test", {"id": 1})"#, result: Ok(r#"{"id": 1, "firstname": "Bob", "surname": "Smith"}"#), - }] + )] } fn compile( diff --git a/lib/enrichment/src/lib.rs b/lib/enrichment/src/lib.rs index e69cf979a1dc0..a73f18f163181 100644 --- a/lib/enrichment/src/lib.rs +++ b/lib/enrichment/src/lib.rs @@ -9,6 +9,7 @@ mod test_util; mod vrl_util; use dyn_clone::DynClone; +use indoc::indoc; pub use tables::{TableRegistry, TableSearch}; use vrl::{ compiler::Function, @@ -97,3 +98,59 @@ pub fn vrl_functions() -> Vec> { Box::new(find_enrichment_table_records::FindEnrichmentTableRecords) as _, ] } + +pub(crate) const ENRICHMENT_TABLE_EXPLAINER: &str = indoc! {r#" + For `file` enrichment tables, this condition needs to be a VRL object in which + the key-value pairs indicate a field to search mapped to a value to search in that field. + This function returns the rows that match the provided condition(s). _All_ fields need to + match for rows to be returned; if any fields do not match, then no rows are returned. + + There are currently three forms of search criteria: + + 1. **Exact match search**. The given field must match the value exactly. Case sensitivity + can be specified using the `case_sensitive` argument. An exact match search can use an + index directly into the dataset, which should make this search fairly "cheap" from a + performance perspective. + + 2. **Wildcard match search**. The given fields specified by the exact match search may also + be matched exactly to the value provided to the `wildcard` parameter. + A wildcard match search can also use an index directly into the dataset. + + 3. **Date range search**. The given field must be greater than or equal to the `from` date + and/or less than or equal to the `to` date. A date range search involves + sequentially scanning through the rows that have been located using any exact match + criteria. This can be an expensive operation if there are many rows returned by any exact + match criteria. Therefore, use date ranges as the _only_ criteria when the enrichment + data set is very small. + + For `geoip` and `mmdb` enrichment tables, this condition needs to be a VRL object with a single key-value pair + whose value needs to be a valid IP address. Example: `{"ip": .ip }`. If a return field is expected + and without a value, `null` is used. This table can return the following fields: + + * ISP databases: + * `autonomous_system_number` + * `autonomous_system_organization` + * `isp` + * `organization` + + * City databases: + * `city_name` + * `continent_code` + * `country_code` + * `country_name` + * `region_code` + * `region_name` + * `metro_code` + * `latitude` + * `longitude` + * `postal_code` + * `timezone` + + * Connection-Type databases: + * `connection_type` + + To use this function, you need to update your configuration to + include an + [`enrichment_tables`](/docs/reference/configuration/global-options/#enrichment_tables) + parameter. +"#}; diff --git a/lib/file-source-common/src/buffer.rs b/lib/file-source-common/src/buffer.rs index e2a4e5596c920..097331fbf2be3 100644 --- a/lib/file-source-common/src/buffer.rs +++ b/lib/file-source-common/src/buffer.rs @@ -46,13 +46,56 @@ pub async fn read_until_with_max_size<'a, R: AsyncBufRead + ?Sized + Unpin>( let delim_len = delim.len(); let mut discarded_for_size_and_truncated = Vec::new(); let mut reader = Box::new(reader); + + // Used to track partial delimiter matches across buffer boundaries. + // Data is read in chunks from the reader (see `fill_buf` below). + // A multi-byte delimiter may be split across the "old" and "new" buffers. + // Any potential partial delimiter that was found in the "old" buffer is stored in this variable. + let mut partial_delim: BytesMut = BytesMut::with_capacity(delim_len); + loop { + // Read the next chunk of data let available: &[u8] = match reader.fill_buf().await { Ok(n) => n, Err(ref e) if e.kind() == io::ErrorKind::Interrupted => continue, Err(e) => return Err(e), }; + // First, check if we have a partial delimiter from the previous iteration/buffer + if !partial_delim.is_empty() { + let expected_suffix = &delim[partial_delim.len()..]; + let expected_suffix_len = expected_suffix.len(); + + // We already know that we have a partial delimiter match from the previous buffer. + // Here we check what part of the delimiter is missing and whether the new buffer + // contains the remaining part. + if available.len() >= expected_suffix_len + && &available[..expected_suffix_len] == expected_suffix + { + // Complete delimiter found! Consume the remainder of the delimiter so we can start + // processing data after the delimiter. + reader.consume(expected_suffix_len); + *position += expected_suffix_len as u64; + total_read += expected_suffix_len; + partial_delim.clear(); + + // Found a complete delimiter, return the current buffer so we can proceed with the + // next record after this delimiter in the next call. + return Ok(ReadResult { + successfully_read: Some(total_read), + discarded_for_size_and_truncated, + }); + } else { + // Not a complete delimiter after all. + // Add partial_delim to output buffer as it is actual data. + if !discarding { + buf.extend_from_slice(&partial_delim); + } + partial_delim.clear(); + // Continue processing current available buffer + } + } + let (done, used) = { match delim_finder.find(available) { Some(i) => { @@ -62,13 +105,75 @@ pub async fn read_until_with_max_size<'a, R: AsyncBufRead + ?Sized + Unpin>( (true, i + delim_len) } None => { - if !discarding { - buf.extend_from_slice(available); + // No delimiter found in current buffer. But there could be a partial delimiter + // at the end of this buffer. For multi-byte delimiters like \r\n, we need + // to handle the case where the delimiter is split across buffer boundaries + // (e.g. \r in the "old" buffer, then we read new data and find \n in the new + // buffer). + let mut partial_match_len = 0; + + // We only need to check if we're not already at the end of the buffer and if we + // have a delimiter that has more than one byte. + if !available.is_empty() && delim_len > 1 { + // Check if the end of the current buffer matches a prefix of the delimiter + // by testing from longest to shortest possible prefix. + // + // This loop runs at most (delim_len - 1) iterations: + // - 2-byte delimiter (\r\n): 1 iteration max + // - 5-byte delimiter: 4 iterations max + // + // This part of the code is only called if all of these are true: + // + // - We have a new buffer (e.g. every 8kB, i.e. only called once per buffer) + // - We have a multi-byte delimiter + // - This delimiter could not be found in the current buffer + // + // Even for longer delimiters the performance impact is negligible. + // + // Example 1: + // Delimiter: \r\n + // Iteration 1: It checks if the current buffer ends with "\r", + // if it does we have a potential partial delimiter. + // The next chunk will confirm whether this is truly part of a delimiter. + + // Example 2: + // Delimiter: ABCDE + // Iteration 1: It checks if the current buffer ends with "ABCD" (we don't + // need to check "ABCDE" because that would have been caught by + // `delim_finder.find` earlier) + // Iteration 2: It checks if the current buffer ends with "ABC" + // Iterations 3-4: Same for "AB" and "A" + for prefix_len in (1..delim_len).rev() { + if available.len() >= prefix_len + && available.ends_with(&delim[..prefix_len]) + { + partial_match_len = prefix_len; + break; + } + } + } + + let bytes_to_copy = available.len() - partial_match_len; + + if !discarding && bytes_to_copy > 0 { + buf.extend_from_slice(&available[..bytes_to_copy]); } + + // If we found a potential partial delimiter, save it for the next iteration + if partial_match_len > 0 { + partial_delim.clear(); + partial_delim.extend_from_slice(&available[bytes_to_copy..]); + } + (false, available.len()) } } }; + + // Check if we're at EOF before we start processing + // (for borrow checker, has to come before `consume`) + let at_eof = available.is_empty(); + reader.consume(used); *position += used as u64; // do this at exactly same time total_read += used; @@ -92,11 +197,12 @@ pub async fn read_until_with_max_size<'a, R: AsyncBufRead + ?Sized + Unpin>( discarding = false; buf.clear(); } - } else if used == 0 { - // We've hit EOF but not yet seen a newline. This can happen when unlucky timing causes - // us to observe an incomplete write. We return None here and let the loop continue - // next time the method is called. This is safe because the buffer is specific to this - // FileWatcher. + } else if used == 0 && at_eof { + // We've hit EOF but haven't seen a delimiter. This can happen when: + // 1. The file ends without a trailing delimiter + // 2. We're observing an incomplete write + // + // Return None to signal the caller to retry later. return Ok(ReadResult { successfully_read: None, discarded_for_size_and_truncated, @@ -262,4 +368,113 @@ mod test { .await .unwrap() } + + /// Generic test helper that tests delimiter splits across buffer boundaries + /// for any delimiter length. This function: + /// 1. Creates test data with delimiters positioned to split at buffer boundaries + /// 2. Tests multiple iterations to ensure state tracking works correctly + /// 3. Verifies all lines are correctly separated without merging + async fn test_delimiter_boundary_split_helper(delimiter: &[u8], num_lines: usize) { + let delimiter_len = delimiter.len(); + + // Use a buffer capacity that will force splits + // We'll position delimiters to split at this boundary + let buffer_capacity = 10; + + // Build test data where each delimiter is positioned to split across buffer boundary + // Strategy: For each line, calculate position so delimiter starts at boundary - (delimiter_len - 1) + let mut data = Vec::new(); + let mut expected_lines = Vec::new(); + + for i in 0..num_lines { + // Create line content that positions the delimiter to split at buffer boundary + // We want the delimiter to straddle a buffer_capacity boundary + + // Calculate how many bytes until the next buffer boundary + let current_pos = data.len(); + let bytes_until_boundary = buffer_capacity - (current_pos % buffer_capacity); + + // Create line content that will position delimiter to split + // We want (delimiter_len - 1) bytes before boundary, then 1 byte after + let line_content = if bytes_until_boundary > delimiter_len { + let content_len = bytes_until_boundary - (delimiter_len - 1); + format!("line{:0width$}", i, width = content_len.saturating_sub(4)).into_bytes() + } else { + // Not enough room in this buffer, pad to next boundary + let padding = bytes_until_boundary; + let extra_content = buffer_capacity - (delimiter_len - 1); + let mut content = vec![b'X'; padding]; + content.extend_from_slice( + format!("L{:0width$}", i, width = extra_content.saturating_sub(1)).as_bytes(), + ); + content + }; + + expected_lines.push(line_content.clone()); + data.extend_from_slice(&line_content); + data.extend_from_slice(delimiter); + } + + // Now test reading this data + let cursor = Cursor::new(data); + let mut reader = BufReader::with_capacity(buffer_capacity, cursor); + let mut position = 0; + let max_size = 1024; + + // Read each line and verify it matches expected + for (i, expected_line) in expected_lines.iter().enumerate() { + let mut buffer = BytesMut::new(); + let result = read_until_with_max_size( + Box::pin(&mut reader), + &mut position, + delimiter, + &mut buffer, + max_size, + ) + .await + .unwrap(); + + assert_eq!( + buffer.as_ref(), + expected_line.as_slice(), + "Line {} should match expected content. Got: {:?}, Expected: {:?}", + i, + String::from_utf8_lossy(&buffer), + String::from_utf8_lossy(expected_line) + ); + + assert!( + result.successfully_read.is_some(), + "Should find delimiter for line {}", + i + ); + } + } + + #[tokio::test] + async fn test_single_byte_delimiter_boundary() { + // Test single-byte delimiter (should work without any special handling) + test_delimiter_boundary_split_helper(b"\n", 5).await; + } + + #[tokio::test] + async fn test_two_byte_delimiter_boundary() { + // Test two-byte delimiter (CRLF case) + test_delimiter_boundary_split_helper(b"\r\n", 5).await; + } + + #[tokio::test] + async fn test_three_byte_delimiter_boundary() { + test_delimiter_boundary_split_helper(b"|||", 5).await; + } + + #[tokio::test] + async fn test_four_byte_delimiter_boundary() { + test_delimiter_boundary_split_helper(b"<|>|", 5).await; + } + + #[tokio::test] + async fn test_five_byte_delimiter_boundary() { + test_delimiter_boundary_split_helper(b"<<>>>", 5).await; + } } diff --git a/lib/portpicker/Cargo.toml b/lib/portpicker/Cargo.toml deleted file mode 100644 index 0b296fdce8301..0000000000000 --- a/lib/portpicker/Cargo.toml +++ /dev/null @@ -1,10 +0,0 @@ -[package] -name = "portpicker" -version = "1.0.0" -authors = ["Vector Contributors ", "Hannes Karppila "] -edition = "2024" -publish = false -license = "Unlicense" - -[dependencies] -rand.workspace = true diff --git a/lib/portpicker/LICENSE b/lib/portpicker/LICENSE deleted file mode 100644 index cf1ab25da0349..0000000000000 --- a/lib/portpicker/LICENSE +++ /dev/null @@ -1,24 +0,0 @@ -This is free and unencumbered software released into the public domain. - -Anyone is free to copy, modify, publish, use, compile, sell, or -distribute this software, either in source code form or as a compiled -binary, for any purpose, commercial or non-commercial, and by any -means. - -In jurisdictions that recognize copyright laws, the author or authors -of this software dedicate any and all copyright interest in the -software to the public domain. We make this dedication for the benefit -of the public at large and to the detriment of our heirs and -successors. We intend this dedication to be an overt act of -relinquishment in perpetuity of all present and future rights to this -software under copyright law. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -OTHER DEALINGS IN THE SOFTWARE. - -For more information, please refer to diff --git a/lib/portpicker/src/lib.rs b/lib/portpicker/src/lib.rs deleted file mode 100644 index 8c83f6e4d137d..0000000000000 --- a/lib/portpicker/src/lib.rs +++ /dev/null @@ -1,134 +0,0 @@ -// Modified by `Vector Contributors `. -// Based on `https://github.com/Dentosal/portpicker-rs` by `Hannes Karppila `. -// `portpicker-rs` LICENSE: -// This is free and unencumbered software released into the public domain. - -// Anyone is free to copy, modify, publish, use, compile, sell, or -// distribute this software, either in source code form or as a compiled -// binary, for any purpose, commercial or non-commercial, and by any -// means. - -// In jurisdictions that recognize copyright laws, the author or authors -// of this software dedicate any and all copyright interest in the -// software to the public domain. We make this dedication for the benefit -// of the public at large and to the detriment of our heirs and -// successors. We intend this dedication to be an overt act of -// relinquishment in perpetuity of all present and future rights to this -// software under copyright law. - -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -// IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -// OTHER DEALINGS IN THE SOFTWARE. - -// For more information, please refer to - -#![deny(warnings)] - -use std::net::{IpAddr, SocketAddr, TcpListener, ToSocketAddrs, UdpSocket}; - -use rand::{Rng, rng}; - -pub type Port = u16; - -// Try to bind to a socket using UDP -fn test_bind_udp(addr: A) -> Option { - Some(UdpSocket::bind(addr).ok()?.local_addr().ok()?.port()) -} - -// Try to bind to a socket using TCP -fn test_bind_tcp(addr: A) -> Option { - Some(TcpListener::bind(addr).ok()?.local_addr().ok()?.port()) -} - -/// Check if a port is free on UDP -pub fn is_free_udp(ip: IpAddr, port: Port) -> bool { - test_bind_udp(SocketAddr::new(ip, port)).is_some() -} - -/// Check if a port is free on TCP -pub fn is_free_tcp(ip: IpAddr, port: Port) -> bool { - test_bind_tcp(SocketAddr::new(ip, port)).is_some() -} - -/// Check if a port is free on both TCP and UDP -pub fn is_free(ip: IpAddr, port: Port) -> bool { - is_free_tcp(ip, port) && is_free_udp(ip, port) -} - -/// Asks the OS for a free port -fn ask_free_tcp_port(ip: IpAddr) -> Option { - test_bind_tcp(SocketAddr::new(ip, 0)) -} - -/// Picks an available port that is available on both TCP and UDP -/// ```rust -/// use portpicker::pick_unused_port; -/// use std::net::{IpAddr, Ipv4Addr}; -/// let port: u16 = pick_unused_port(IpAddr::V4(Ipv4Addr::LOCALHOST)); -/// ``` -pub fn pick_unused_port(ip: IpAddr) -> Port { - let mut rng = rng(); - - loop { - // Try random port first - for _ in 0..10 { - let port = rng.random_range(15000..25000); - if is_free(ip, port) { - return port; - } - } - - // Ask the OS for a port - for _ in 0..10 { - if let Some(port) = ask_free_tcp_port(ip) { - // Test that the udp port is free as well - if is_free_udp(ip, port) { - return port; - } - } - } - } -} - -pub fn bind_unused_udp(ip: IpAddr) -> UdpSocket { - let mut rng = rng(); - - loop { - // Try random port first - for _ in 0..10 { - let port = rng.random_range(15000..25000); - - if let Ok(socket) = UdpSocket::bind(SocketAddr::new(ip, port)) { - return socket; - } - } - - // Ask the OS for a port - for _ in 0..10 { - if let Ok(socket) = UdpSocket::bind(SocketAddr::new(ip, 0)) { - return socket; - } - } - } -} - -#[cfg(test)] -mod tests { - use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; - - use super::pick_unused_port; - - #[test] - fn ipv4_localhost() { - pick_unused_port(IpAddr::V4(Ipv4Addr::LOCALHOST)); - } - - #[test] - fn ipv6_localhost() { - pick_unused_port(IpAddr::V6(Ipv6Addr::LOCALHOST)); - } -} diff --git a/lib/tracing-limit/Cargo.toml b/lib/tracing-limit/Cargo.toml index 69311622ab120..77d3d54ef4371 100644 --- a/lib/tracing-limit/Cargo.toml +++ b/lib/tracing-limit/Cargo.toml @@ -6,6 +6,9 @@ edition = "2024" publish = false license = "MPL-2.0" +[lints.clippy] +unwrap-used = "forbid" + [dependencies] tracing-core = { version = "0.1", default-features = false } tracing-subscriber = { workspace = true, features = ["registry", "std"] } diff --git a/lib/tracing-limit/benches/limit.rs b/lib/tracing-limit/benches/limit.rs index 4f959a540d80d..25d905bc2ae85 100644 --- a/lib/tracing-limit/benches/limit.rs +++ b/lib/tracing-limit/benches/limit.rs @@ -114,17 +114,17 @@ where } fn on_new_span(&self, span: &span::Attributes<'_>, _id: &span::Id, _ctx: Context<'_, S>) { - let mut visitor = Visitor(self.mutex.lock().unwrap()); + let mut visitor = Visitor(self.mutex.lock().expect("mutex should not be poisoned")); span.record(&mut visitor); } fn on_record(&self, _id: &span::Id, values: &span::Record<'_>, _ctx: Context<'_, S>) { - let mut visitor = Visitor(self.mutex.lock().unwrap()); + let mut visitor = Visitor(self.mutex.lock().expect("mutex should not be poisoned")); values.record(&mut visitor); } fn on_event(&self, event: &Event<'_>, _ctx: Context<'_, S>) { - let mut visitor = Visitor(self.mutex.lock().unwrap()); + let mut visitor = Visitor(self.mutex.lock().expect("mutex should not be poisoned")); event.record(&mut visitor); } diff --git a/lib/tracing-limit/src/lib.rs b/lib/tracing-limit/src/lib.rs index 62113f36ce7cc..ebb7398dfc7ef 100644 --- a/lib/tracing-limit/src/lib.rs +++ b/lib/tracing-limit/src/lib.rs @@ -377,15 +377,17 @@ where let valueset = fields.value_set(&values); let event = Event::new(metadata, &valueset); self.inner.on_event(&event, ctx.clone()); - } else { - let values = [( - &fields.field(RATE_LIMIT_FIELD).unwrap(), - Some(&rate_limit as &dyn Value), - )]; + } else if let Some(rate_limit_field) = fields.field(RATE_LIMIT_FIELD) { + let values = [(&rate_limit_field, Some(&rate_limit as &dyn Value))]; let valueset = fields.value_set(&values); let event = Event::new(metadata, &valueset); self.inner.on_event(&event, ctx.clone()); + } else { + // If the event metadata has neither a "message" nor "internal_log_rate_limit" field, + // we cannot create a proper synthetic event. This can happen with custom debug events + // that have their own field structure. In this case, we simply skip emitting the + // rate limit notification rather than panicking. } } } @@ -1012,4 +1014,42 @@ mod test { ] ); } + + #[test] + #[serial] + fn events_with_custom_fields_no_message_dont_panic() { + // Verify events without "message" or "internal_log_rate_limit" fields don't panic + // when rate limiting skips suppression notifications. + let (events, sub) = setup_test(1); + tracing::subscriber::with_default(sub, || { + // Use closure to ensure all events share the same callsite + let emit_event = || { + debug!(component_id = "test_component", utilization = 0.85); + }; + + // First window: emit 5 events, only the first one should be logged + for _ in 0..5 { + emit_event(); + MockClock::advance(Duration::from_millis(100)); + } + + // Advance to the next window + MockClock::advance(Duration::from_millis(1000)); + + // Second window: this event should be logged + emit_event(); + }); + + let events = events.lock().unwrap(); + + // First event from window 1, first event from window 2 + // Suppression notifications are skipped (no message field) + assert_eq!( + *events, + vec![ + event!("", component_id: "test_component", utilization: "0.85"), + event!("", component_id: "test_component", utilization: "0.85"), + ] + ); + } } diff --git a/lib/vector-api-client/Cargo.toml b/lib/vector-api-client/Cargo.toml index 5b3b329088afe..3fbc200af231b 100644 --- a/lib/vector-api-client/Cargo.toml +++ b/lib/vector-api-client/Cargo.toml @@ -18,14 +18,14 @@ anyhow.workspace = true # Tokio / Futures futures.workspace = true tokio = { workspace = true, features = ["macros", "rt", "sync"] } -tokio-stream = { version = "0.1.17", default-features = false, features = ["sync"] } +tokio-stream = { workspace = true, features = ["sync"] } # GraphQL graphql_client = { version = "0.14.0", default-features = false, features = ["graphql_query_derive"] } # HTTP / WebSockets reqwest = { version = "0.11.26", default-features = false, features = ["json"] } -tokio-tungstenite = { version = "0.20.1", default-features = false, features = ["connect", "rustls"] } +tokio-tungstenite = { workspace = true, features = ["connect", "rustls"] } # External libs chrono.workspace = true diff --git a/lib/vector-buffers/Cargo.toml b/lib/vector-buffers/Cargo.toml index 3eb7a674ca1c4..c89500fad6499 100644 --- a/lib/vector-buffers/Cargo.toml +++ b/lib/vector-buffers/Cargo.toml @@ -24,7 +24,7 @@ memmap2 = { version = "0.9.8", default-features = false } metrics.workspace = true num-traits = { version = "0.2.19", default-features = false } paste.workspace = true -rkyv = { version = "0.7.45", default-features = false, features = ["size_32", "std", "strict", "validation"] } +rkyv = { version = "0.7.46", default-features = false, features = ["size_32", "std", "strict", "validation"] } serde.workspace = true snafu.workspace = true tokio-util = { version = "0.7.0", default-features = false } @@ -47,7 +47,7 @@ quickcheck = "1.0" rand.workspace = true serde_yaml.workspace = true temp-dir = "0.1.16" -tokio-test = "0.4.4" +tokio-test.workspace = true tracing-fluent-assertions = { version = "0.3" } tracing-subscriber = { workspace = true, features = ["env-filter", "fmt", "registry", "std", "ansi"] } diff --git a/lib/vector-buffers/src/internal_events.rs b/lib/vector-buffers/src/internal_events.rs index cf852d7d04674..6d2735c7bb75f 100644 --- a/lib/vector-buffers/src/internal_events.rs +++ b/lib/vector-buffers/src/internal_events.rs @@ -1,11 +1,13 @@ use std::time::Duration; use metrics::{Histogram, counter, gauge, histogram}; +use vector_common::NamedInternalEvent; use vector_common::{ internal_event::{InternalEvent, error_type}, registered_event, }; +#[derive(NamedInternalEvent)] pub struct BufferCreated { pub buffer_id: String, pub idx: usize, @@ -16,25 +18,41 @@ pub struct BufferCreated { impl InternalEvent for BufferCreated { #[expect(clippy::cast_precision_loss)] fn emit(self) { + let stage = self.idx.to_string(); if self.max_size_events != 0 { + gauge!( + "buffer_max_size_events", + "buffer_id" => self.buffer_id.clone(), + "stage" => stage.clone(), + ) + .set(self.max_size_events as f64); + // DEPRECATED: buffer-bytes-events-metrics gauge!( "buffer_max_event_size", "buffer_id" => self.buffer_id.clone(), - "stage" => self.idx.to_string(), + "stage" => stage.clone(), ) .set(self.max_size_events as f64); } if self.max_size_bytes != 0 { + gauge!( + "buffer_max_size_bytes", + "buffer_id" => self.buffer_id.clone(), + "stage" => stage.clone(), + ) + .set(self.max_size_bytes as f64); + // DEPRECATED: buffer-bytes-events-metrics gauge!( "buffer_max_byte_size", "buffer_id" => self.buffer_id, - "stage" => self.idx.to_string(), + "stage" => stage, ) .set(self.max_size_bytes as f64); } } } +#[derive(NamedInternalEvent)] pub struct BufferEventsReceived { pub buffer_id: String, pub idx: usize, @@ -60,12 +78,26 @@ impl InternalEvent for BufferEventsReceived { "stage" => self.idx.to_string() ) .increment(self.byte_size); + // DEPRECATED: buffer-bytes-events-metrics gauge!( "buffer_events", "buffer_id" => self.buffer_id.clone(), "stage" => self.idx.to_string() ) .set(self.total_count as f64); + gauge!( + "buffer_size_events", + "buffer_id" => self.buffer_id.clone(), + "stage" => self.idx.to_string() + ) + .set(self.total_count as f64); + gauge!( + "buffer_size_bytes", + "buffer_id" => self.buffer_id.clone(), + "stage" => self.idx.to_string() + ) + .set(self.total_byte_size as f64); + // DEPRECATED: buffer-bytes-events-metrics gauge!( "buffer_byte_size", "buffer_id" => self.buffer_id, @@ -75,6 +107,7 @@ impl InternalEvent for BufferEventsReceived { } } +#[derive(NamedInternalEvent)] pub struct BufferEventsSent { pub buffer_id: String, pub idx: usize, @@ -99,12 +132,26 @@ impl InternalEvent for BufferEventsSent { "stage" => self.idx.to_string() ) .increment(self.byte_size); + // DEPRECATED: buffer-bytes-events-metrics gauge!( "buffer_events", "buffer_id" => self.buffer_id.clone(), "stage" => self.idx.to_string() ) .set(self.total_count as f64); + gauge!( + "buffer_size_events", + "buffer_id" => self.buffer_id.clone(), + "stage" => self.idx.to_string() + ) + .set(self.total_count as f64); + gauge!( + "buffer_size_bytes", + "buffer_id" => self.buffer_id.clone(), + "stage" => self.idx.to_string() + ) + .set(self.total_byte_size as f64); + // DEPRECATED: buffer-bytes-events-metrics gauge!( "buffer_byte_size", "buffer_id" => self.buffer_id, @@ -114,6 +161,7 @@ impl InternalEvent for BufferEventsSent { } } +#[derive(NamedInternalEvent)] pub struct BufferEventsDropped { pub buffer_id: String, pub idx: usize, @@ -165,12 +213,26 @@ impl InternalEvent for BufferEventsDropped { "intentional" => intentional_str, ) .increment(self.byte_size); + // DEPRECATED: buffer-bytes-events-metrics gauge!( "buffer_events", "buffer_id" => self.buffer_id.clone(), "stage" => self.idx.to_string() ) .set(self.total_count as f64); + gauge!( + "buffer_size_events", + "buffer_id" => self.buffer_id.clone(), + "stage" => self.idx.to_string() + ) + .set(self.total_count as f64); + gauge!( + "buffer_size_bytes", + "buffer_id" => self.buffer_id.clone(), + "stage" => self.idx.to_string() + ) + .set(self.total_byte_size as f64); + // DEPRECATED: buffer-bytes-events-metrics gauge!( "buffer_byte_size", "buffer_id" => self.buffer_id, @@ -180,6 +242,7 @@ impl InternalEvent for BufferEventsDropped { } } +#[derive(NamedInternalEvent)] pub struct BufferReadError { pub error_code: &'static str, pub error: String, diff --git a/lib/vector-buffers/src/lib.rs b/lib/vector-buffers/src/lib.rs index d9c6373831c50..5f6ade35f9c78 100644 --- a/lib/vector-buffers/src/lib.rs +++ b/lib/vector-buffers/src/lib.rs @@ -108,6 +108,13 @@ pub trait Bufferable: InMemoryBufferable + Encodable {} // Blanket implementation for anything that is already bufferable. impl Bufferable for T where T: InMemoryBufferable + Encodable {} +/// Hook for observing items as they are sent into a `BufferSender`. +pub trait BufferInstrumentation: Send + Sync + 'static { + /// Called immediately before the item is emitted to the underlying buffer. + /// The underlying type is stored in an `Arc`, so we cannot have `&mut self`. + fn on_send(&self, item: &mut T); +} + pub trait EventCount { fn event_count(&self) -> usize; } diff --git a/lib/vector-buffers/src/topology/builder.rs b/lib/vector-buffers/src/topology/builder.rs index 0d909a7eb1941..0f54759c3bc65 100644 --- a/lib/vector-buffers/src/topology/builder.rs +++ b/lib/vector-buffers/src/topology/builder.rs @@ -4,12 +4,12 @@ use async_trait::async_trait; use snafu::{ResultExt, Snafu}; use tracing::Span; -use super::channel::{ReceiverAdapter, SenderAdapter}; +use super::channel::{ChannelMetricMetadata, ReceiverAdapter, SenderAdapter}; use crate::{ Bufferable, WhenFull, buffer_usage_data::{BufferUsage, BufferUsageHandle}, - topology::channel::{BufferReceiver, BufferSender}, - variants::MemoryBuffer, + config::MemoryBufferSize, + topology::channel::{BufferReceiver, BufferSender, limited}, }; /// Value that can be used as a stage in a buffer topology. @@ -186,26 +186,26 @@ impl TopologyBuilder { /// create the stage, installing buffer usage metrics that aren't required, and so on. /// #[allow(clippy::print_stderr)] - pub async fn standalone_memory( + pub fn standalone_memory( max_events: NonZeroUsize, when_full: WhenFull, receiver_span: &Span, + metadata: Option, + ewma_alpha: Option, ) -> (BufferSender, BufferReceiver) { let usage_handle = BufferUsageHandle::noop(); + usage_handle.set_buffer_limits(None, Some(max_events.get())); - let memory_buffer = Box::new(MemoryBuffer::with_max_events(max_events)); - let (sender, receiver) = memory_buffer - .into_buffer_parts(usage_handle.clone()) - .await - .unwrap_or_else(|_| unreachable!("should not fail to directly create a memory buffer")); + let limit = MemoryBufferSize::MaxEvents(max_events); + let (sender, receiver) = limited(limit, metadata, ewma_alpha); let mode = match when_full { WhenFull::Overflow => WhenFull::Block, m => m, }; - let mut sender = BufferSender::new(sender, mode); + let mut sender = BufferSender::new(sender.into(), mode); sender.with_send_duration_instrumentation(0, receiver_span); - let receiver = BufferReceiver::new(receiver); + let receiver = BufferReceiver::new(receiver.into()); (sender, receiver) } @@ -224,23 +224,23 @@ impl TopologyBuilder { /// can simplifying needing to require callers to do all the boilerplate to create the builder, /// create the stage, installing buffer usage metrics that aren't required, and so on. #[cfg(test)] - pub async fn standalone_memory_test( + pub fn standalone_memory_test( max_events: NonZeroUsize, when_full: WhenFull, usage_handle: BufferUsageHandle, + metadata: Option, ) -> (BufferSender, BufferReceiver) { - let memory_buffer = Box::new(MemoryBuffer::with_max_events(max_events)); - let (sender, receiver) = memory_buffer - .into_buffer_parts(usage_handle.clone()) - .await - .unwrap_or_else(|_| unreachable!("should not fail to directly create a memory buffer")); + usage_handle.set_buffer_limits(None, Some(max_events.get())); + + let limit = MemoryBufferSize::MaxEvents(max_events); + let (sender, receiver) = limited(limit, metadata, None); let mode = match when_full { WhenFull::Overflow => WhenFull::Block, m => m, }; - let mut sender = BufferSender::new(sender, mode); - let mut receiver = BufferReceiver::new(receiver); + let mut sender = BufferSender::new(sender.into(), mode); + let mut receiver = BufferReceiver::new(receiver.into()); sender.with_usage_instrumentation(usage_handle.clone()); receiver.with_usage_instrumentation(usage_handle); diff --git a/lib/vector-buffers/src/topology/channel/limited_queue.rs b/lib/vector-buffers/src/topology/channel/limited_queue.rs index 4320936cec750..739f7cca9cdaa 100644 --- a/lib/vector-buffers/src/topology/channel/limited_queue.rs +++ b/lib/vector-buffers/src/topology/channel/limited_queue.rs @@ -8,10 +8,15 @@ use std::{ }, }; +#[cfg(test)] +use std::sync::Mutex; + use async_stream::stream; use crossbeam_queue::{ArrayQueue, SegQueue}; use futures::Stream; +use metrics::{Gauge, Histogram, gauge, histogram}; use tokio::sync::{Notify, OwnedSemaphorePermit, Semaphore, TryAcquireError}; +use vector_common::stats::EwmaGauge; use crate::{InMemoryBufferable, config::MemoryBufferSize}; @@ -88,12 +93,113 @@ where } } +#[derive(Clone, Debug)] +pub struct ChannelMetricMetadata { + prefix: &'static str, + output: Option, +} + +impl ChannelMetricMetadata { + pub fn new(prefix: &'static str, output: Option) -> Self { + Self { prefix, output } + } +} + +#[derive(Clone, Debug)] +struct Metrics { + histogram: Histogram, + gauge: Gauge, + mean_gauge: EwmaGauge, + // We hold a handle to the max gauge to avoid it being dropped by the metrics collector, but + // since the value is static, we never need to update it. The compiler detects this as an unused + // field, so we need to suppress the warning here. + #[expect(dead_code)] + max_gauge: Gauge, + #[expect(dead_code)] + legacy_max_gauge: Gauge, + #[cfg(test)] + recorded_values: Arc>>, +} + +impl Metrics { + #[expect(clippy::cast_precision_loss)] // We have to convert buffer sizes for a gauge, it's okay to lose precision here. + fn new( + limit: MemoryBufferSize, + metadata: ChannelMetricMetadata, + ewma_alpha: Option, + ) -> Self { + let ChannelMetricMetadata { prefix, output } = metadata; + let (legacy_suffix, gauge_suffix, max_value) = match limit { + MemoryBufferSize::MaxEvents(max_events) => ( + "_max_event_size", + "_max_size_events", + max_events.get() as f64, + ), + MemoryBufferSize::MaxSize(max_bytes) => { + ("_max_byte_size", "_max_size_bytes", max_bytes.get() as f64) + } + }; + let max_gauge_name = format!("{prefix}{gauge_suffix}"); + let legacy_max_gauge_name = format!("{prefix}{legacy_suffix}"); + let histogram_name = format!("{prefix}_utilization"); + let gauge_name = format!("{prefix}_utilization_level"); + let mean_name = format!("{prefix}_utilization_mean"); + #[cfg(test)] + let recorded_values = Arc::new(Mutex::new(Vec::new())); + if let Some(label_value) = output { + let max_gauge = gauge!(max_gauge_name, "output" => label_value.clone()); + max_gauge.set(max_value); + let mean_gauge_handle = gauge!(mean_name, "output" => label_value.clone()); + // DEPRECATED: buffer-bytes-events-metrics + let legacy_max_gauge = gauge!(legacy_max_gauge_name, "output" => label_value.clone()); + legacy_max_gauge.set(max_value); + Self { + histogram: histogram!(histogram_name, "output" => label_value.clone()), + gauge: gauge!(gauge_name, "output" => label_value.clone()), + mean_gauge: EwmaGauge::new(mean_gauge_handle, ewma_alpha), + max_gauge, + legacy_max_gauge, + #[cfg(test)] + recorded_values, + } + } else { + let max_gauge = gauge!(max_gauge_name); + max_gauge.set(max_value); + let mean_gauge_handle = gauge!(mean_name); + // DEPRECATED: buffer-bytes-events-metrics + let legacy_max_gauge = gauge!(legacy_max_gauge_name); + legacy_max_gauge.set(max_value); + Self { + histogram: histogram!(histogram_name), + gauge: gauge!(gauge_name), + mean_gauge: EwmaGauge::new(mean_gauge_handle, ewma_alpha), + max_gauge, + legacy_max_gauge, + #[cfg(test)] + recorded_values, + } + } + } + + #[expect(clippy::cast_precision_loss)] + fn record(&self, value: usize) { + self.histogram.record(value as f64); + self.gauge.set(value as f64); + self.mean_gauge.record(value as f64); + #[cfg(test)] + if let Ok(mut recorded) = self.recorded_values.lock() { + recorded.push(value); + } + } +} + #[derive(Debug)] struct Inner { data: Arc>, limit: MemoryBufferSize, limiter: Arc, read_waker: Arc, + metrics: Option, } impl Clone for Inner { @@ -103,6 +209,50 @@ impl Clone for Inner { limit: self.limit, limiter: self.limiter.clone(), read_waker: self.read_waker.clone(), + metrics: self.metrics.clone(), + } + } +} + +impl Inner { + fn new( + limit: MemoryBufferSize, + metric_metadata: Option, + ewma_alpha: Option, + ) -> Self { + let read_waker = Arc::new(Notify::new()); + let metrics = metric_metadata.map(|metadata| Metrics::new(limit, metadata, ewma_alpha)); + match limit { + MemoryBufferSize::MaxEvents(max_events) => Inner { + data: Arc::new(ArrayQueue::new(max_events.get())), + limit, + limiter: Arc::new(Semaphore::new(max_events.get())), + read_waker, + metrics, + }, + MemoryBufferSize::MaxSize(max_bytes) => Inner { + data: Arc::new(SegQueue::new()), + limit, + limiter: Arc::new(Semaphore::new(max_bytes.get())), + read_waker, + metrics, + }, + } + } + + /// Records a send after acquiring all required permits. + /// + /// The `total` value represents the channel utilization after this send completes. It may be + /// greater than the configured limit because the channel intentionally allows a single + /// oversized payload to flow through rather than forcing the sender to split it. + fn send_with_permit(&mut self, total: usize, permits: OwnedSemaphorePermit, item: T) { + self.data.push((permits, item)); + self.read_waker.notify_one(); + // Due to the race between getting the available capacity, acquiring the permits, and the + // above push, the total may be inaccurate. Record it anyways as the histogram totals will + // _eventually_ converge on a true picture of the buffer utilization. + if let Some(metrics) = self.metrics.as_ref() { + metrics.record(total); } } } @@ -115,7 +265,7 @@ pub struct LimitedSender { impl LimitedSender { #[allow(clippy::cast_possible_truncation)] - fn get_required_permits_for_item(&self, item: &T) -> u32 { + fn calc_required_permits(&self, item: &T) -> (usize, usize, u32) { // We have to limit the number of permits we ask for to the overall limit since we're always // willing to store more items than the limit if the queue is entirely empty, because // otherwise we might deadlock ourselves by not being able to send a single item. @@ -123,7 +273,8 @@ impl LimitedSender { MemoryBufferSize::MaxSize(max_size) => (max_size, item.allocated_bytes()), MemoryBufferSize::MaxEvents(max_events) => (max_events, item.event_count()), }; - cmp::min(limit.get(), value) as u32 + let limit = limit.get(); + (limit, value, cmp::min(limit, value) as u32) } /// Gets the number of items that this channel could accept. @@ -139,23 +290,22 @@ impl LimitedSender { /// with the given `item`. pub async fn send(&mut self, item: T) -> Result<(), SendError> { // Calculate how many permits we need, and wait until we can acquire all of them. - let permits_required = self.get_required_permits_for_item(&item); - let Ok(permits) = self + let (limit, count, permits_required) = self.calc_required_permits(&item); + let in_use = limit.saturating_sub(self.available_capacity()); + match self .inner .limiter .clone() .acquire_many_owned(permits_required) .await - else { - return Err(SendError(item)); - }; - - self.inner.data.push((permits, item)); - self.inner.read_waker.notify_one(); - - trace!("Sent item."); - - Ok(()) + { + Ok(permits) => { + self.inner.send_with_permit(in_use + count, permits, item); + trace!("Sent item."); + Ok(()) + } + Err(_) => Err(SendError(item)), + } } /// Attempts to send an item into the channel. @@ -172,28 +322,22 @@ impl LimitedSender { /// Will panic if adding ack amount overflows. pub fn try_send(&mut self, item: T) -> Result<(), TrySendError> { // Calculate how many permits we need, and try to acquire them all without waiting. - let permits_required = self.get_required_permits_for_item(&item); - let permits = match self + let (limit, count, permits_required) = self.calc_required_permits(&item); + let in_use = limit.saturating_sub(self.available_capacity()); + match self .inner .limiter .clone() .try_acquire_many_owned(permits_required) { - Ok(permits) => permits, - Err(ae) => { - return match ae { - TryAcquireError::NoPermits => Err(TrySendError::InsufficientCapacity(item)), - TryAcquireError::Closed => Err(TrySendError::Disconnected(item)), - }; + Ok(permits) => { + self.inner.send_with_permit(in_use + count, permits, item); + trace!("Attempt to send item succeeded."); + Ok(()) } - }; - - self.inner.data.push((permits, item)); - self.inner.read_waker.notify_one(); - - trace!("Attempt to send item succeeded."); - - Ok(()) + Err(TryAcquireError::NoPermits) => Err(TrySendError::InsufficientCapacity(item)), + Err(TryAcquireError::Closed) => Err(TrySendError::Disconnected(item)), + } } } @@ -269,21 +413,10 @@ impl Drop for LimitedReceiver { pub fn limited( limit: MemoryBufferSize, + metric_metadata: Option, + ewma_alpha: Option, ) -> (LimitedSender, LimitedReceiver) { - let inner = match limit { - MemoryBufferSize::MaxEvents(max_events) => Inner { - data: Arc::new(ArrayQueue::new(max_events.get())), - limit, - limiter: Arc::new(Semaphore::new(max_events.get())), - read_waker: Arc::new(Notify::new()), - }, - MemoryBufferSize::MaxSize(max_size) => Inner { - data: Arc::new(SegQueue::new()), - limit, - limiter: Arc::new(Semaphore::new(max_size.get())), - read_waker: Arc::new(Notify::new()), - }, - }; + let inner = Inner::new(limit, metric_metadata, ewma_alpha); let sender = LimitedSender { inner: inner.clone(), @@ -301,7 +434,7 @@ mod tests { use tokio_test::{assert_pending, assert_ready, task::spawn}; use vector_common::byte_size_of::ByteSizeOf; - use super::limited; + use super::{ChannelMetricMetadata, limited}; use crate::{ MemoryBufferSize, test::MultiEventRecord, @@ -310,7 +443,8 @@ mod tests { #[tokio::test] async fn send_receive() { - let (mut tx, mut rx) = limited(MemoryBufferSize::MaxEvents(NonZeroUsize::new(2).unwrap())); + let limit = MemoryBufferSize::MaxEvents(NonZeroUsize::new(2).unwrap()); + let (mut tx, mut rx) = limited(limit, None, None); assert_eq!(2, tx.available_capacity()); @@ -336,6 +470,23 @@ mod tests { assert_eq!(Some(Sample::new(42)), assert_ready!(recv.poll())); } + #[tokio::test] + async fn records_utilization_on_send() { + let limit = MemoryBufferSize::MaxEvents(NonZeroUsize::new(2).unwrap()); + let (mut tx, mut rx) = limited( + limit, + Some(ChannelMetricMetadata::new("test_channel", None)), + None, + ); + + let metrics = tx.inner.metrics.as_ref().unwrap().recorded_values.clone(); + + tx.send(Sample::new(1)).await.expect("send should succeed"); + assert_eq!(metrics.lock().unwrap().last().copied(), Some(1)); + + let _ = rx.next().await; + } + #[test] fn test_limiting_by_byte_size() { let max_elements = 10; @@ -344,9 +495,8 @@ mod tests { let max_allowed_bytes = msg_size * max_elements; // With this configuration a maximum of exactly 10 messages can fit in the channel - let (mut tx, mut rx) = limited(MemoryBufferSize::MaxSize( - NonZeroUsize::new(max_allowed_bytes).unwrap(), - )); + let limit = MemoryBufferSize::MaxSize(NonZeroUsize::new(max_allowed_bytes).unwrap()); + let (mut tx, mut rx) = limited(limit, None, None); assert_eq!(max_allowed_bytes, tx.available_capacity()); @@ -379,7 +529,8 @@ mod tests { #[test] fn sender_waits_for_more_capacity_when_none_available() { - let (mut tx, mut rx) = limited(MemoryBufferSize::MaxEvents(NonZeroUsize::new(1).unwrap())); + let limit = MemoryBufferSize::MaxEvents(NonZeroUsize::new(1).unwrap()); + let (mut tx, mut rx) = limited(limit, None, None); assert_eq!(1, tx.available_capacity()); @@ -440,7 +591,8 @@ mod tests { #[test] fn sender_waits_for_more_capacity_when_partial_available() { - let (mut tx, mut rx) = limited(MemoryBufferSize::MaxEvents(NonZeroUsize::new(7).unwrap())); + let limit = MemoryBufferSize::MaxEvents(NonZeroUsize::new(7).unwrap()); + let (mut tx, mut rx) = limited(limit, None, None); assert_eq!(7, tx.available_capacity()); @@ -528,7 +680,8 @@ mod tests { #[test] fn empty_receiver_returns_none_when_last_sender_drops() { - let (mut tx, mut rx) = limited(MemoryBufferSize::MaxEvents(NonZeroUsize::new(1).unwrap())); + let limit = MemoryBufferSize::MaxEvents(NonZeroUsize::new(1).unwrap()); + let (mut tx, mut rx) = limited(limit, None, None); assert_eq!(1, tx.available_capacity()); @@ -570,8 +723,8 @@ mod tests { #[test] fn receiver_returns_none_once_empty_when_last_sender_drops() { - let (tx, mut rx) = - limited::(MemoryBufferSize::MaxEvents(NonZeroUsize::new(1).unwrap())); + let limit = MemoryBufferSize::MaxEvents(NonZeroUsize::new(1).unwrap()); + let (tx, mut rx) = limited::(limit, None, None); assert_eq!(1, tx.available_capacity()); @@ -600,7 +753,8 @@ mod tests { #[test] fn oversized_send_allowed_when_empty() { - let (mut tx, mut rx) = limited(MemoryBufferSize::MaxEvents(NonZeroUsize::new(1).unwrap())); + let limit = MemoryBufferSize::MaxEvents(NonZeroUsize::new(1).unwrap()); + let (mut tx, mut rx) = limited(limit, None, None); assert_eq!(1, tx.available_capacity()); @@ -632,7 +786,8 @@ mod tests { #[test] fn oversized_send_allowed_when_partial_capacity() { - let (mut tx, mut rx) = limited(MemoryBufferSize::MaxEvents(NonZeroUsize::new(2).unwrap())); + let limit = MemoryBufferSize::MaxEvents(NonZeroUsize::new(2).unwrap()); + let (mut tx, mut rx) = limited(limit, None, None); assert_eq!(2, tx.available_capacity()); diff --git a/lib/vector-buffers/src/topology/channel/mod.rs b/lib/vector-buffers/src/topology/channel/mod.rs index b6bb0bb9661a2..300e79d88c872 100644 --- a/lib/vector-buffers/src/topology/channel/mod.rs +++ b/lib/vector-buffers/src/topology/channel/mod.rs @@ -2,9 +2,12 @@ mod limited_queue; mod receiver; mod sender; -pub use limited_queue::{LimitedReceiver, LimitedSender, SendError, limited}; +pub use limited_queue::{ + ChannelMetricMetadata, LimitedReceiver, LimitedSender, SendError, limited, +}; pub use receiver::*; pub use sender::*; +pub use vector_common::stats::DEFAULT_EWMA_ALPHA; #[cfg(test)] mod tests; diff --git a/lib/vector-buffers/src/topology/channel/sender.rs b/lib/vector-buffers/src/topology/channel/sender.rs index cdfc41912f05b..c3eb22c3c952e 100644 --- a/lib/vector-buffers/src/topology/channel/sender.rs +++ b/lib/vector-buffers/src/topology/channel/sender.rs @@ -8,7 +8,7 @@ use vector_common::internal_event::{InternalEventHandle, Registered, register}; use super::limited_queue::LimitedSender; use crate::{ - Bufferable, WhenFull, + BufferInstrumentation, Bufferable, WhenFull, buffer_usage_data::BufferUsageHandle, internal_events::BufferSendDuration, variants::disk_v2::{self, ProductionFilesystem}, @@ -134,9 +134,11 @@ pub struct BufferSender { base: SenderAdapter, overflow: Option>>, when_full: WhenFull, - instrumentation: Option, + usage_instrumentation: Option, #[derivative(Debug = "ignore")] send_duration: Option>, + #[derivative(Debug = "ignore")] + custom_instrumentation: Option>>, } impl BufferSender { @@ -146,8 +148,9 @@ impl BufferSender { base, overflow: None, when_full, - instrumentation: None, + usage_instrumentation: None, send_duration: None, + custom_instrumentation: None, } } @@ -157,8 +160,9 @@ impl BufferSender { base, overflow: Some(Box::new(overflow)), when_full: WhenFull::Overflow, - instrumentation: None, + usage_instrumentation: None, send_duration: None, + custom_instrumentation: None, } } @@ -174,7 +178,7 @@ impl BufferSender { /// Configures this sender to instrument the items passing through it. pub fn with_usage_instrumentation(&mut self, handle: BufferUsageHandle) { - self.instrumentation = Some(handle); + self.usage_instrumentation = Some(handle); } /// Configures this sender to instrument the send duration. @@ -182,6 +186,11 @@ impl BufferSender { let _enter = span.enter(); self.send_duration = Some(register(BufferSendDuration { stage })); } + + /// Configures this sender to invoke a custom instrumentation hook. + pub fn with_custom_instrumentation(&mut self, instrumentation: impl BufferInstrumentation) { + self.custom_instrumentation = Some(Arc::new(instrumentation)); + } } impl BufferSender { @@ -196,14 +205,27 @@ impl BufferSender { } #[async_recursion] - pub async fn send(&mut self, item: T, send_reference: Option) -> crate::Result<()> { + pub async fn send( + &mut self, + mut item: T, + send_reference: Option, + ) -> crate::Result<()> { + if let Some(instrumentation) = self.custom_instrumentation.as_ref() { + instrumentation.on_send(&mut item); + } let item_sizing = self - .instrumentation + .usage_instrumentation .as_ref() .map(|_| (item.event_count(), item.size_of())); - let mut sent_to_base = true; let mut was_dropped = false; + + if let Some(instrumentation) = self.usage_instrumentation.as_ref() + && let Some((item_count, item_size)) = item_sizing + { + instrumentation + .increment_received_event_count_and_byte_size(item_count as u64, item_size as u64); + } match self.when_full { WhenFull::Block => self.base.send(item).await?, WhenFull::DropNewest => { @@ -213,7 +235,7 @@ impl BufferSender { } WhenFull::Overflow => { if let Some(item) = self.base.try_send(item).await? { - sent_to_base = false; + was_dropped = true; self.overflow .as_mut() .unwrap_or_else(|| unreachable!("overflow must exist")) @@ -223,30 +245,20 @@ impl BufferSender { } } - if (sent_to_base || was_dropped) - && let (Some(send_duration), Some(send_reference)) = - (self.send_duration.as_ref(), send_reference) + if let Some(instrumentation) = self.usage_instrumentation.as_ref() + && let Some((item_count, item_size)) = item_sizing + && was_dropped { - send_duration.emit(send_reference.elapsed()); + instrumentation.increment_dropped_event_count_and_byte_size( + item_count as u64, + item_size as u64, + true, + ); } - - if let Some(instrumentation) = self.instrumentation.as_ref() - && let Some((item_count, item_size)) = item_sizing + if let Some(send_duration) = self.send_duration.as_ref() + && let Some(send_reference) = send_reference { - if sent_to_base { - instrumentation.increment_received_event_count_and_byte_size( - item_count as u64, - item_size as u64, - ); - } - - if was_dropped { - instrumentation.increment_dropped_event_count_and_byte_size( - item_count as u64, - item_size as u64, - true, - ); - } + send_duration.emit(send_reference.elapsed()); } Ok(()) diff --git a/lib/vector-buffers/src/topology/channel/tests.rs b/lib/vector-buffers/src/topology/channel/tests.rs index 3ebfdda32add4..d36fd036f33ac 100644 --- a/lib/vector-buffers/src/topology/channel/tests.rs +++ b/lib/vector-buffers/src/topology/channel/tests.rs @@ -90,7 +90,7 @@ where #[tokio::test] async fn test_sender_block() { // Get a non-overflow buffer in blocking mode with a capacity of 3. - let (mut tx, rx, _) = build_buffer(3, WhenFull::Block, None).await; + let (mut tx, rx, _) = build_buffer(3, WhenFull::Block, None); // We should be able to send three messages through unimpeded. assert_current_send_capacity(&mut tx, Some(3), None); @@ -113,7 +113,7 @@ async fn test_sender_block() { #[tokio::test] async fn test_sender_drop_newest() { // Get a non-overflow buffer in "drop newest" mode with a capacity of 3. - let (mut tx, rx, _) = build_buffer(3, WhenFull::DropNewest, None).await; + let (mut tx, rx, _) = build_buffer(3, WhenFull::DropNewest, None); // We should be able to send three messages through unimpeded. assert_current_send_capacity(&mut tx, Some(3), None); @@ -138,7 +138,7 @@ async fn test_sender_drop_newest() { async fn test_sender_overflow_block() { // Get an overflow buffer, where the overflow buffer is in blocking mode, and both the base // and overflow buffers have a capacity of 2. - let (mut tx, rx, _) = build_buffer(2, WhenFull::Overflow, Some(WhenFull::Block)).await; + let (mut tx, rx, _) = build_buffer(2, WhenFull::Overflow, Some(WhenFull::Block)); // We should be able to send four message through unimpeded -- two for the base sender, and // two for the overflow sender. @@ -164,7 +164,7 @@ async fn test_sender_overflow_block() { async fn test_sender_overflow_drop_newest() { // Get an overflow buffer, where the overflow buffer is in "drop newest" mode, and both the // base and overflow buffers have a capacity of 2. - let (mut tx, rx, _) = build_buffer(2, WhenFull::Overflow, Some(WhenFull::DropNewest)).await; + let (mut tx, rx, _) = build_buffer(2, WhenFull::Overflow, Some(WhenFull::DropNewest)); // We should be able to send four message through unimpeded -- two for the base sender, and // two for the overflow sender. @@ -190,7 +190,7 @@ async fn test_sender_overflow_drop_newest() { #[tokio::test] async fn test_buffer_metrics_normal() { // Get a regular blocking buffer. - let (mut tx, rx, handle) = build_buffer(5, WhenFull::Block, None).await; + let (mut tx, rx, handle) = build_buffer(5, WhenFull::Block, None); // Send three items through, and make sure the buffer usage stats reflect that. assert_current_send_capacity(&mut tx, Some(5), None); @@ -217,7 +217,7 @@ async fn test_buffer_metrics_normal() { #[tokio::test] async fn test_buffer_metrics_drop_newest() { // Get a buffer that drops the newest items when full. - let (mut tx, rx, handle) = build_buffer(2, WhenFull::DropNewest, None).await; + let (mut tx, rx, handle) = build_buffer(2, WhenFull::DropNewest, None); // Send three items through, and make sure the buffer usage stats reflect that. assert_current_send_capacity(&mut tx, Some(2), None); diff --git a/lib/vector-buffers/src/topology/test_util.rs b/lib/vector-buffers/src/topology/test_util.rs index 89ae4917af3aa..684aecc977764 100644 --- a/lib/vector-buffers/src/topology/test_util.rs +++ b/lib/vector-buffers/src/topology/test_util.rs @@ -137,7 +137,7 @@ impl error::Error for BasicError {} /// If `mode` is set to `WhenFull::Overflow`, then the buffer will be set to overflow mode, with /// another in-memory channel buffer being used as the overflow buffer. The overflow buffer will /// also use the same capacity as the outer buffer. -pub(crate) async fn build_buffer( +pub(crate) fn build_buffer( capacity: usize, mode: WhenFull, overflow_mode: Option, @@ -154,27 +154,25 @@ pub(crate) async fn build_buffer( NonZeroUsize::new(capacity).expect("capacity must be nonzero"), overflow_mode, handle.clone(), - ) - .await; + None, + ); let (mut base_sender, mut base_receiver) = TopologyBuilder::standalone_memory_test( NonZeroUsize::new(capacity).expect("capacity must be nonzero"), WhenFull::Overflow, handle.clone(), - ) - .await; + None, + ); base_sender.switch_to_overflow(overflow_sender); base_receiver.switch_to_overflow(overflow_receiver); (base_sender, base_receiver) } - m => { - TopologyBuilder::standalone_memory_test( - NonZeroUsize::new(capacity).expect("capacity must be nonzero"), - m, - handle.clone(), - ) - .await - } + m => TopologyBuilder::standalone_memory_test( + NonZeroUsize::new(capacity).expect("capacity must be nonzero"), + m, + handle.clone(), + None, + ), }; (tx, rx, handle) diff --git a/lib/vector-buffers/src/variants/disk_v2/common.rs b/lib/vector-buffers/src/variants/disk_v2/common.rs index 1c50416afccfe..c7b4c2d4818bc 100644 --- a/lib/vector-buffers/src/variants/disk_v2/common.rs +++ b/lib/vector-buffers/src/variants/disk_v2/common.rs @@ -208,7 +208,6 @@ where /// /// Defaults to `usize::MAX`, or effectively no limit. Due to the internal design of the /// buffer, the effective maximum limit is around `max_data_file_size` * 2^16. - #[allow(dead_code)] pub fn max_buffer_size(mut self, amount: u64) -> Self { self.max_buffer_size = Some(amount); self diff --git a/lib/vector-buffers/src/variants/in_memory.rs b/lib/vector-buffers/src/variants/in_memory.rs index 1c9fd541f263e..93937a591b133 100644 --- a/lib/vector-buffers/src/variants/in_memory.rs +++ b/lib/vector-buffers/src/variants/in_memory.rs @@ -1,4 +1,4 @@ -use std::{error::Error, num::NonZeroUsize}; +use std::error::Error; use async_trait::async_trait; @@ -21,7 +21,8 @@ impl MemoryBuffer { MemoryBuffer { capacity } } - pub fn with_max_events(n: NonZeroUsize) -> Self { + #[cfg(test)] + pub fn with_max_events(n: std::num::NonZeroUsize) -> Self { Self { capacity: MemoryBufferSize::MaxEvents(n), } @@ -44,7 +45,7 @@ where usage_handle.set_buffer_limits(max_bytes, max_size); - let (tx, rx) = limited(self.capacity); + let (tx, rx) = limited(self.capacity, None, None); Ok((tx.into(), rx.into())) } } diff --git a/lib/vector-common-macros/Cargo.toml b/lib/vector-common-macros/Cargo.toml new file mode 100644 index 0000000000000..64ea946ecf3c1 --- /dev/null +++ b/lib/vector-common-macros/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "vector-common-macros" +version = "0.1.0" +edition = "2024" +license = "MPL-2.0" + +[lib] +proc-macro = true + +[dependencies] +proc-macro2 = { version = "1.0", default-features = false } +quote = { version = "1.0", default-features = false } +syn = { version = "2.0", default-features = false, features = ["full", "extra-traits", "visit-mut", "visit"] } diff --git a/lib/vector-common-macros/src/internal_event.rs b/lib/vector-common-macros/src/internal_event.rs new file mode 100644 index 0000000000000..58972625d64f2 --- /dev/null +++ b/lib/vector-common-macros/src/internal_event.rs @@ -0,0 +1,46 @@ +use proc_macro::TokenStream; +use quote::quote; +use syn::{Data, DeriveInput, parse_macro_input, spanned::Spanned}; + +/// Implements `NamedInternalEvent` for structs via `#[derive(NamedInternalEvent)]`. +pub fn derive_impl_named_internal_event(item: TokenStream) -> TokenStream { + let input = parse_macro_input!(item as DeriveInput); + + if !matches!(input.data, Data::Struct(_)) { + return syn::Error::new( + input.span(), + "#[derive(NamedInternalEvent)] can only be used with structs", + ) + .to_compile_error() + .into(); + } + + let DeriveInput { + ident, generics, .. + } = input; + let (impl_generics, ty_generics, where_clause) = generics.split_for_impl(); + + // Use a path that works from both vector-common (crate::internal_event) + // and from other crates using vector-lib (vector_lib::internal_event). + // For crates that don't depend on vector-lib but do depend on vector-common, + // we use vector_common::internal_event. + let pkg_name = std::env::var("CARGO_PKG_NAME").unwrap_or_default(); + let internal_event_path = if pkg_name == "vector-common" { + quote! { crate::internal_event } + } else if pkg_name.starts_with("vector-") || pkg_name == "codecs" || pkg_name == "dnstap-parser" + { + // Most vector-* crates depend on vector-common but not vector-lib + quote! { ::vector_common::internal_event } + } else { + // Main vector crate and its internal modules use vector_lib + quote! { ::vector_lib::internal_event } + }; + + let expanded = quote! { + impl #impl_generics #internal_event_path::NamedInternalEvent for #ident #ty_generics #where_clause { + fn name(&self) -> &'static str { stringify!(#ident) } + } + }; + + TokenStream::from(expanded) +} diff --git a/lib/vector-common-macros/src/lib.rs b/lib/vector-common-macros/src/lib.rs new file mode 100644 index 0000000000000..a34b86d9c94fc --- /dev/null +++ b/lib/vector-common-macros/src/lib.rs @@ -0,0 +1,37 @@ +#![deny(warnings)] + +use proc_macro::TokenStream; + +mod internal_event; + +/// Derives `NamedInternalEvent` so `InternalEvent::name()` returns a stable +/// compile-time identifier for the event type. +/// +/// Apply this derive to any struct that also implements `InternalEvent` or `RegisterInternalEvent`: +/// +/// ```ignore +/// use vector_lib::internal_event::{InternalEvent, NamedInternalEvent}; +/// +/// #[derive(Debug, NamedInternalEvent)] +/// pub struct UdpSendIncompleteError { +/// pub data_size: usize, +/// pub sent: usize, +/// } +/// +/// impl InternalEvent for UdpSendIncompleteError { +/// fn emit(self) { +/// // ... emit metrics/logging ... +/// } +/// } +/// +/// // Later, `UdpSendIncompleteError::name()` returns the string "UdpSendIncompleteError". +/// ``` +/// +/// Notes: +/// - Works with lifetimes and generics. +/// - The generated implementation returns `stringify!(TypeName)` which avoids +/// compiler-version-dependent module paths. +#[proc_macro_derive(NamedInternalEvent)] +pub fn derive_internal_event_name(input: TokenStream) -> TokenStream { + internal_event::derive_impl_named_internal_event(input) +} diff --git a/lib/vector-common/Cargo.toml b/lib/vector-common/Cargo.toml index 98f18180000e9..3604dfa94cc1e 100644 --- a/lib/vector-common/Cargo.toml +++ b/lib/vector-common/Cargo.toml @@ -41,6 +41,7 @@ crossbeam-utils.workspace = true derivative.workspace = true futures.workspace = true indexmap.workspace = true +itertools.workspace = true metrics.workspace = true paste.workspace = true pin-project.workspace = true @@ -51,7 +52,8 @@ stream-cancel = { version = "0.8.2", default-features = false } tokio = { workspace = true, features = ["macros", "time"] } tracing.workspace = true vrl.workspace = true -vector-config = { path = "../vector-config" } +vector-config.workspace = true +vector-common-macros.workspace = true [dev-dependencies] futures = { version = "0.3.31", default-features = false, features = ["async-await"] } diff --git a/lib/vector-common/src/atomic.rs b/lib/vector-common/src/atomic.rs new file mode 100644 index 0000000000000..a2d342771819a --- /dev/null +++ b/lib/vector-common/src/atomic.rs @@ -0,0 +1,49 @@ +use std::sync::atomic::{AtomicU64, Ordering}; + +use metrics::GaugeFn; + +/// Simple atomic wrapper for `f64` values. +#[derive(Debug)] +pub struct AtomicF64(AtomicU64); + +impl AtomicF64 { + /// Creates a new `AtomicF64` with the given initial value. + #[must_use] + pub fn new(init: f64) -> Self { + Self(AtomicU64::new(init.to_bits())) + } + + pub fn load(&self, order: Ordering) -> f64 { + f64::from_bits(self.0.load(order)) + } + + #[expect(clippy::missing_panics_doc, reason = "fetch_update always succeeds")] + pub fn fetch_update( + &self, + set_order: Ordering, + fetch_order: Ordering, + mut f: impl FnMut(f64) -> f64, + ) -> f64 { + f64::from_bits( + self.0 + .fetch_update(set_order, fetch_order, |x| { + Some(f(f64::from_bits(x)).to_bits()) + }) + .expect("fetch_update always succeeds"), + ) + } +} + +impl GaugeFn for AtomicF64 { + fn increment(&self, amount: f64) { + self.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |value| value + amount); + } + + fn decrement(&self, amount: f64) { + self.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |value| value - amount); + } + + fn set(&self, value: f64) { + self.0.store(f64::to_bits(value), Ordering::Relaxed); + } +} diff --git a/lib/vector-common/src/event_test_util.rs b/lib/vector-common/src/event_test_util.rs index fa759fcdf8a8e..4e2e9e728b440 100644 --- a/lib/vector-common/src/event_test_util.rs +++ b/lib/vector-common/src/event_test_util.rs @@ -1,4 +1,6 @@ -use std::{cell::RefCell, collections::HashSet, fmt::Write as _}; +use std::{cell::RefCell, collections::HashSet}; + +use itertools::Itertools as _; thread_local! { /// A buffer for recording internal events emitted by a single test. @@ -12,27 +14,25 @@ thread_local! { /// Will return `Err` if `pattern` is not found in the event record, or is found multiple times. pub fn contains_name_once(pattern: &str) -> Result<(), String> { EVENTS_RECORDED.with(|events| { - let mut n_events = 0; - let mut names = String::new(); - for event in &*events.borrow() { - if event.ends_with(pattern) { - if n_events > 0 { - names.push_str(", "); - } - n_events += 1; - _ = write!(names, "`{event}`"); + let events = events.borrow(); + let matches: Vec<_> = events + .iter() + .filter(|event| event_name_matches(event, pattern)) + .collect(); + match matches.len() { + 0 => Err(format!("Missing event {pattern:?}")), + 1 => Ok(()), + n => { + let names = matches + .into_iter() + .map(|event| format!("{event:?}")) + .join(", "); + Err(format!( + "Multiple ({n}) events matching {pattern:?}: ({names}). Hint! Don't use the `assert_x_` test \ + helpers on round-trip tests (tests that run more than a single component)." + )) } } - if n_events == 0 { - Err(format!("Missing event `{pattern}`")) - } else if n_events > 1 { - Err(format!( - "Multiple ({n_events}) events matching `{pattern}`: ({names}). Hint! Don't use the `assert_x_` \ - test helpers on round-trip tests (tests that run more than a single component)." - )) - } else { - Ok(()) - } }) } @@ -49,6 +49,19 @@ pub fn debug_print_events() { }); } +fn event_name_matches(event: &str, pattern: &str) -> bool { + let segment = event.rsplit_once("::").map_or(event, |(_, suffix)| suffix); + segment == pattern || (segment.ends_with(pattern) && !ignore_prefixed_match(segment, pattern)) +} + +fn ignore_prefixed_match(segment: &str, pattern: &str) -> bool { + // Buffer telemetry emits its own `BufferEvents{{Received|Sent}}` events for destinations in the + // topology. Component compliance only cares about the component-scoped + // `Events{{Received|Sent}}` signals, so we explicitly filter out the buffer-prefixed + // forms when matching these shared names. Other prefixes remain eligible. + matches!(pattern, "EventsReceived" | "EventsSent") && segment.starts_with("Buffer") +} + /// Record an emitted internal event. This is somewhat dumb at this /// point, just recording the pure string value of the `emit!` call /// parameter. At some point, making all internal events implement @@ -66,3 +79,71 @@ pub fn record_internal_event(event: &str) { EVENTS_RECORDED.with(|er| er.borrow_mut().insert(event.trim().into())); } + +#[cfg(test)] +mod tests { + use super::*; + + fn reset_events() { + clear_recorded_events(); + } + + fn insert_raw_event(event: &str) { + super::EVENTS_RECORDED.with(|events| { + events.borrow_mut().insert(event.into()); + }); + } + + #[test] + fn contains_name_once_accepts_exact_match() { + reset_events(); + record_internal_event("EventsReceived"); + assert!(contains_name_once("EventsReceived").is_ok()); + } + + #[test] + fn contains_name_once_ignores_prefix_matches() { + reset_events(); + record_internal_event("EventsReceived"); + record_internal_event("BufferEventsReceived"); + + assert!(contains_name_once("EventsReceived").is_ok()); + } + + #[test] + fn contains_name_once_matches_module_qualified_names() { + reset_events(); + insert_raw_event("vector::internal_events::EventsSent"); + + assert!(contains_name_once("EventsSent").is_ok()); + } + + #[test] + fn contains_name_once_still_flags_multiple_exact_matches() { + reset_events(); + record_internal_event("EventsSent"); + insert_raw_event("vector::internal_events::EventsSent"); + + let err = contains_name_once("EventsSent").unwrap_err(); + assert!( + err.contains("Multiple (2) events matching \"EventsSent\""), + "{err}" + ); + } + + #[test] + fn contains_name_once_matches_prefixed_component_events() { + reset_events(); + record_internal_event("SocketEventsReceived"); + + assert!(contains_name_once("EventsReceived").is_ok()); + } + + #[test] + fn contains_name_once_ignores_buffer_prefixed_events() { + reset_events(); + record_internal_event("BufferEventsReceived"); + + assert!(contains_name_once("EventsReceived").is_err()); + } +} diff --git a/lib/vector-common/src/finalization.rs b/lib/vector-common/src/finalization.rs index 56310a37d31b4..9fc403c99f399 100644 --- a/lib/vector-common/src/finalization.rs +++ b/lib/vector-common/src/finalization.rs @@ -280,10 +280,12 @@ impl Drop for OwnedBatchNotifier { /// The status of an individual batch. #[derive(Copy, Clone, Debug, Eq, PartialEq)] #[repr(u8)] +#[derive(Default)] pub enum BatchStatus { /// All events in the batch were accepted. /// /// This is the default. + #[default] Delivered, /// At least one event in the batch had a transient error in delivery. Errored, @@ -291,12 +293,6 @@ pub enum BatchStatus { Rejected, } -impl Default for BatchStatus { - fn default() -> Self { - Self::Delivered - } -} - impl BatchStatus { /// Updates the delivery status based on another batch's delivery status, returning the result. /// @@ -320,10 +316,12 @@ impl BatchStatus { /// The status of an individual event. #[derive(Copy, Clone, Debug, Eq, PartialEq)] #[repr(u8)] +#[derive(Default)] pub enum EventStatus { /// All copies of this event were dropped without being finalized. /// /// This is the default. + #[default] Dropped, /// All copies of this event were delivered successfully. Delivered, @@ -335,12 +333,6 @@ pub enum EventStatus { Recorded, } -impl Default for EventStatus { - fn default() -> Self { - Self::Dropped - } -} - impl EventStatus { /// Updates the status based on another event's status, returning the result. /// diff --git a/lib/vector-common/src/internal_event/component_events_dropped.rs b/lib/vector-common/src/internal_event/component_events_dropped.rs index fde3434fc4b3e..bf45c054e45e1 100644 --- a/lib/vector-common/src/internal_event/component_events_dropped.rs +++ b/lib/vector-common/src/internal_event/component_events_dropped.rs @@ -1,11 +1,12 @@ use metrics::{Counter, counter}; use super::{Count, InternalEvent, InternalEventHandle, RegisterInternalEvent}; +use crate::NamedInternalEvent; pub const INTENTIONAL: bool = true; pub const UNINTENTIONAL: bool = false; -#[derive(Debug)] +#[derive(Debug, NamedInternalEvent)] pub struct ComponentEventsDropped<'a, const INTENTIONAL: bool> { pub count: usize, pub reason: &'a str, @@ -16,10 +17,6 @@ impl InternalEvent for ComponentEventsDropped<'_, INTEN let count = self.count; self.register().emit(Count(count)); } - - fn name(&self) -> Option<&'static str> { - Some("ComponentEventsDropped") - } } impl<'a, const INTENTIONAL: bool> From<&'a str> for ComponentEventsDropped<'a, INTENTIONAL> { diff --git a/lib/vector-common/src/internal_event/component_events_timed_out.rs b/lib/vector-common/src/internal_event/component_events_timed_out.rs new file mode 100644 index 0000000000000..bf138dd1481c7 --- /dev/null +++ b/lib/vector-common/src/internal_event/component_events_timed_out.rs @@ -0,0 +1,23 @@ +use metrics::{Counter, counter}; + +use super::Count; + +crate::registered_event! { + ComponentEventsTimedOut { + reason: &'static str, + } => { + timed_out_events: Counter = counter!("component_timed_out_events_total"), + timed_out_requests: Counter = counter!("component_timed_out_requests_total"), + reason: &'static str = self.reason, + } + + fn emit(&self, data: Count) { + warn!( + message = "Events timed out", + events = data.0, + reason = self.reason, + ); + self.timed_out_events.increment(data.0 as u64); + self.timed_out_requests.increment(1); + } +} diff --git a/lib/vector-common/src/internal_event/mod.rs b/lib/vector-common/src/internal_event/mod.rs index 83c06260c0d5e..369c4b9063c88 100644 --- a/lib/vector-common/src/internal_event/mod.rs +++ b/lib/vector-common/src/internal_event/mod.rs @@ -2,6 +2,7 @@ mod bytes_received; mod bytes_sent; pub mod cached_event; pub mod component_events_dropped; +pub mod component_events_timed_out; mod events_received; mod events_sent; mod optional_tag; @@ -15,6 +16,7 @@ pub use bytes_sent::BytesSent; #[allow(clippy::module_name_repetitions)] pub use cached_event::{RegisterTaggedInternalEvent, RegisteredEventCache}; pub use component_events_dropped::{ComponentEventsDropped, INTENTIONAL, UNINTENTIONAL}; +pub use component_events_timed_out::ComponentEventsTimedOut; pub use events_received::{EventsReceived, EventsReceivedHandle}; pub use events_sent::{DEFAULT_OUTPUT, EventsSent, TaggedEventsSent}; pub use metrics::SharedString; @@ -24,24 +26,19 @@ pub use service::{CallError, PollReadyError}; use crate::json_size::JsonSize; -pub trait InternalEvent: Sized { - fn emit(self); +pub trait NamedInternalEvent { + fn name(&self) -> &'static str; +} - // Optional for backwards compat until all events implement this - fn name(&self) -> Option<&'static str> { - None - } +pub trait InternalEvent: NamedInternalEvent + Sized { + fn emit(self); } #[allow(clippy::module_name_repetitions)] -pub trait RegisterInternalEvent: Sized { +pub trait RegisterInternalEvent: NamedInternalEvent + Sized { type Handle: InternalEventHandle; fn register(self) -> Self::Handle; - - fn name(&self) -> Option<&'static str> { - None - } } #[allow(clippy::module_name_repetitions)] @@ -50,39 +47,9 @@ pub trait InternalEventHandle: Sized { fn emit(&self, data: Self::Data); } -// Sets the name of an event if it doesn't have one -pub struct DefaultName { - pub name: &'static str, - pub event: E, -} - -impl InternalEvent for DefaultName { - fn emit(self) { - self.event.emit(); - } - - fn name(&self) -> Option<&'static str> { - Some(self.event.name().unwrap_or(self.name)) - } -} - -impl RegisterInternalEvent for DefaultName { - type Handle = E::Handle; - - fn register(self) -> Self::Handle { - self.event.register() - } - - fn name(&self) -> Option<&'static str> { - Some(self.event.name().unwrap_or(self.name)) - } -} - #[cfg(any(test, feature = "test"))] pub fn emit(event: impl InternalEvent) { - if let Some(name) = event.name() { - super::event_test_util::record_internal_event(name); - } + super::event_test_util::record_internal_event(event.name()); event.emit(); } @@ -93,9 +60,7 @@ pub fn emit(event: impl InternalEvent) { #[cfg(any(test, feature = "test"))] pub fn register(event: E) -> E::Handle { - if let Some(name) = event.name() { - super::event_test_util::record_internal_event(name); - } + super::event_test_util::record_internal_event(event.name()); event.register() } @@ -195,7 +160,7 @@ impl From for SharedString { macro_rules! registered_event { // A registered event struct with no fields (zero-sized type). ($event:ident => $($tail:tt)*) => { - #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] + #[derive(Debug, Clone, Eq, Hash, $crate::NamedInternalEvent, Ord, PartialEq, PartialOrd)] pub struct $event; $crate::registered_event!(=> $event $($tail)*); @@ -203,7 +168,7 @@ macro_rules! registered_event { // A normal registered event struct. ($event:ident { $( $field:ident: $type:ty, )* } => $($tail:tt)*) => { - #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] + #[derive(Debug, Clone, Eq, Hash, $crate::NamedInternalEvent, Ord, PartialEq, PartialOrd)] pub struct $event { $( pub $field: $type, )* } @@ -232,10 +197,6 @@ macro_rules! registered_event { impl $crate::internal_event::RegisterInternalEvent for $event { type Handle = [<$event Handle>]; - fn name(&self) -> Option<&'static str> { - Some(stringify!($event)) - } - fn register($slf) -> Self::Handle { Self::Handle { $( $field: $value, )* diff --git a/lib/vector-common/src/internal_event/service.rs b/lib/vector-common/src/internal_event/service.rs index bf1fdd5234d84..93a3c9b745722 100644 --- a/lib/vector-common/src/internal_event/service.rs +++ b/lib/vector-common/src/internal_event/service.rs @@ -1,8 +1,9 @@ use metrics::counter; use super::{ComponentEventsDropped, InternalEvent, UNINTENTIONAL, emit, error_stage, error_type}; +use crate::NamedInternalEvent; -#[derive(Debug)] +#[derive(Debug, NamedInternalEvent)] pub struct PollReadyError { pub error: E, } @@ -22,13 +23,9 @@ impl InternalEvent for PollReadyError { ) .increment(1); } - - fn name(&self) -> Option<&'static str> { - Some("ServicePollReadyError") - } } -#[derive(Debug)] +#[derive(Debug, NamedInternalEvent)] pub struct CallError { pub error: E, pub request_id: usize, @@ -57,8 +54,4 @@ impl InternalEvent for CallError { count: self.count, }); } - - fn name(&self) -> Option<&'static str> { - Some("ServiceCallError") - } } diff --git a/lib/vector-common/src/lib.rs b/lib/vector-common/src/lib.rs index 3e6d179f13b95..099a068578090 100644 --- a/lib/vector-common/src/lib.rs +++ b/lib/vector-common/src/lib.rs @@ -12,6 +12,8 @@ #![deny(unused_assignments)] #![deny(unused_comparisons)] +pub use vector_common_macros::NamedInternalEvent; + #[cfg(feature = "btreemap")] pub use vrl::btreemap; @@ -57,6 +59,8 @@ pub mod shutdown; #[cfg(feature = "sensitive_string")] pub mod sensitive_string; +pub mod atomic; +pub mod stats; pub mod trigger; #[macro_use] diff --git a/lib/vector-common/src/stats/ewma_gauge.rs b/lib/vector-common/src/stats/ewma_gauge.rs new file mode 100644 index 0000000000000..ebd635cad1f78 --- /dev/null +++ b/lib/vector-common/src/stats/ewma_gauge.rs @@ -0,0 +1,32 @@ +use std::sync::Arc; + +use metrics::Gauge; + +use super::AtomicEwma; + +/// The default alpha parameter used when constructing EWMA-backed gauges. +pub const DEFAULT_EWMA_ALPHA: f64 = 0.9; + +/// Couples a [`Gauge`] with an [`AtomicEwma`] so gauge readings reflect the EWMA. +#[derive(Clone, Debug)] +pub struct EwmaGauge { + gauge: Gauge, + // Note that the `Gauge` internally is equivalent to an `Arc` so we need to use the + // same semantics for the EWMA calculation as well. + ewma: Arc, +} + +impl EwmaGauge { + #[must_use] + pub fn new(gauge: Gauge, alpha: Option) -> Self { + let alpha = alpha.unwrap_or(DEFAULT_EWMA_ALPHA); + let ewma = Arc::new(AtomicEwma::new(alpha)); + Self { gauge, ewma } + } + + /// Records a new value, updates the EWMA, and sets the gauge accordingly. + pub fn record(&self, value: f64) { + let average = self.ewma.update(value); + self.gauge.set(average); + } +} diff --git a/src/stats.rs b/lib/vector-common/src/stats/mod.rs similarity index 69% rename from src/stats.rs rename to lib/vector-common/src/stats/mod.rs index 9bcd3253515a9..5c6c5bccc0240 100644 --- a/src/stats.rs +++ b/lib/vector-common/src/stats/mod.rs @@ -1,4 +1,13 @@ #![allow(missing_docs)] + +pub mod ewma_gauge; + +pub use ewma_gauge::{DEFAULT_EWMA_ALPHA, EwmaGauge}; + +use std::sync::atomic::Ordering; + +use crate::atomic::AtomicF64; + /// Exponentially Weighted Moving Average #[derive(Clone, Copy, Debug)] pub struct Ewma { @@ -7,11 +16,13 @@ pub struct Ewma { } impl Ewma { + #[must_use] pub const fn new(alpha: f64) -> Self { let average = None; Self { average, alpha } } + #[must_use] pub const fn average(&self) -> Option { self.average } @@ -35,6 +46,7 @@ pub struct EwmaDefault { } impl EwmaDefault { + #[must_use] pub const fn new(alpha: f64, initial_value: f64) -> Self { Self { average: initial_value, @@ -42,6 +54,7 @@ impl EwmaDefault { } } + #[must_use] pub const fn average(&self) -> f64 { self.average } @@ -67,21 +80,25 @@ pub struct MeanVariance { } impl EwmaVar { + #[must_use] pub const fn new(alpha: f64) -> Self { let state = None; Self { state, alpha } } + #[must_use] pub const fn state(&self) -> Option { self.state } #[cfg(test)] + #[must_use] pub fn average(&self) -> Option { self.state.map(|state| state.mean) } #[cfg(test)] + #[must_use] pub fn variance(&self) -> Option { self.state.map(|state| state.variance) } @@ -114,11 +131,16 @@ pub struct Mean { impl Mean { /// Update the and return the current average + #[expect( + clippy::cast_precision_loss, + reason = "We have to convert count to f64 for the calculation, it's okay to lose precision for very large counts." + )] pub fn update(&mut self, point: f64) { self.count += 1; self.mean += (point - self.mean) / self.count as f64; } + #[must_use] pub const fn average(&self) -> Option { match self.count { 0 => None, @@ -127,6 +149,43 @@ impl Mean { } } +/// Atomic EWMA that uses an `AtomicF64` to store the current average. +#[derive(Debug)] +pub struct AtomicEwma { + average: AtomicF64, + alpha: f64, +} + +impl AtomicEwma { + #[must_use] + pub fn new(alpha: f64) -> Self { + Self { + average: AtomicF64::new(f64::NAN), + alpha, + } + } + + pub fn update(&self, point: f64) -> f64 { + let mut result = f64::NAN; + self.average + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| { + let average = if current.is_nan() { + point + } else { + point.mul_add(self.alpha, current * (1.0 - self.alpha)) + }; + result = average; + average + }); + result + } + + pub fn average(&self) -> Option { + let value = self.average.load(Ordering::Relaxed); + if value.is_nan() { None } else { Some(value) } + } +} + #[cfg(test)] mod tests { use super::*; @@ -144,16 +203,17 @@ mod tests { } #[test] + #[expect(clippy::float_cmp, reason = "none of the values will be rounded")] fn ewma_update_works() { let mut mean = Ewma::new(0.5); assert_eq!(mean.average(), None); - mean.update(2.0); + assert_eq!(mean.update(2.0), 2.0); assert_eq!(mean.average(), Some(2.0)); - mean.update(2.0); + assert_eq!(mean.update(2.0), 2.0); assert_eq!(mean.average(), Some(2.0)); - mean.update(1.0); + assert_eq!(mean.update(1.0), 1.5); assert_eq!(mean.average(), Some(1.5)); - mean.update(2.0); + assert_eq!(mean.update(2.0), 1.75); assert_eq!(mean.average(), Some(1.75)); assert_eq!(mean.average, Some(1.75)); @@ -185,4 +245,15 @@ mod tests { }) ); } + + #[test] + #[expect(clippy::float_cmp, reason = "none of the values will be rounded")] + fn atomic_ewma_update_works() { + let ewma = AtomicEwma::new(0.5); + assert_eq!(ewma.average(), None); + assert_eq!(ewma.update(2.0), 2.0); + assert_eq!(ewma.average(), Some(2.0)); + assert_eq!(ewma.update(1.0), 1.5); + assert_eq!(ewma.average(), Some(1.5)); + } } diff --git a/lib/vector-config-macros/src/lib.rs b/lib/vector-config-macros/src/lib.rs index cfb10e09c258b..65757abe2f70a 100644 --- a/lib/vector-config-macros/src/lib.rs +++ b/lib/vector-config-macros/src/lib.rs @@ -10,7 +10,7 @@ mod configurable_component; /// Designates a type as being part of a Vector configuration. /// -/// This will automatically derive the [`Configurable`][vector-config::Configurable] trait for the given struct/enum, as +/// This will automatically derive the `Configurable` trait for the given struct/enum, as /// well as ensuring that serialization/deserialization (via `serde`) is derived. /// /// ## Basics diff --git a/lib/vector-config/src/named.rs b/lib/vector-config/src/named.rs index 9d791d3c1f9ba..48da2bb4fd6cb 100644 --- a/lib/vector-config/src/named.rs +++ b/lib/vector-config/src/named.rs @@ -1,7 +1,7 @@ /// A component with a well-known name. /// /// Users can derive this trait automatically by using the -/// [`component_name`][vector-config::component_name] macro on their structs/enums. +/// [`NamedComponent`][derive@crate::NamedComponent] derive macro on their structs/enums. pub trait NamedComponent { /// Gets the name of the component. fn get_component_name(&self) -> &'static str; diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml index 76969fbe42cb5..9efe3226aa7d6 100644 --- a/lib/vector-core/Cargo.toml +++ b/lib/vector-core/Cargo.toml @@ -51,7 +51,7 @@ snafu.workspace = true socket2.workspace = true tokio = { workspace = true, features = ["net"] } tokio-openssl = { version = "0.6.5", default-features = false } -tokio-stream = { version = "0.1", default-features = false, features = ["time"], optional = true } +tokio-stream = { workspace = true, features = ["time"], optional = true } tokio-util = { version = "0.7.0", default-features = false, features = ["time"] } toml.workspace = true tonic.workspace = true @@ -84,7 +84,7 @@ quickcheck = "1" quickcheck_macros = "1" proptest = "1.8" similar-asserts = "1.7.0" -tokio-test = "0.4.4" +tokio-test.workspace = true toml.workspace = true ndarray = "0.16.1" ndarray-stats = "0.6.0" diff --git a/lib/vector-core/src/config/global_options.rs b/lib/vector-core/src/config/global_options.rs index cc64b3a35bd35..93ad1d80f5934 100644 --- a/lib/vector-core/src/config/global_options.rs +++ b/lib/vector-core/src/config/global_options.rs @@ -139,6 +139,27 @@ pub struct GlobalOptions { /// the global default value, defined using `expire_metrics_secs`. #[serde(skip_serializing_if = "crate::serde::is_default")] pub expire_metrics_per_metric_set: Option>, + + /// The alpha value for the exponential weighted moving average (EWMA) of source and transform + /// buffer utilization metrics. + /// + /// This value specifies how much of the existing value is retained when each update is made. + /// Values closer to 1.0 result in the value adjusting slower to changes. The default value of + /// 0.9 is equivalent to a "half life" of 6-7 measurements. + /// + /// Must be between 0 and 1 exclusive (0 < alpha < 1). + #[serde(default, skip_serializing_if = "crate::serde::is_default")] + #[configurable(validation(range(min = 0.0, max = 1.0)))] + #[configurable(metadata(docs::advanced))] + pub buffer_utilization_ewma_alpha: Option, + + /// The interval, in seconds, at which the internal metrics cache for VRL is refreshed. + /// This must be set to be able to access metrics in VRL functions. + /// + /// Higher values lead to stale metric values from `get_vector_metric`, + /// `find_vector_metrics`, and `aggregate_vector_metrics` functions. + #[serde(default, skip_serializing_if = "crate::serde::is_default")] + pub metrics_storage_refresh_period: Option, } impl_generate_config_from_default!(GlobalOptions); @@ -287,6 +308,12 @@ impl GlobalOptions { expire_metrics: self.expire_metrics.or(with.expire_metrics), expire_metrics_secs: self.expire_metrics_secs.or(with.expire_metrics_secs), expire_metrics_per_metric_set: merged_expire_metrics_per_metric_set, + buffer_utilization_ewma_alpha: self + .buffer_utilization_ewma_alpha + .or(with.buffer_utilization_ewma_alpha), + metrics_storage_refresh_period: self + .metrics_storage_refresh_period + .or(with.metrics_storage_refresh_period), }) } else { Err(errors) diff --git a/lib/vector-core/src/config/mod.rs b/lib/vector-core/src/config/mod.rs index be04b8c58186f..c86848d7b0be5 100644 --- a/lib/vector-core/src/config/mod.rs +++ b/lib/vector-core/src/config/mod.rs @@ -393,7 +393,7 @@ impl From for AcknowledgementsConfig { } } -#[derive(Debug, Copy, Clone, PartialEq, Serialize, Deserialize, PartialOrd, Ord, Eq)] +#[derive(Debug, Copy, Clone, PartialEq, Serialize, Deserialize, PartialOrd, Ord, Eq, Default)] pub enum LogNamespace { /// Vector native namespacing /// @@ -405,6 +405,7 @@ pub enum LogNamespace { /// /// All data is set in the root of the event. Since this can lead /// to collisions, deserialized data has priority over metadata + #[default] Legacy, } @@ -420,9 +421,9 @@ impl From for LogNamespace { } } -impl Default for LogNamespace { - fn default() -> Self { - Self::Legacy +impl From for bool { + fn from(x: LogNamespace) -> Self { + x == LogNamespace::Vector } } diff --git a/lib/vector-core/src/event/log_event.rs b/lib/vector-core/src/event/log_event.rs index 1f507cf9ec881..336e15a43ab57 100644 --- a/lib/vector-core/src/event/log_event.rs +++ b/lib/vector-core/src/event/log_event.rs @@ -1200,15 +1200,15 @@ mod test { } #[test] - fn metadata_set_unique_uuid_v7_source_event_id() { - // Check if event id is UUID v7 + fn metadata_set_unique_uuid_v4_source_event_id() { + // Check if event id is UUID v4 let log1 = LogEvent::default(); assert_eq!( log1.metadata() .source_event_id() .expect("source_event_id should be auto-generated for new events") .get_version(), - Some(Version::SortRand) + Some(Version::Random) ); // Check if event id is unique on creation diff --git a/lib/vector-core/src/event/lua/metric.rs b/lib/vector-core/src/event/lua/metric.rs index 1bad80f1df191..a968fbed2fbd0 100644 --- a/lib/vector-core/src/event/lua/metric.rs +++ b/lib/vector-core/src/event/lua/metric.rs @@ -88,7 +88,7 @@ impl FromLua for TagValueSet { } Ok(Self::from(string_values)) } - LuaValue::String(x) => Ok(Self::from([x.to_string_lossy().to_string()])), + LuaValue::String(x) => Ok(Self::from([x.to_string_lossy().clone()])), _ => Err(mlua::Error::FromLuaConversionError { from: value.type_name(), to: String::from("metric tag value"), diff --git a/lib/vector-core/src/event/metadata.rs b/lib/vector-core/src/event/metadata.rs index cd9cecbe5501c..f860b03bb207b 100644 --- a/lib/vector-core/src/event/metadata.rs +++ b/lib/vector-core/src/event/metadata.rs @@ -40,7 +40,7 @@ pub(super) struct Inner { pub(crate) secrets: Secrets, #[serde(default, skip)] - finalizers: EventFinalizers, + pub(crate) finalizers: EventFinalizers, /// The id of the source pub(crate) source_id: Option>, @@ -60,7 +60,7 @@ pub(super) struct Inner { /// /// TODO(Jean): must not skip serialization to track schemas across restarts. #[serde(default = "default_schema_definition", skip)] - schema_definition: Arc, + pub(crate) schema_definition: Arc, /// A store of values that may be dropped during the encoding process but may be needed /// later on. The map is indexed by meaning. @@ -68,7 +68,7 @@ pub(super) struct Inner { /// we need to ensure it is still available later on for emitting metrics tagged by the service. /// This field could almost be keyed by `&'static str`, but because it needs to be deserializable /// we have to use `String`. - dropped_fields: ObjectMap, + pub(crate) dropped_fields: ObjectMap, /// Metadata to track the origin of metrics. This is always `None` for log and trace events. /// Only a small set of Vector sources and transforms explicitly set this field. @@ -253,7 +253,7 @@ impl Default for Inner { upstream_id: None, dropped_fields: ObjectMap::new(), datadog_origin_metadata: None, - source_event_id: Some(Uuid::now_v7()), + source_event_id: Some(Uuid::new_v4()), } } } @@ -264,7 +264,7 @@ impl Default for EventMetadata { } } -fn default_schema_definition() -> Arc { +pub(super) fn default_schema_definition() -> Arc { Arc::new(schema::Definition::new_with_default_metadata( Kind::any(), [LogNamespace::Legacy, LogNamespace::Vector], @@ -348,14 +348,8 @@ impl EventMetadata { inner.secrets.merge(other.secrets); // Update `source_event_id` if necessary. - match (inner.source_event_id, other.source_event_id) { - (None, Some(id)) => { - inner.source_event_id = Some(id); - } - (Some(uuid1), Some(uuid2)) if uuid2 < uuid1 => { - inner.source_event_id = Some(uuid2); - } - _ => {} // Keep the existing value. + if inner.source_event_id.is_none() { + inner.source_event_id = other.source_event_id; } } @@ -561,6 +555,7 @@ mod test { let m1 = EventMetadata::default(); let m2 = EventMetadata::default(); + // Always maintain the original source event id when merging, similar to how we handle other metadata. { let mut merged = m1.clone(); merged.merge(m2.clone()); @@ -570,7 +565,7 @@ mod test { { let mut merged = m2.clone(); merged.merge(m1.clone()); - assert_eq!(merged.source_event_id(), m1.source_event_id()); + assert_eq!(merged.source_event_id(), m2.source_event_id()); } } } diff --git a/lib/vector-core/src/event/metric/mod.rs b/lib/vector-core/src/event/metric/mod.rs index 884fbbdc1d81e..5e956f8042c02 100644 --- a/lib/vector-core/src/event/metric/mod.rs +++ b/lib/vector-core/src/event/metric/mod.rs @@ -896,6 +896,33 @@ mod test { assert!(!new_reset_histogram.subtract(&old_histogram)); } + #[test] + fn subtract_aggregated_histograms_bucket_redistribution() { + // Test for issue #24415: when total count is higher but individual bucket counts is sometimes lower + let old_histogram = Metric::new( + "histogram", + MetricKind::Absolute, + MetricValue::AggregatedHistogram { + count: 15, + sum: 15.0, + buckets: buckets!(1.0 => 10, 2.0 => 5), + }, + ); + + let mut new_histogram_with_redistribution = Metric::new( + "histogram", + MetricKind::Absolute, + MetricValue::AggregatedHistogram { + count: 20, + sum: 20.0, + // Total count is higher (20 > 15), but bucket1 count is lower (8 < 10) + buckets: buckets!(1.0 => 8, 2.0 => 12), + }, + ); + + assert!(!new_histogram_with_redistribution.subtract(&old_histogram)); + } + #[test] // `too_many_lines` is mostly just useful for production code but we're not // able to flag the lint on only for non-test. diff --git a/lib/vector-core/src/event/metric/tags.rs b/lib/vector-core/src/event/metric/tags.rs index a88531cf812cd..416f239af5ab5 100644 --- a/lib/vector-core/src/event/metric/tags.rs +++ b/lib/vector-core/src/event/metric/tags.rs @@ -94,9 +94,10 @@ type TagValueRef<'a> = Option<&'a str>; /// Tag values for a metric series. This may be empty, a single value, or a set of values. This is /// used to provide the storage for `TagValueSet`. -#[derive(Clone, Configurable, Debug, Eq, PartialEq)] +#[derive(Clone, Configurable, Debug, Eq, PartialEq, Default)] pub enum TagValueSet { /// This represents a set containing no value. + #[default] Empty, /// This represents a set containing a single value. This is stored separately to avoid the @@ -111,12 +112,6 @@ pub enum TagValueSet { Set(IndexSet), } -impl Default for TagValueSet { - fn default() -> Self { - Self::Empty - } -} - impl Display for TagValueSet { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { for (i, value) in self.iter().enumerate() { diff --git a/lib/vector-core/src/event/metric/value.rs b/lib/vector-core/src/event/metric/value.rs index ce21e80bbec24..7e301ae7943ce 100644 --- a/lib/vector-core/src/event/metric/value.rs +++ b/lib/vector-core/src/event/metric/value.rs @@ -327,6 +327,11 @@ impl MetricValue { // fewer values -- would not make sense, since buckets should never be able to have negative counts... and // it's not clear that a saturating subtraction is technically correct either. Instead, we avoid having to // make that decision, and simply force the metric to be reinitialized. + // + // We also check that each individual bucket count is >= the corresponding count in the + // other histogram, since bucket value redistribution (e.g., after a source restart or + // cache eviction) can cause individual buckets to have lower counts even when the total + // count is higher. Failing here leads to the metric being reinitialized. ( Self::AggregatedHistogram { buckets, @@ -343,7 +348,7 @@ impl MetricValue { && buckets .iter() .zip(buckets2.iter()) - .all(|(b1, b2)| b1.upper_limit == b2.upper_limit) => + .all(|(b1, b2)| b1.upper_limit == b2.upper_limit && b1.count >= b2.count) => { for (b1, b2) in buckets.iter_mut().zip(buckets2) { b1.count -= b2.count; diff --git a/lib/vector-core/src/event/proto.rs b/lib/vector-core/src/event/proto.rs index 548065447c291..6a4796906d287 100644 --- a/lib/vector-core/src/event/proto.rs +++ b/lib/vector-core/src/event/proto.rs @@ -16,6 +16,8 @@ pub use metric::Value as MetricValue; pub use proto_event::*; use vrl::value::{ObjectMap, Value as VrlValue}; +use super::EventFinalizers; +use super::metadata::{Inner, default_schema_definition}; use super::{EventMetadata, array, metric::MetricSketch}; impl event_array::Events { @@ -644,49 +646,49 @@ impl From for Metadata { impl From for EventMetadata { fn from(value: Metadata) -> Self { - let mut metadata = EventMetadata::default(); - - if let Some(value) = value.value.and_then(decode_value) { - *metadata.value_mut() = value; - } - - if let Some(source_id) = value.source_id { - metadata.set_source_id(Arc::new(source_id.into())); - } - - if let Some(source_type) = value.source_type { - metadata.set_source_type(source_type); - } - - if let Some(upstream_id) = value.upstream_id { - metadata.set_upstream_id(Arc::new(upstream_id.into())); - } - - if let Some(secrets) = value.secrets { - metadata.secrets_mut().merge(secrets.into()); - } - - if let Some(origin_metadata) = value.datadog_origin_metadata { - metadata = metadata.with_origin_metadata(origin_metadata.into()); - } - - let maybe_source_event_id = if value.source_event_id.is_empty() { + let Metadata { + value: metadata_value, + source_id, + source_type, + upstream_id, + secrets, + datadog_origin_metadata, + source_event_id, + } = value; + + let metadata_value = metadata_value.and_then(decode_value); + let source_id = source_id.map(|s| Arc::new(s.into())); + let upstream_id = upstream_id.map(|id| Arc::new(id.into())); + let secrets = secrets.map(Into::into); + let datadog_origin_metadata = datadog_origin_metadata.map(Into::into); + let source_event_id = if source_event_id.is_empty() { None } else { - match Uuid::from_slice(&value.source_event_id) { + match Uuid::from_slice(&source_event_id) { Ok(id) => Some(id), Err(error) => { error!( - message = "Failed to parse source_event_id: {}", - %error + %error, + source_event_id = %String::from_utf8_lossy(&source_event_id), + "Failed to parse source_event_id.", ); None } } }; - metadata = metadata.with_source_event_id(maybe_source_event_id); - metadata + EventMetadata(Arc::new(Inner { + value: metadata_value.unwrap_or_else(|| vrl::value::Value::Object(ObjectMap::new())), + secrets: secrets.unwrap_or_default(), + finalizers: EventFinalizers::default(), + source_id, + source_type: source_type.map(Into::into), + upstream_id, + schema_definition: default_schema_definition(), + dropped_fields: ObjectMap::new(), + datadog_origin_metadata, + source_event_id, + })) } } diff --git a/lib/vector-core/src/event/trace.rs b/lib/vector-core/src/event/trace.rs index 276bac75668b5..026751cb98816 100644 --- a/lib/vector-core/src/event/trace.rs +++ b/lib/vector-core/src/event/trace.rs @@ -131,6 +131,12 @@ impl From for TraceEvent { } } +impl From for LogEvent { + fn from(trace: TraceEvent) -> Self { + trace.0 + } +} + impl From for TraceEvent { fn from(map: ObjectMap) -> Self { Self(map.into()) diff --git a/lib/vector-core/src/fanout.rs b/lib/vector-core/src/fanout.rs index b6061774c2564..0fd5cb034b703 100644 --- a/lib/vector-core/src/fanout.rs +++ b/lib/vector-core/src/fanout.rs @@ -482,28 +482,29 @@ mod tests { test_util::{collect_ready, collect_ready_events}, }; - async fn build_sender_pair( + fn build_sender_pair( capacity: usize, ) -> (BufferSender, BufferReceiver) { TopologyBuilder::standalone_memory( NonZeroUsize::new(capacity).expect("capacity must be nonzero"), WhenFull::Block, &Span::current(), + None, + None, ) - .await } - async fn build_sender_pairs( + fn build_sender_pairs( capacities: &[usize], ) -> Vec<(BufferSender, BufferReceiver)> { let mut pairs = Vec::new(); for capacity in capacities { - pairs.push(build_sender_pair(*capacity).await); + pairs.push(build_sender_pair(*capacity)); } pairs } - async fn fanout_from_senders( + fn fanout_from_senders( capacities: &[usize], ) -> ( Fanout, @@ -511,7 +512,7 @@ mod tests { Vec>, ) { let (mut fanout, control) = Fanout::new(); - let pairs = build_sender_pairs(capacities).await; + let pairs = build_sender_pairs(capacities); let mut receivers = Vec::new(); for (i, (sender, receiver)) in pairs.into_iter().enumerate() { @@ -522,13 +523,13 @@ mod tests { (fanout, control, receivers) } - async fn add_sender_to_fanout( + fn add_sender_to_fanout( fanout: &mut Fanout, receivers: &mut Vec>, sender_id: usize, capacity: usize, ) { - let (sender, receiver) = build_sender_pair(capacity).await; + let (sender, receiver) = build_sender_pair(capacity); receivers.push(receiver); fanout.add(ComponentKey::from(sender_id.to_string()), sender); @@ -542,13 +543,13 @@ mod tests { .expect("sending control message should not fail"); } - async fn replace_sender_in_fanout( + fn replace_sender_in_fanout( control: &UnboundedSender, receivers: &mut [BufferReceiver], sender_id: usize, capacity: usize, ) -> BufferReceiver { - let (sender, receiver) = build_sender_pair(capacity).await; + let (sender, receiver) = build_sender_pair(capacity); let old_receiver = mem::replace(&mut receivers[sender_id], receiver); control @@ -567,13 +568,13 @@ mod tests { old_receiver } - async fn start_sender_replace( + fn start_sender_replace( control: &UnboundedSender, receivers: &mut [BufferReceiver], sender_id: usize, capacity: usize, ) -> (BufferReceiver, BufferSender) { - let (sender, receiver) = build_sender_pair(capacity).await; + let (sender, receiver) = build_sender_pair(capacity); let old_receiver = mem::replace(&mut receivers[sender_id], receiver); control @@ -616,7 +617,7 @@ mod tests { #[tokio::test] async fn fanout_writes_to_all() { - let (mut fanout, _, receivers) = fanout_from_senders(&[2, 2]).await; + let (mut fanout, _, receivers) = fanout_from_senders(&[2, 2]); let events = make_event_array(2); let clones = events.clone(); @@ -632,7 +633,7 @@ mod tests { #[tokio::test] async fn fanout_notready() { - let (mut fanout, _, mut receivers) = fanout_from_senders(&[2, 1, 2]).await; + let (mut fanout, _, mut receivers) = fanout_from_senders(&[2, 1, 2]); let events = make_events(2); // First send should immediately complete because all senders have capacity: @@ -661,7 +662,7 @@ mod tests { #[tokio::test] async fn fanout_grow() { - let (mut fanout, _, mut receivers) = fanout_from_senders(&[4, 4]).await; + let (mut fanout, _, mut receivers) = fanout_from_senders(&[4, 4]); let events = make_events(3); // Send in the first two events to our initial two senders: @@ -675,7 +676,7 @@ mod tests { .expect("should not fail"); // Now add a third sender: - add_sender_to_fanout(&mut fanout, &mut receivers, 2, 4).await; + add_sender_to_fanout(&mut fanout, &mut receivers, 2, 4); // Send in the last event which all three senders will now get: fanout @@ -696,7 +697,7 @@ mod tests { #[tokio::test] async fn fanout_shrink() { - let (mut fanout, control, receivers) = fanout_from_senders(&[4, 4]).await; + let (mut fanout, control, receivers) = fanout_from_senders(&[4, 4]); let events = make_events(3); // Send in the first two events to our initial two senders: @@ -772,7 +773,7 @@ mod tests { ]; for (sender_id, should_complete, expected_last_seen) in cases { - let (mut fanout, control, mut receivers) = fanout_from_senders(&[2, 1, 2]).await; + let (mut fanout, control, mut receivers) = fanout_from_senders(&[2, 1, 2]); // First send should immediately complete because all senders have capacity: let mut first_send = spawn(fanout.send(events[0].clone().into(), None)); @@ -827,7 +828,7 @@ mod tests { #[tokio::test] async fn fanout_replace() { - let (mut fanout, control, mut receivers) = fanout_from_senders(&[4, 4, 4]).await; + let (mut fanout, control, mut receivers) = fanout_from_senders(&[4, 4, 4]); let events = make_events(3); // First two sends should immediately complete because all senders have capacity: @@ -841,7 +842,7 @@ mod tests { .expect("should not fail"); // Replace the first sender with a brand new one before polling again: - let old_first_receiver = replace_sender_in_fanout(&control, &mut receivers, 0, 4).await; + let old_first_receiver = replace_sender_in_fanout(&control, &mut receivers, 0, 4); // And do the third send which should also complete since all senders still have capacity: fanout @@ -868,7 +869,7 @@ mod tests { #[tokio::test] async fn fanout_wait() { - let (mut fanout, control, mut receivers) = fanout_from_senders(&[4, 4]).await; + let (mut fanout, control, mut receivers) = fanout_from_senders(&[4, 4]); let events = make_events(3); // First two sends should immediately complete because all senders have capacity: @@ -881,7 +882,7 @@ mod tests { // doesn't let any writes through until we replace it properly. We get back the receiver // we've replaced, but also the sender that we want to eventually install: let (old_first_receiver, new_first_sender) = - start_sender_replace(&control, &mut receivers, 0, 4).await; + start_sender_replace(&control, &mut receivers, 0, 4); // Third send should return pending because now we have an in-flight replacement: let mut third_send = spawn(fanout.send(events[2].clone().into(), None)); diff --git a/lib/vector-core/src/lib.rs b/lib/vector-core/src/lib.rs index ef63141d102ff..e12b4dc047519 100644 --- a/lib/vector-core/src/lib.rs +++ b/lib/vector-core/src/lib.rs @@ -37,6 +37,7 @@ pub mod schema; pub mod serde; pub mod sink; pub mod source; +pub mod source_sender; pub mod tcp; #[cfg(test)] mod test_util; @@ -68,19 +69,6 @@ pub(crate) fn float_eq(l_value: f64, r_value: f64) -> bool { } // These macros aren't actually usable in lib crates without some `vector_lib` shenanigans. -// This test version won't be needed once all `InternalEvent`s implement `name()`. -#[cfg(feature = "test")] -#[macro_export] -macro_rules! emit { - ($event:expr) => { - vector_lib::internal_event::emit(vector_lib::internal_event::DefaultName { - event: $event, - name: stringify!($event), - }) - }; -} - -#[cfg(not(feature = "test"))] #[macro_export] macro_rules! emit { ($event:expr) => { @@ -88,18 +76,6 @@ macro_rules! emit { }; } -#[cfg(feature = "test")] -#[macro_export] -macro_rules! register { - ($event:expr) => { - vector_lib::internal_event::register(vector_lib::internal_event::DefaultName { - event: $event, - name: stringify!($event), - }) - }; -} - -#[cfg(not(feature = "test"))] #[macro_export] macro_rules! register { ($event:expr) => { diff --git a/lib/vector-core/src/metrics/recency.rs b/lib/vector-core/src/metrics/recency.rs index dcb237d6206c5..e63de81c9b1e3 100644 --- a/lib/vector-core/src/metrics/recency.rs +++ b/lib/vector-core/src/metrics/recency.rs @@ -63,8 +63,9 @@ use metrics_util::{ }; use parking_lot::Mutex; use quanta::{Clock, Instant}; +use vector_common::atomic::AtomicF64; -use super::storage::{AtomicF64, Histogram}; +use super::storage::Histogram; /// The generation of a metric. /// diff --git a/lib/vector-core/src/metrics/storage.rs b/lib/vector-core/src/metrics/storage.rs index b102849a6f2a9..86cf82c1ef7ea 100644 --- a/lib/vector-core/src/metrics/storage.rs +++ b/lib/vector-core/src/metrics/storage.rs @@ -3,8 +3,9 @@ use std::sync::{ atomic::{AtomicU32, Ordering}, }; -use metrics::{GaugeFn, HistogramFn, atomics::AtomicU64}; +use metrics::{HistogramFn, atomics::AtomicU64}; use metrics_util::registry::Storage; +use vector_common::atomic::AtomicF64; use crate::event::{MetricValue, metric::Bucket}; @@ -28,61 +29,17 @@ impl Storage for VectorStorage { } } -#[derive(Debug)] -pub(super) struct AtomicF64 { - inner: AtomicU64, -} - -impl AtomicF64 { - fn new(init: f64) -> Self { - Self { - inner: AtomicU64::new(init.to_bits()), - } - } - - fn fetch_update( - &self, - set_order: Ordering, - fetch_order: Ordering, - mut f: impl FnMut(f64) -> f64, - ) { - self.inner - .fetch_update(set_order, fetch_order, |x| { - Some(f(f64::from_bits(x)).to_bits()) - }) - .expect("Cannot fail"); - } - - pub(super) fn load(&self, order: Ordering) -> f64 { - f64::from_bits(self.inner.load(order)) - } -} - -impl GaugeFn for AtomicF64 { - fn increment(&self, amount: f64) { - self.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |value| value + amount); - } - - fn decrement(&self, amount: f64) { - self.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |value| value - amount); - } - - fn set(&self, value: f64) { - self.inner.store(f64::to_bits(value), Ordering::Relaxed); - } -} - #[derive(Debug)] pub(super) struct Histogram { - buckets: Box<[(f64, AtomicU32); 20]>, + buckets: Box<[(f64, AtomicU32); 26]>, count: AtomicU64, sum: AtomicF64, } impl Histogram { - const MIN_BUCKET: f64 = 0.015_625; // (-6_f64).exp2() is not const yet - const MIN_BUCKET_EXP: f64 = -6.0; - const BUCKETS: usize = 20; + const MIN_BUCKET: f64 = 1.0 / (1 << 12) as f64; // f64::powi() is not const yet + const MIN_BUCKET_EXP: f64 = -12.0; + const BUCKETS: usize = 26; pub(crate) fn new() -> Self { // Box to avoid having this large array inline to the structure, blowing @@ -95,25 +52,31 @@ impl Histogram { // long-tail. This also lets us find the right bucket to record into using simple // constant-time math operations instead of a loop-and-compare construct. let buckets = Box::new([ - ((-6_f64).exp2(), AtomicU32::new(0)), - ((-5_f64).exp2(), AtomicU32::new(0)), - ((-4_f64).exp2(), AtomicU32::new(0)), - ((-3_f64).exp2(), AtomicU32::new(0)), - ((-2_f64).exp2(), AtomicU32::new(0)), - ((-1_f64).exp2(), AtomicU32::new(0)), - (0_f64.exp2(), AtomicU32::new(0)), - (1_f64.exp2(), AtomicU32::new(0)), - (2_f64.exp2(), AtomicU32::new(0)), - (3_f64.exp2(), AtomicU32::new(0)), - (4_f64.exp2(), AtomicU32::new(0)), - (5_f64.exp2(), AtomicU32::new(0)), - (6_f64.exp2(), AtomicU32::new(0)), - (7_f64.exp2(), AtomicU32::new(0)), - (8_f64.exp2(), AtomicU32::new(0)), - (9_f64.exp2(), AtomicU32::new(0)), - (10_f64.exp2(), AtomicU32::new(0)), - (11_f64.exp2(), AtomicU32::new(0)), - (12_f64.exp2(), AtomicU32::new(0)), + (2.0f64.powi(-12), AtomicU32::new(0)), + (2.0f64.powi(-11), AtomicU32::new(0)), + (2.0f64.powi(-10), AtomicU32::new(0)), + (2.0f64.powi(-9), AtomicU32::new(0)), + (2.0f64.powi(-8), AtomicU32::new(0)), + (2.0f64.powi(-7), AtomicU32::new(0)), + (2.0f64.powi(-6), AtomicU32::new(0)), + (2.0f64.powi(-5), AtomicU32::new(0)), + (2.0f64.powi(-4), AtomicU32::new(0)), + (2.0f64.powi(-3), AtomicU32::new(0)), + (2.0f64.powi(-2), AtomicU32::new(0)), + (2.0f64.powi(-1), AtomicU32::new(0)), + (2.0f64.powi(0), AtomicU32::new(0)), + (2.0f64.powi(1), AtomicU32::new(0)), + (2.0f64.powi(2), AtomicU32::new(0)), + (2.0f64.powi(3), AtomicU32::new(0)), + (2.0f64.powi(4), AtomicU32::new(0)), + (2.0f64.powi(5), AtomicU32::new(0)), + (2.0f64.powi(6), AtomicU32::new(0)), + (2.0f64.powi(7), AtomicU32::new(0)), + (2.0f64.powi(8), AtomicU32::new(0)), + (2.0f64.powi(9), AtomicU32::new(0)), + (2.0f64.powi(10), AtomicU32::new(0)), + (2.0f64.powi(11), AtomicU32::new(0)), + (2.0f64.powi(12), AtomicU32::new(0)), (f64::INFINITY, AtomicU32::new(0)), ]); Self { diff --git a/lib/vector-core/src/source_sender/builder.rs b/lib/vector-core/src/source_sender/builder.rs new file mode 100644 index 0000000000000..1992dc531ee9a --- /dev/null +++ b/lib/vector-core/src/source_sender/builder.rs @@ -0,0 +1,98 @@ +use std::{collections::HashMap, time::Duration}; + +use metrics::{Histogram, histogram}; +use vector_buffers::topology::channel::LimitedReceiver; +use vector_common::internal_event::DEFAULT_OUTPUT; + +use super::{CHUNK_SIZE, LAG_TIME_NAME, Output, SourceSender, SourceSenderItem}; +use crate::config::{ComponentKey, OutputId, SourceOutput}; + +pub struct Builder { + buf_size: usize, + default_output: Option, + named_outputs: HashMap, + lag_time: Option, + timeout: Option, + ewma_alpha: Option, +} + +impl Default for Builder { + fn default() -> Self { + Self { + buf_size: CHUNK_SIZE, + default_output: None, + named_outputs: Default::default(), + lag_time: Some(histogram!(LAG_TIME_NAME)), + timeout: None, + ewma_alpha: None, + } + } +} + +impl Builder { + #[must_use] + pub fn with_buffer(mut self, n: usize) -> Self { + self.buf_size = n; + self + } + + #[must_use] + pub fn with_timeout(mut self, timeout: Option) -> Self { + self.timeout = timeout; + self + } + + #[must_use] + pub fn with_ewma_alpha(mut self, alpha: Option) -> Self { + self.ewma_alpha = alpha; + self + } + + pub fn add_source_output( + &mut self, + output: SourceOutput, + component_key: ComponentKey, + ) -> LimitedReceiver { + let lag_time = self.lag_time.clone(); + let log_definition = output.schema_definition.clone(); + let output_id = OutputId { + component: component_key, + port: output.port.clone(), + }; + match output.port { + None => { + let (output, rx) = Output::new_with_buffer( + self.buf_size, + DEFAULT_OUTPUT.to_owned(), + lag_time, + log_definition, + output_id, + self.timeout, + self.ewma_alpha, + ); + self.default_output = Some(output); + rx + } + Some(name) => { + let (output, rx) = Output::new_with_buffer( + self.buf_size, + name.clone(), + lag_time, + log_definition, + output_id, + self.timeout, + self.ewma_alpha, + ); + self.named_outputs.insert(name, output); + rx + } + } + } + + pub fn build(self) -> SourceSender { + SourceSender { + default_output: self.default_output, + named_outputs: self.named_outputs, + } + } +} diff --git a/lib/vector-core/src/source_sender/errors.rs b/lib/vector-core/src/source_sender/errors.rs new file mode 100644 index 0000000000000..57f7be8752d64 --- /dev/null +++ b/lib/vector-core/src/source_sender/errors.rs @@ -0,0 +1,26 @@ +use std::fmt; + +use vector_buffers::topology::channel; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum SendError { + Timeout, + Closed, +} + +impl From> for SendError { + fn from(_: channel::SendError) -> Self { + Self::Closed + } +} + +impl fmt::Display for SendError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Timeout => f.write_str("Send timed out."), + Self::Closed => f.write_str("Sender is closed."), + } + } +} + +impl std::error::Error for SendError {} diff --git a/lib/vector-core/src/source_sender/mod.rs b/lib/vector-core/src/source_sender/mod.rs new file mode 100644 index 0000000000000..c8af2db8bbf87 --- /dev/null +++ b/lib/vector-core/src/source_sender/mod.rs @@ -0,0 +1,25 @@ +#![allow( + missing_docs, + clippy::missing_errors_doc, + clippy::doc_markdown, + clippy::missing_panics_doc +)] + +mod builder; +mod errors; +mod output; +mod sender; +#[cfg(test)] +mod tests; + +pub use builder::Builder; +pub use errors::SendError; +use output::Output; +pub use sender::{SourceSender, SourceSenderItem}; + +pub const CHUNK_SIZE: usize = 1000; + +#[cfg(any(test, feature = "test"))] +const TEST_BUFFER_SIZE: usize = 100; + +const LAG_TIME_NAME: &str = "source_lag_time_seconds"; diff --git a/lib/vector-core/src/source_sender/output.rs b/lib/vector-core/src/source_sender/output.rs new file mode 100644 index 0000000000000..4b9f29efc1f58 --- /dev/null +++ b/lib/vector-core/src/source_sender/output.rs @@ -0,0 +1,283 @@ +use std::{ + fmt, + num::NonZeroUsize, + sync::Arc, + time::{Duration, Instant}, +}; + +use chrono::Utc; +use futures::{Stream, StreamExt as _}; +use metrics::Histogram; +use tracing::Span; +use vector_buffers::{ + config::MemoryBufferSize, + topology::channel::{self, ChannelMetricMetadata, LimitedReceiver, LimitedSender}, +}; +use vector_common::{ + byte_size_of::ByteSizeOf, + internal_event::{ + self, ComponentEventsDropped, ComponentEventsTimedOut, Count, CountByteSize, EventsSent, + InternalEventHandle as _, RegisterInternalEvent as _, Registered, UNINTENTIONAL, + }, +}; +use vrl::value::Value; + +use super::{CHUNK_SIZE, SendError, SourceSenderItem}; +use crate::{ + EstimatedJsonEncodedSizeOf, + config::{OutputId, log_schema}, + event::{Event, EventArray, EventContainer as _, EventRef, array}, + schema::Definition, +}; + +const UTILIZATION_METRIC_PREFIX: &str = "source_buffer"; + +/// UnsentEvents tracks the number of events yet to be sent in the buffer. This is used to +/// increment the appropriate counters when a future is not polled to completion. Particularly, +/// this is known to happen in a Warp server when a client sends a new HTTP request on a TCP +/// connection that already has a pending request. +/// +/// If its internal count is greater than 0 when dropped, the appropriate [ComponentEventsDropped] +/// event is emitted. +pub(super) struct UnsentEventCount { + count: usize, + span: Span, +} + +impl UnsentEventCount { + fn new(count: usize) -> Self { + Self { + count, + span: Span::current(), + } + } + + const fn decr(&mut self, count: usize) { + self.count = self.count.saturating_sub(count); + } + + const fn discard(&mut self) { + self.count = 0; + } + + fn timed_out(&mut self) { + ComponentEventsTimedOut { + reason: "Source send timed out.", + } + .register() + .emit(Count(self.count)); + self.count = 0; + } +} + +impl Drop for UnsentEventCount { + fn drop(&mut self) { + if self.count > 0 { + let _enter = self.span.enter(); + internal_event::emit(ComponentEventsDropped:: { + count: self.count, + reason: "Source send cancelled.", + }); + } + } +} + +#[derive(Clone)] +pub(super) struct Output { + sender: LimitedSender, + lag_time: Option, + events_sent: Registered, + /// The schema definition that will be attached to Log events sent through here + log_definition: Option>, + /// The OutputId related to this source sender. This is set as the `upstream_id` in + /// `EventMetadata` for all event sent through here. + id: Arc, + timeout: Option, +} + +#[expect(clippy::missing_fields_in_debug)] +impl fmt::Debug for Output { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt.debug_struct("Output") + .field("sender", &self.sender) + .field("output_id", &self.id) + .field("timeout", &self.timeout) + // `metrics::Histogram` is missing `impl Debug` + .finish() + } +} + +impl Output { + pub(super) fn new_with_buffer( + n: usize, + output: String, + lag_time: Option, + log_definition: Option>, + output_id: OutputId, + timeout: Option, + ewma_alpha: Option, + ) -> (Self, LimitedReceiver) { + let limit = MemoryBufferSize::MaxEvents(NonZeroUsize::new(n).unwrap()); + let metrics = ChannelMetricMetadata::new(UTILIZATION_METRIC_PREFIX, Some(output.clone())); + let (tx, rx) = channel::limited(limit, Some(metrics), ewma_alpha); + ( + Self { + sender: tx, + lag_time, + events_sent: internal_event::register(EventsSent::from(internal_event::Output( + Some(output.into()), + ))), + log_definition, + id: Arc::new(output_id), + timeout, + }, + rx, + ) + } + + pub(super) async fn send( + &mut self, + mut events: EventArray, + unsent_event_count: &mut UnsentEventCount, + ) -> Result<(), SendError> { + let send_reference = Instant::now(); + let reference = Utc::now().timestamp_millis(); + events + .iter_events() + .for_each(|event| self.emit_lag_time(event, reference)); + + events.iter_events_mut().for_each(|mut event| { + // attach runtime schema definitions from the source + if let Some(log_definition) = &self.log_definition { + event.metadata_mut().set_schema_definition(log_definition); + } + event.metadata_mut().set_upstream_id(Arc::clone(&self.id)); + }); + + let byte_size = events.estimated_json_encoded_size_of(); + let count = events.len(); + self.send_with_timeout(events, send_reference).await?; + self.events_sent.emit(CountByteSize(count, byte_size)); + unsent_event_count.decr(count); + Ok(()) + } + + async fn send_with_timeout( + &mut self, + events: EventArray, + send_reference: Instant, + ) -> Result<(), SendError> { + let item = SourceSenderItem { + events, + send_reference, + }; + if let Some(timeout) = self.timeout { + match tokio::time::timeout(timeout, self.sender.send(item)).await { + Ok(Ok(())) => Ok(()), + Ok(Err(error)) => Err(error.into()), + Err(_elapsed) => Err(SendError::Timeout), + } + } else { + self.sender.send(item).await.map_err(Into::into) + } + } + + pub(super) async fn send_event( + &mut self, + event: impl Into, + ) -> Result<(), SendError> { + let event: EventArray = event.into(); + // It's possible that the caller stops polling this future while it is blocked waiting + // on `self.send()`. When that happens, we use `UnsentEventCount` to correctly emit + // `ComponentEventsDropped` events. + let mut unsent_event_count = UnsentEventCount::new(event.len()); + self.send(event, &mut unsent_event_count) + .await + .inspect_err(|error| { + if let SendError::Timeout = error { + unsent_event_count.timed_out(); + } + }) + } + + pub(super) async fn send_event_stream(&mut self, events: S) -> Result<(), SendError> + where + S: Stream + Unpin, + E: Into + ByteSizeOf, + { + let mut stream = events.ready_chunks(CHUNK_SIZE); + while let Some(events) = stream.next().await { + self.send_batch(events.into_iter()).await?; + } + Ok(()) + } + + pub(super) async fn send_batch(&mut self, events: I) -> Result<(), SendError> + where + E: Into + ByteSizeOf, + I: IntoIterator, + ::IntoIter: ExactSizeIterator, + { + // It's possible that the caller stops polling this future while it is blocked waiting + // on `self.send()`. When that happens, we use `UnsentEventCount` to correctly emit + // `ComponentEventsDropped` events. + let events = events.into_iter().map(Into::into); + let mut unsent_event_count = UnsentEventCount::new(events.len()); + for events in array::events_into_arrays(events, Some(CHUNK_SIZE)) { + self.send(events, &mut unsent_event_count) + .await + .inspect_err(|error| match error { + SendError::Timeout => { + unsent_event_count.timed_out(); + } + SendError::Closed => { + // The unsent event count is discarded here because the callee emits the + // `StreamClosedError`. + unsent_event_count.discard(); + } + })?; + } + Ok(()) + } + + /// Calculate the difference between the reference time and the + /// timestamp stored in the given event reference, and emit the + /// different, as expressed in milliseconds, as a histogram. + pub(super) fn emit_lag_time(&self, event: EventRef<'_>, reference: i64) { + if let Some(lag_time_metric) = &self.lag_time { + let timestamp = match event { + EventRef::Log(log) => { + log_schema() + .timestamp_key_target_path() + .and_then(|timestamp_key| { + log.get(timestamp_key).and_then(get_timestamp_millis) + }) + } + EventRef::Metric(metric) => metric + .timestamp() + .map(|timestamp| timestamp.timestamp_millis()), + EventRef::Trace(trace) => { + log_schema() + .timestamp_key_target_path() + .and_then(|timestamp_key| { + trace.get(timestamp_key).and_then(get_timestamp_millis) + }) + } + }; + if let Some(timestamp) = timestamp { + // This will truncate precision for values larger than 2**52, but at that point the user + // probably has much larger problems than precision. + #[expect(clippy::cast_precision_loss)] + let lag_time = (reference - timestamp) as f64 / 1000.0; + lag_time_metric.record(lag_time); + } + } + } +} + +const fn get_timestamp_millis(value: &Value) -> Option { + match value { + Value::Timestamp(timestamp) => Some(timestamp.timestamp_millis()), + _ => None, + } +} diff --git a/lib/vector-core/src/source_sender/sender.rs b/lib/vector-core/src/source_sender/sender.rs new file mode 100644 index 0000000000000..88cb50172c70a --- /dev/null +++ b/lib/vector-core/src/source_sender/sender.rs @@ -0,0 +1,259 @@ +#[cfg(any(test, feature = "test"))] +use std::time::Duration; +use std::{collections::HashMap, time::Instant}; + +use futures::Stream; +#[cfg(any(test, feature = "test"))] +use futures::StreamExt as _; +#[cfg(any(test, feature = "test"))] +use metrics::histogram; +use vector_buffers::EventCount; +#[cfg(any(test, feature = "test"))] +use vector_buffers::topology::channel::LimitedReceiver; +#[cfg(any(test, feature = "test"))] +use vector_common::internal_event::DEFAULT_OUTPUT; +#[cfg(doc)] +use vector_common::internal_event::{ComponentEventsDropped, EventsSent}; +use vector_common::{ + byte_size_of::ByteSizeOf, + finalization::{AddBatchNotifier, BatchNotifier}, + json_size::JsonSize, +}; + +use super::{Builder, Output, SendError}; +#[cfg(any(test, feature = "test"))] +use super::{LAG_TIME_NAME, TEST_BUFFER_SIZE}; +use crate::{ + EstimatedJsonEncodedSizeOf, + event::{Event, EventArray, EventContainer, array::EventArrayIntoIter}, +}; +#[cfg(any(test, feature = "test"))] +use crate::{ + config::OutputId, + event::{EventStatus, into_event_stream}, +}; + +/// SourceSenderItem is a thin wrapper around [EventArray] used to track the send duration of a batch. +/// +/// This is needed because the send duration is calculated as the difference between when the batch +/// is sent from the origin component to when the batch is enqueued on the receiving component's input buffer. +/// For sources in particular, this requires the batch to be enqueued on two channels: the origin component's pump +/// channel and then the receiving component's input buffer. +#[derive(Debug)] +pub struct SourceSenderItem { + /// The batch of events to send. + pub events: EventArray, + /// Reference instant used to calculate send duration. + pub send_reference: Instant, +} + +impl AddBatchNotifier for SourceSenderItem { + fn add_batch_notifier(&mut self, notifier: BatchNotifier) { + self.events.add_batch_notifier(notifier); + } +} + +impl ByteSizeOf for SourceSenderItem { + fn allocated_bytes(&self) -> usize { + self.events.allocated_bytes() + } +} + +impl EventCount for SourceSenderItem { + fn event_count(&self) -> usize { + self.events.event_count() + } +} + +impl EstimatedJsonEncodedSizeOf for SourceSenderItem { + fn estimated_json_encoded_size_of(&self) -> JsonSize { + self.events.estimated_json_encoded_size_of() + } +} + +impl EventContainer for SourceSenderItem { + type IntoIter = EventArrayIntoIter; + + fn len(&self) -> usize { + self.events.len() + } + + fn into_events(self) -> Self::IntoIter { + self.events.into_events() + } +} + +impl From for EventArray { + fn from(val: SourceSenderItem) -> Self { + val.events + } +} + +#[derive(Debug, Clone)] +pub struct SourceSender { + // The default output is optional because some sources, e.g. `datadog_agent` + // and `opentelemetry`, can be configured to only output to named outputs. + pub(super) default_output: Option, + pub(super) named_outputs: HashMap, +} + +impl SourceSender { + pub fn builder() -> Builder { + Builder::default() + } + + #[cfg(any(test, feature = "test"))] + pub fn new_test_sender_with_options( + n: usize, + timeout: Option, + ) -> (Self, LimitedReceiver) { + let lag_time = Some(histogram!(LAG_TIME_NAME)); + let output_id = OutputId { + component: "test".to_string().into(), + port: None, + }; + let (default_output, rx) = Output::new_with_buffer( + n, + DEFAULT_OUTPUT.to_owned(), + lag_time, + None, + output_id, + timeout, + None, + ); + ( + Self { + default_output: Some(default_output), + named_outputs: Default::default(), + }, + rx, + ) + } + + #[cfg(any(test, feature = "test"))] + pub fn new_test() -> (Self, impl Stream + Unpin) { + let (pipe, recv) = Self::new_test_sender_with_options(TEST_BUFFER_SIZE, None); + let recv = recv.into_stream().flat_map(into_event_stream); + (pipe, recv) + } + + #[cfg(any(test, feature = "test"))] + pub fn new_test_finalize(status: EventStatus) -> (Self, impl Stream + Unpin) { + let (pipe, recv) = Self::new_test_sender_with_options(TEST_BUFFER_SIZE, None); + // In a source test pipeline, there is no sink to acknowledge + // events, so we have to add a map to the receiver to handle the + // finalization. + let recv = recv.into_stream().flat_map(move |mut item| { + item.events.iter_events_mut().for_each(|mut event| { + let metadata = event.metadata_mut(); + metadata.update_status(status); + metadata.update_sources(); + }); + into_event_stream(item) + }); + (pipe, recv) + } + + #[cfg(any(test, feature = "test"))] + pub fn new_test_errors( + error_at: impl Fn(usize) -> bool, + ) -> (Self, impl Stream + Unpin) { + let (pipe, recv) = Self::new_test_sender_with_options(TEST_BUFFER_SIZE, None); + // In a source test pipeline, there is no sink to acknowledge + // events, so we have to add a map to the receiver to handle the + // finalization. + let mut count: usize = 0; + let recv = recv.into_stream().flat_map(move |mut item| { + let status = if error_at(count) { + EventStatus::Errored + } else { + EventStatus::Delivered + }; + count += 1; + item.events.iter_events_mut().for_each(|mut event| { + let metadata = event.metadata_mut(); + metadata.update_status(status); + metadata.update_sources(); + }); + into_event_stream(item) + }); + (pipe, recv) + } + + #[cfg(any(test, feature = "test"))] + pub fn add_outputs( + &mut self, + status: EventStatus, + name: String, + ) -> impl Stream + Unpin + use<> { + // The lag_time parameter here will need to be filled in if this function is ever used for + // non-test situations. + let output_id = OutputId { + component: "test".to_string().into(), + port: Some(name.clone()), + }; + let (output, recv) = + Output::new_with_buffer(100, name.clone(), None, None, output_id, None, None); + let recv = recv.into_stream().map(move |mut item| { + item.events.iter_events_mut().for_each(|mut event| { + let metadata = event.metadata_mut(); + metadata.update_status(status); + metadata.update_sources(); + }); + item + }); + self.named_outputs.insert(name, output); + recv + } + + /// Get a mutable reference to the default output, panicking if none exists. + const fn default_output_mut(&mut self) -> &mut Output { + self.default_output.as_mut().expect("no default output") + } + + /// Send an event to the default output. + /// + /// This internally handles emitting [EventsSent] and [ComponentEventsDropped] events. + pub async fn send_event(&mut self, event: impl Into) -> Result<(), SendError> { + self.default_output_mut().send_event(event).await + } + + /// Send a stream of events to the default output. + /// + /// This internally handles emitting [EventsSent] and [ComponentEventsDropped] events. + pub async fn send_event_stream(&mut self, events: S) -> Result<(), SendError> + where + S: Stream + Unpin, + E: Into + ByteSizeOf, + { + self.default_output_mut().send_event_stream(events).await + } + + /// Send a batch of events to the default output. + /// + /// This internally handles emitting [EventsSent] and [ComponentEventsDropped] events. + pub async fn send_batch(&mut self, events: I) -> Result<(), SendError> + where + E: Into + ByteSizeOf, + I: IntoIterator, + ::IntoIter: ExactSizeIterator, + { + self.default_output_mut().send_batch(events).await + } + + /// Send a batch of events event to a named output. + /// + /// This internally handles emitting [EventsSent] and [ComponentEventsDropped] events. + pub async fn send_batch_named(&mut self, name: &str, events: I) -> Result<(), SendError> + where + E: Into + ByteSizeOf, + I: IntoIterator, + ::IntoIter: ExactSizeIterator, + { + self.named_outputs + .get_mut(name) + .expect("unknown output") + .send_batch(events) + .await + } +} diff --git a/lib/vector-core/src/source_sender/tests.rs b/lib/vector-core/src/source_sender/tests.rs new file mode 100644 index 0000000000000..3c3ffac73bb4a --- /dev/null +++ b/lib/vector-core/src/source_sender/tests.rs @@ -0,0 +1,299 @@ +use chrono::{DateTime, Duration, Utc}; +use rand::{Rng, rng}; +use std::time::{Duration as StdDuration, Instant}; +use tokio::time::timeout; +use vrl::event_path; + +use super::*; +use crate::{ + event::{Event, LogEvent, Metric, MetricKind, MetricValue, TraceEvent}, + metrics::{self, Controller}, +}; + +#[tokio::test] +async fn emits_lag_time_for_log() { + emit_and_test(|timestamp| { + let mut log = LogEvent::from("Log message"); + log.insert("timestamp", timestamp); + Event::Log(log) + }) + .await; +} + +#[tokio::test] +async fn emits_lag_time_for_metric() { + emit_and_test(|timestamp| { + Event::Metric( + Metric::new( + "name", + MetricKind::Absolute, + MetricValue::Gauge { value: 123.4 }, + ) + .with_timestamp(Some(timestamp)), + ) + }) + .await; +} + +#[tokio::test] +async fn emits_lag_time_for_trace() { + emit_and_test(|timestamp| { + let mut trace = TraceEvent::default(); + trace.insert(event_path!("timestamp"), timestamp); + Event::Trace(trace) + }) + .await; +} + +async fn emit_and_test(make_event: impl FnOnce(DateTime) -> Event) { + metrics::init_test(); + let (mut sender, _stream) = SourceSender::new_test(); + let millis = rng().random_range(10..10000); + let timestamp = Utc::now() - Duration::milliseconds(millis); + #[expect(clippy::cast_precision_loss)] + let expected = millis as f64 / 1000.0; + + let event = make_event(timestamp); + sender + .send_event(event) + .await + .expect("Send should not fail"); + + let lag_times = Controller::get() + .expect("There must be a controller") + .capture_metrics() + .into_iter() + .filter(|metric| metric.name() == "source_lag_time_seconds") + .collect::>(); + assert_eq!(lag_times.len(), 1); + + let lag_time = &lag_times[0]; + match lag_time.value() { + MetricValue::AggregatedHistogram { + buckets, + count, + sum, + } => { + let mut done = false; + for bucket in buckets { + if !done && bucket.upper_limit >= expected { + assert_eq!(bucket.count, 1); + done = true; + } else { + assert_eq!(bucket.count, 0); + } + } + assert_eq!(*count, 1); + assert!( + (*sum - expected).abs() <= 0.002, + "Histogram sum does not match expected sum: {} vs {}", + *sum, + expected, + ); + } + _ => panic!("source_lag_time_seconds has invalid type"), + } +} + +#[tokio::test] +async fn emits_component_discarded_events_total_for_send_event() { + metrics::init_test(); + let (mut sender, _recv) = SourceSender::new_test_sender_with_options(1, None); + + let event = Event::Metric(Metric::new( + "name", + MetricKind::Absolute, + MetricValue::Gauge { value: 123.4 }, + )); + + // First send will succeed. + sender + .send_event(event.clone()) + .await + .expect("First send should not fail"); + + // Second send will timeout, so the future will not be polled to completion. + let res = timeout( + std::time::Duration::from_millis(100), + sender.send_event(event.clone()), + ) + .await; + assert!(res.is_err(), "Send should have timed out."); + + let component_discarded_events_total = Controller::get() + .expect("There must be a controller") + .capture_metrics() + .into_iter() + .filter(|metric| metric.name() == "component_discarded_events_total") + .collect::>(); + assert_eq!(component_discarded_events_total.len(), 1); + + let component_discarded_events_total = &component_discarded_events_total[0]; + let MetricValue::Counter { value } = component_discarded_events_total.value() else { + panic!("component_discarded_events_total has invalid type") + }; + assert_eq!(*value, 1.0); +} + +#[tokio::test] +#[expect(clippy::cast_precision_loss)] +async fn emits_component_discarded_events_total_for_send_batch() { + metrics::init_test(); + let (mut sender, _recv) = SourceSender::new_test_sender_with_options(1, None); + + let expected_drop = 100; + let events: Vec = (0..(CHUNK_SIZE + expected_drop)) + .map(|_| { + Event::Metric(Metric::new( + "name", + MetricKind::Absolute, + MetricValue::Gauge { value: 123.4 }, + )) + }) + .collect(); + + // `CHUNK_SIZE` events will be sent into buffer but then the future will not be polled to completion. + let res = timeout( + std::time::Duration::from_millis(100), + sender.send_batch(events), + ) + .await; + assert!(res.is_err(), "Send should have timed out."); + + let metrics = get_component_metrics(); + assert_no_metric(&metrics, "component_timed_out_events_total"); + assert_no_metric(&metrics, "component_timed_out_requests_total"); + assert_counter_metric( + &metrics, + "component_discarded_events_total", + expected_drop as f64, + ); +} + +#[tokio::test] +async fn times_out_send_event_with_timeout() { + metrics::init_test(); + + let timeout_duration = StdDuration::from_millis(10); + let (mut sender, _recv) = SourceSender::new_test_sender_with_options(1, Some(timeout_duration)); + + let event = Event::Metric(Metric::new( + "name", + MetricKind::Absolute, + MetricValue::Gauge { value: 123.4 }, + )); + + sender + .send_event(event.clone()) + .await + .expect("First send should succeed"); + + let start = Instant::now(); + let result = sender.send_event(event).await; + let elapsed = start.elapsed(); + + assert!( + matches!(result, Err(SendError::Timeout)), + "Send should return a timeout error." + ); + assert!( + elapsed >= timeout_duration, + "Send did not wait for the configured timeout" + ); + assert!(elapsed <= timeout_duration * 2, "Send waited too long"); + + let metrics = get_component_metrics(); + assert_no_metric(&metrics, "component_discarded_events_total"); + assert_counter_metric(&metrics, "component_timed_out_events_total", 1.0); + assert_counter_metric(&metrics, "component_timed_out_requests_total", 1.0); +} + +fn get_component_metrics() -> Vec { + Controller::get() + .expect("There must be a controller") + .capture_metrics() + .into_iter() + .filter(|metric| metric.name().starts_with("component_")) + .collect() +} + +fn assert_no_metric(metrics: &[Metric], name: &str) { + assert!( + !metrics.iter().any(|metric| metric.name() == name), + "Metric {name} should not be present" + ); +} + +fn assert_counter_metric(metrics: &[Metric], name: &str, expected: f64) { + let mut filter = metrics.iter().filter(|metric| metric.name() == name); + let Some(metric) = filter.next() else { + panic!("Metric {name} should be present"); + }; + let MetricValue::Counter { value } = metric.value() else { + panic!("Metric {name} should be a counter"); + }; + assert_eq!(*value, expected); + assert!( + filter.next().is_none(), + "Only one {name} metric should be present" + ); +} + +#[tokio::test] +#[expect(clippy::cast_precision_loss)] +async fn emits_buffer_utilization_histogram_on_send_and_receive() { + metrics::init_test(); + let buffer_size = 2; + let (mut sender, mut recv) = SourceSender::new_test_sender_with_options(buffer_size, None); + + let event = Event::Log(LogEvent::from("test event")); + sender + .send_event(event.clone()) + .await + .expect("first send succeeds"); + sender + .send_event(event) + .await + .expect("second send succeeds"); + + // Drain the channel so both the send and receive paths are exercised. + assert!(recv.next().await.is_some()); + assert!(recv.next().await.is_some()); + + let metrics: Vec<_> = Controller::get() + .expect("metrics controller available") + .capture_metrics() + .into_iter() + .filter(|metric| metric.name().starts_with("source_buffer_")) + .collect(); + assert_eq!(metrics.len(), 5, "expected 5 utilization metrics"); + + let find_metric = |name: &str| { + metrics + .iter() + .find(|m| m.name() == name) + .unwrap_or_else(|| panic!("missing metric: {name}")) + }; + + let metric = find_metric("source_buffer_utilization"); + let tags = metric.tags().expect("utilization histogram has tags"); + assert_eq!(tags.get("output"), Some("_default")); + + let metric = find_metric("source_buffer_utilization_level"); + let MetricValue::Gauge { value } = metric.value() else { + panic!("source_buffer_utilization_level should be a gauge"); + }; + assert_eq!(*value, 2.0); + + let metric = find_metric("source_buffer_max_event_size"); + let MetricValue::Gauge { value } = metric.value() else { + panic!("source_buffer_max_event_size should be a gauge"); + }; + assert_eq!(*value, buffer_size as f64); + + let metric = find_metric("source_buffer_max_size_events"); + let MetricValue::Gauge { value } = metric.value() else { + panic!("source_buffer_max_size_events should be a gauge"); + }; + assert_eq!(*value, buffer_size as f64); +} diff --git a/lib/vector-core/src/task.rs b/lib/vector-core/src/task.rs new file mode 100644 index 0000000000000..f79615a514c19 --- /dev/null +++ b/lib/vector-core/src/task.rs @@ -0,0 +1,26 @@ +use std::{collections::BTreeMap, fmt}; + +#[derive(Clone, Debug)] +pub struct TaskCompletedError { + pub message: String, + pub fields: BTreeMap<&'static str, String>, +} + +impl TaskCompletedError { + pub fn new(message: String, fields: impl IntoIterator) -> Self { + let fields = fields.into_iter().collect(); + Self { message, fields } + } +} + +impl fmt::Display for TaskCompletedError { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(fmt, "{:?}", self.message)?; + let mut sep = " "; + for field in &self.fields { + write!(fmt, "{sep}{} = {:?}", field.0, field.1)?; + sep = ", "; + } + Ok(()) + } +} diff --git a/lib/vector-core/src/tls/settings.rs b/lib/vector-core/src/tls/settings.rs index 999972b05ad9c..c301537082c6f 100644 --- a/lib/vector-core/src/tls/settings.rs +++ b/lib/vector-core/src/tls/settings.rs @@ -5,10 +5,16 @@ use std::{ path::{Path, PathBuf}, }; +use super::{ + AddCertToStoreSnafu, AddExtraChainCertSnafu, CaStackPushSnafu, EncodeAlpnProtocolsSnafu, + FileOpenFailedSnafu, FileReadFailedSnafu, MaybeTls, NewCaStackSnafu, NewStoreBuilderSnafu, + ParsePkcs12Snafu, PrivateKeyParseSnafu, Result, SetAlpnProtocolsSnafu, SetCertificateSnafu, + SetPrivateKeySnafu, SetVerifyCertSnafu, TlsError, X509ParseSnafu, +}; use cfg_if::cfg_if; use lookup::lookup_v2::OptionalValuePath; use openssl::{ - pkcs12::{ParsedPkcs12_2, Pkcs12}, + pkcs12::Pkcs12, pkey::{PKey, Private}, ssl::{AlpnError, ConnectConfiguration, SslContextBuilder, SslVerifyMode, select_next_proto}, stack::Stack, @@ -17,14 +23,6 @@ use openssl::{ use snafu::ResultExt; use vector_config::configurable_component; -use super::{ - AddCertToStoreSnafu, AddExtraChainCertSnafu, CaStackPushSnafu, DerExportSnafu, - EncodeAlpnProtocolsSnafu, FileOpenFailedSnafu, FileReadFailedSnafu, MaybeTls, NewCaStackSnafu, - NewStoreBuilderSnafu, ParsePkcs12Snafu, Pkcs12Snafu, PrivateKeyParseSnafu, Result, - SetAlpnProtocolsSnafu, SetCertificateSnafu, SetPrivateKeySnafu, SetVerifyCertSnafu, TlsError, - TlsIdentitySnafu, X509ParseSnafu, -}; - pub const PEM_START_MARKER: &str = "-----BEGIN "; pub const TEST_PEM_CA_PATH: &str = "tests/data/ca/certs/ca.cert.pem"; @@ -177,13 +175,18 @@ pub struct TlsSettings { verify_certificate: bool, pub(super) verify_hostname: bool, authorities: Vec, - pub(super) identity: Option, // openssl::pkcs12::ParsedPkcs12 doesn't impl Clone yet + pub(super) identity: Option, alpn_protocols: Option>, server_name: Option, } +/// Identity store in PEM format #[derive(Clone)] -pub(super) struct IdentityStore(Vec, String); +pub(super) struct IdentityStore { + cert: X509, + key: PKey, + ca: Option>, +} impl TlsSettings { /// Generate a filled out settings struct from the given optional @@ -220,37 +223,20 @@ impl TlsSettings { }) } - /// Returns the identity as PKCS12 - /// - /// # Panics - /// - /// Panics if the identity is invalid. - fn identity(&self) -> Option { - // This data was test-built previously, so we can just use it - // here and expect the results will not fail. This can all be - // reworked when `openssl::pkcs12::ParsedPkcs12` gains the Clone - // impl. - self.identity.as_ref().map(|identity| { - Pkcs12::from_der(&identity.0) - .expect("Could not build PKCS#12 archive from parsed data") - .parse2(&identity.1) - .expect("Could not parse stored PKCS#12 archive") - }) - } - - /// Returns the identity as PEM data + /// Returns the identity as PEM encoded byte arrays /// /// # Panics /// /// Panics if the identity is missing, invalid, or the authorities to chain are invalid. pub fn identity_pem(&self) -> Option<(Vec, Vec)> { - self.identity().map(|identity| { - let mut cert = identity - .cert - .expect("Identity required") - .to_pem() + self.identity.as_ref().map(|identity| { + // we have verified correct formatting at ingest time + let mut cert = identity.cert.to_pem().expect("Invalid stored identity"); + let key = identity + .key + .private_key_to_pem_pkcs8() .expect("Invalid stored identity"); - if let Some(chain) = identity.ca { + if let Some(chain) = identity.ca.as_ref() { for authority in chain { cert.extend( authority @@ -259,11 +245,6 @@ impl TlsSettings { ); } } - let key = identity - .pkey - .expect("Private key required") - .private_key_to_pem_pkcs8() - .expect("Invalid stored private key"); (cert, key) }) } @@ -295,18 +276,18 @@ impl TlsSettings { } else { SslVerifyMode::NONE }); - if let Some(identity) = self.identity() { - if let Some(cert) = &identity.cert { - context.set_certificate(cert).context(SetCertificateSnafu)?; - } - if let Some(pkey) = &identity.pkey { - context.set_private_key(pkey).context(SetPrivateKeySnafu)?; - } + if let Some(identity) = &self.identity { + context + .set_certificate(&identity.cert) + .context(SetCertificateSnafu)?; + context + .set_private_key(&identity.key) + .context(SetPrivateKeySnafu)?; - if let Some(chain) = identity.ca { + if let Some(chain) = &identity.ca { for cert in chain { context - .add_extra_chain_cert(cert) + .add_extra_chain_cert(cert.clone()) .context(AddExtraChainCertSnafu)?; } } @@ -401,7 +382,7 @@ impl TlsConfig { let (data, filename) = open_read(filename, "certificate")?; der_or_pem( data, - |der| self.parse_pkcs12_identity(der), + |der| self.parse_pkcs12_identity(&der), |pem| self.parse_pem_identity(&pem, &filename), ) } @@ -430,45 +411,43 @@ impl TlsConfig { match &self.key_file { None => Err(TlsError::MissingKey), Some(key_file) => { - let name = crt_file.to_string_lossy().to_string(); let mut crt_stack = X509::stack_from_pem(pem.as_bytes()) .with_context(|_| X509ParseSnafu { filename: crt_file })? .into_iter(); - let crt = crt_stack.next().ok_or(TlsError::MissingCertificate)?; + let cert = crt_stack.next().ok_or(TlsError::MissingCertificate)?; let key = load_key(key_file.as_path(), self.key_pass.as_ref())?; let mut ca_stack = Stack::new().context(NewCaStackSnafu)?; for intermediate in crt_stack { ca_stack.push(intermediate).context(CaStackPushSnafu)?; } - - let pkcs12 = Pkcs12::builder() - .ca(ca_stack) - .name(&name) - .pkey(&key) - .cert(&crt) - .build2("") - .context(Pkcs12Snafu)?; - let identity = pkcs12.to_der().context(DerExportSnafu)?; - - // Build the resulting parsed PKCS#12 archive, - // but don't store it, as it cannot be cloned. - // This is just for error checking. - pkcs12.parse2("").context(TlsIdentitySnafu)?; - - Ok(Some(IdentityStore(identity, String::new()))) + let ca: Vec = ca_stack + .iter() + .map(std::borrow::ToOwned::to_owned) + .collect(); + Ok(Some(IdentityStore { + cert, + key, + ca: Some(ca), + })) } } } /// Parse identity from a DER encoded PKCS#12 archive - fn parse_pkcs12_identity(&self, der: Vec) -> Result> { - let pkcs12 = Pkcs12::from_der(&der).context(ParsePkcs12Snafu)?; + fn parse_pkcs12_identity(&self, der: &[u8]) -> Result> { + let pkcs12 = Pkcs12::from_der(der).context(ParsePkcs12Snafu)?; // Verify password let key_pass = self.key_pass.as_deref().unwrap_or(""); - pkcs12.parse2(key_pass).context(ParsePkcs12Snafu)?; - Ok(Some(IdentityStore(der, key_pass.to_string()))) + let parsed = pkcs12.parse2(key_pass).context(ParsePkcs12Snafu)?; + // extract cert, key and ca and store as PEM sow e can return an IdentityStore + let cert = parsed.cert.ok_or(TlsError::MissingCertificate)?; + let key = parsed.pkey.ok_or(TlsError::MissingKey)?; + let ca: Option> = parsed + .ca + .map(|stack| stack.iter().map(std::borrow::ToOwned::to_owned).collect()); + Ok(Some(IdentityStore { cert, key, ca })) } } diff --git a/lib/vector-lib/Cargo.toml b/lib/vector-lib/Cargo.toml index 3ba208e62c5ad..6c46f70a019bd 100644 --- a/lib/vector-lib/Cargo.toml +++ b/lib/vector-lib/Cargo.toml @@ -24,14 +24,15 @@ vector-top = { path = "../vector-top", optional = true } vrl = { workspace = true, optional = true } [features] -api = ["vector-tap/api"] +allocation-tracing = ["vector-top?/allocation-tracing"] api-client = ["dep:vector-api-client"] -lua = ["vector-core/lua"] +arrow = ["codecs/arrow"] +api = ["vector-tap/api"] file-source = ["dep:file-source", "dep:file-source-common"] +lua = ["vector-core/lua"] opentelemetry = ["dep:opentelemetry-proto", "codecs/opentelemetry"] prometheus = ["dep:prometheus-parser"] proptest = ["vector-lookup/proptest", "vrl/proptest"] syslog = ["codecs/syslog"] -test = ["vector-core/test"] +test = ["codecs/test", "vector-core/test"] vrl = ["vector-core/vrl", "dep:vrl"] -allocation-tracing = ["vector-top?/allocation-tracing"] diff --git a/lib/vector-lib/src/lib.rs b/lib/vector-lib/src/lib.rs index 894fde48c1fb7..3400a03c2e4cf 100644 --- a/lib/vector-lib/src/lib.rs +++ b/lib/vector-lib/src/lib.rs @@ -10,10 +10,10 @@ pub use vector_buffers as buffers; #[cfg(feature = "test")] pub use vector_common::event_test_util; pub use vector_common::{ - Error, Result, TimeZone, assert_event_data_eq, btreemap, byte_size_of, - byte_size_of::ByteSizeOf, conversion, encode_logfmt, finalization, finalizer, id, + Error, NamedInternalEvent, Result, TimeZone, assert_event_data_eq, atomic, btreemap, + byte_size_of, byte_size_of::ByteSizeOf, conversion, encode_logfmt, finalization, finalizer, id, impl_event_data_eq, internal_event, json_size, registered_event, request_metadata, - sensitive_string, shutdown, trigger, + sensitive_string, shutdown, stats, trigger, }; pub use vector_config as configurable; pub use vector_config::impl_generate_config_from_default; @@ -22,7 +22,7 @@ pub use vector_core::compile_vrl; pub use vector_core::{ EstimatedJsonEncodedSizeOf, buckets, default_data_dir, emit, event, fanout, ipallowlist, metric_tags, metrics, partition, quantiles, register, samples, schema, serde, sink, source, - tcp, tls, transform, + source_sender, tcp, tls, transform, }; pub use vector_lookup as lookup; pub use vector_stream as stream; diff --git a/lib/vector-stream/src/partitioned_batcher.rs b/lib/vector-stream/src/partitioned_batcher.rs index 42541b381c9bc..30cf039ec3840 100644 --- a/lib/vector-stream/src/partitioned_batcher.rs +++ b/lib/vector-stream/src/partitioned_batcher.rs @@ -359,7 +359,7 @@ mod test { use futures::{Stream, stream}; use pin_project::pin_project; use proptest::prelude::*; - use tokio::{pin, time::advance}; + use tokio::time::advance; use vector_core::{partition::Partitioner, time::KeyedTimer}; use crate::{ diff --git a/lib/vector-tap/Cargo.toml b/lib/vector-tap/Cargo.toml index 325520eac8b48..976233f446915 100644 --- a/lib/vector-tap/Cargo.toml +++ b/lib/vector-tap/Cargo.toml @@ -16,8 +16,8 @@ futures.workspace = true glob.workspace = true serde_yaml.workspace = true tokio = { workspace = true, features = ["time"] } -tokio-stream = { version = "0.1.17", default-features = false, features = ["sync"] } -tokio-tungstenite = { version = "0.20.1", default-features = false } +tokio-stream = { workspace = true, features = ["sync"] } +tokio-tungstenite.workspace = true tracing.workspace = true url = { version = "2.5.4", default-features = false } uuid.workspace = true diff --git a/lib/vector-tap/src/controller.rs b/lib/vector-tap/src/controller.rs index 763f14a98caf1..dba9a7ec7b885 100644 --- a/lib/vector-tap/src/controller.rs +++ b/lib/vector-tap/src/controller.rs @@ -356,7 +356,13 @@ async fn tap_handler( // target for the component, and spawn our transformer task which will // wrap each event payload with the necessary metadata before forwarding // it to our global tap receiver. - let (tap_buffer_tx, mut tap_buffer_rx) = TopologyBuilder::standalone_memory(TAP_BUFFER_SIZE, WhenFull::DropNewest, &Span::current()).await; + let (tap_buffer_tx, mut tap_buffer_rx) = TopologyBuilder::standalone_memory( + TAP_BUFFER_SIZE, + WhenFull::DropNewest, + &Span::current(), + None, + None, + ); let mut tap_transformer = TapTransformer::new(tx.clone(), output.clone()); tokio::spawn(async move { diff --git a/lib/vector-top/Cargo.toml b/lib/vector-top/Cargo.toml index b0c0debf16e76..8e494cc584185 100644 --- a/lib/vector-top/Cargo.toml +++ b/lib/vector-top/Cargo.toml @@ -14,13 +14,13 @@ futures-util = { workspace = true, features = ["alloc"] } glob.workspace = true indoc.workspace = true tokio = { workspace = true, features = ["full"] } -tokio-stream = { version = "0.1.17", default-features = false, features = ["net", "sync", "time"] } +tokio-stream = { workspace = true, features = ["net", "sync", "time"] } url.workspace = true humantime = { version = "2.2.0", default-features = false } crossterm = { version = "0.29.0", default-features = false, features = ["event-stream", "windows"] } -number_prefix = { version = "0.4.0", default-features = false, features = ["std"] } +unit-prefix = { version = "0.5.2", default-features = false, features = ["std"] } num-format = { version = "0.4.4", default-features = false, features = ["with-num-bigint"] } -ratatui = { version = "0.29.0", default-features = false, features = ["crossterm"] } +ratatui = { version = "0.30.0", default-features = false, features = ["crossterm", "layout-cache"] } vector-common = { path = "../vector-common" } vector-api-client = { path = "../vector-api-client" } diff --git a/lib/vector-top/src/dashboard.rs b/lib/vector-top/src/dashboard.rs index 7615c53df4d83..418ee0831d7a8 100644 --- a/lib/vector-top/src/dashboard.rs +++ b/lib/vector-top/src/dashboard.rs @@ -9,7 +9,6 @@ use crossterm::{ tty::IsTty, }; use num_format::{Locale, ToFormattedString}; -use number_prefix::NumberPrefix; use ratatui::{ Frame, Terminal, backend::CrosstermBackend, @@ -19,6 +18,7 @@ use ratatui::{ widgets::{Block, Borders, Cell, Paragraph, Row, Table, Wrap}, }; use tokio::sync::oneshot; +use unit_prefix::NumberPrefix; use super::{ events::capture_key_press, diff --git a/lib/vector-vrl-metrics/Cargo.toml b/lib/vector-vrl-metrics/Cargo.toml new file mode 100644 index 0000000000000..6998448b68f40 --- /dev/null +++ b/lib/vector-vrl-metrics/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "vector-vrl-metrics" +version = "0.1.0" +authors = ["Vector Contributors "] +edition = "2021" +publish = false +license = "MPL-2.0" + +[dependencies] +arc-swap.workspace = true +const-str.workspace = true +vrl.workspace = true +vector-core = { path = "../vector-core", default-features = false, features = ["vrl"] } +vector-common = { path = "../vector-common", default-features = false } +tokio.workspace = true +tokio-stream.workspace = true diff --git a/lib/vector-vrl-metrics/src/aggregate_vector_metrics.rs b/lib/vector-vrl-metrics/src/aggregate_vector_metrics.rs new file mode 100644 index 0000000000000..d51c132a9f22a --- /dev/null +++ b/lib/vector-vrl-metrics/src/aggregate_vector_metrics.rs @@ -0,0 +1,157 @@ +use std::collections::BTreeMap; +use vrl::prelude::expression::Expr; +use vrl::value; + +use vrl::prelude::*; + +use crate::common::resolve_tags; +use crate::common::validate_tags; +use crate::common::{Error, MetricsStorage}; + +fn aggregate_metrics( + metrics_storage: &MetricsStorage, + function: &Bytes, + key: Value, + tags: BTreeMap, +) -> Result { + let key_str = key.as_str().expect("argument must be a string"); + let metrics = metrics_storage.find_metrics(&key_str, tags); + + let metric_values = metrics.into_iter().filter_map(|m| match m.value() { + vector_core::event::MetricValue::Counter { value } + | vector_core::event::MetricValue::Gauge { value } => NotNan::new(*value).ok(), + _ => None, + }); + + Ok(match function.as_ref() { + b"sum" => metric_values.sum::>().into(), + b"avg" => { + let len = metric_values.clone().collect::>().len(); + (metric_values.sum::>() / len as f64).into() + } + b"max" => metric_values.max().map(Into::into).unwrap_or(Value::Null), + b"min" => metric_values.min().map(Into::into).unwrap_or(Value::Null), + _ => unreachable!(), + }) +} + +#[derive(Clone, Copy, Debug)] +pub struct AggregateVectorMetrics; + +fn aggregation_functions() -> Vec { + vec![value!("sum"), value!("avg"), value!("min"), value!("max")] +} + +impl Function for AggregateVectorMetrics { + fn identifier(&self) -> &'static str { + "aggregate_vector_metrics" + } + + fn usage(&self) -> &'static str { + const_str::concat!( + "Aggregates internal Vector metrics, using one of 4 aggregation functions, filtering by name and optionally by tags. Returns the aggregated value. Only includes counter and gauge metrics.\n\n", + crate::VECTOR_METRICS_EXPLAINER + ) + } + + fn parameters(&self) -> &'static [Parameter] { + &[ + Parameter { + keyword: "function", + kind: kind::BYTES, + required: true, + }, + Parameter { + keyword: "key", + kind: kind::BYTES, + required: true, + }, + Parameter { + keyword: "tags", + kind: kind::OBJECT, + required: false, + }, + ] + } + + fn examples(&self) -> &'static [Example] { + &[ + example! { + title: "Sum vector internal metrics matching the name", + source: r#"aggregate_vector_metrics("sum", "utilization")"#, + result: Ok("0.5"), + }, + example! { + title: "Sum vector internal metrics matching the name and tags", + source: r#"aggregate_vector_metrics("sum", "utilization", tags: {"component_id": "test"})"#, + result: Ok("0.5"), + }, + example! { + title: "Average of vector internal metrics matching the name", + source: r#"aggregate_vector_metrics("avg", "utilization")"#, + result: Ok("0.5"), + }, + example! { + title: "Max of vector internal metrics matching the name", + source: r#"aggregate_vector_metrics("max", "utilization")"#, + result: Ok("0.5"), + }, + example! { + title: "Min of vector internal metrics matching the name", + source: r#"aggregate_vector_metrics("max", "utilization")"#, + result: Ok("0.5"), + }, + ] + } + + fn compile( + &self, + state: &TypeState, + ctx: &mut FunctionCompileContext, + arguments: ArgumentList, + ) -> Compiled { + let metrics = ctx + .get_external_context::() + .ok_or(Box::new(Error::MetricsStorageNotLoaded) as Box)? + .clone(); + let function = arguments + .required_enum("function", &aggregation_functions(), state)? + .try_bytes() + .expect("aggregation function not bytes"); + let key = arguments.required("key"); + let tags = arguments.optional_object("tags")?.unwrap_or_default(); + validate_tags(state, &tags)?; + + Ok(AggregateVectorMetricsFn { + metrics, + function, + key, + tags, + } + .as_expr()) + } +} + +#[derive(Debug, Clone)] +struct AggregateVectorMetricsFn { + metrics: MetricsStorage, + function: Bytes, + key: Box, + tags: BTreeMap, +} + +impl FunctionExpression for AggregateVectorMetricsFn { + fn resolve(&self, ctx: &mut Context) -> Resolved { + let key = self.key.resolve(ctx)?; + aggregate_metrics( + &self.metrics, + &self.function, + key, + resolve_tags(ctx, &self.tags)?, + ) + } + + fn type_def(&self, _: &state::TypeState) -> TypeDef { + TypeDef::float().or_null().infallible() + } +} diff --git a/lib/vector-vrl-metrics/src/common.rs b/lib/vector-vrl-metrics/src/common.rs new file mode 100644 index 0000000000000..8f765c486d193 --- /dev/null +++ b/lib/vector-vrl-metrics/src/common.rs @@ -0,0 +1,818 @@ +use std::{collections::BTreeMap, sync::Arc, time::Duration}; +use tokio::time::interval; +use tokio_stream::{wrappers::IntervalStream, StreamExt}; +use vector_common::shutdown::ShutdownSignal; +use vrl::{ + diagnostic::Label, + prelude::{expression::Expr, *}, + value, +}; + +use arc_swap::ArcSwap; +use vector_core::{event::Metric, metrics::Controller}; + +#[derive(Debug)] +pub(crate) enum Error { + MetricsStorageNotLoaded, +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Error::MetricsStorageNotLoaded => write!(f, "metrics storage not loaded"), + } + } +} + +impl std::error::Error for Error {} + +impl DiagnosticMessage for Error { + fn code(&self) -> usize { + 112 + } + + fn labels(&self) -> Vec