diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 13843265c3..9b7fa0951b 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -2,5 +2,14 @@ .github/workflows/regression.yml @vectordotdev/vector @vectordotdev/single-machine-performance regression/config.yaml @vectordotdev/vector @vectordotdev/single-machine-performance -docs/ @vectordotdev/ux-team @vectordotdev/documentation -website/ @vectordotdev/ux-team @vectordotdev/documentation + +docs/ @vectordotdev/vector @vectordotdev/ux-team @vectordotdev/documentation +website/ @vectordotdev/vector @vectordotdev/ux-team +website/content @vectordotdev/vector @vectordotdev/documentation +website/cue/reference @vectordotdev/vector @vectordotdev/documentation + +website/js @vectordotdev/vector @vectordotdev/vector-website +website/layouts @vectordotdev/vector @vectordotdev/vector-website +website/scripts @vectordotdev/vector @vectordotdev/vector-website +website/data @vectordotdev/vector @vectordotdev/vector-website +website/* @vectordotdev/vector @vectordotdev/vector-website diff --git a/.github/DISCUSSION_TEMPLATE/q-a.yml b/.github/DISCUSSION_TEMPLATE/q-a.yml new file mode 100644 index 0000000000..23cb1f8467 --- /dev/null +++ b/.github/DISCUSSION_TEMPLATE/q-a.yml @@ -0,0 +1,33 @@ +title: "Q&A" +labels: [ q-a ] +body: + - type: markdown + attributes: + value: | + Please fill out the following fields to help us assist you effectively. + + - type: textarea + id: question + attributes: + label: Question + description: What are you trying to do? What issue are you encountering? + + - type: textarea + id: config + attributes: + label: Vector Config + description: Your Vector configuration (please redact sensitive data) + placeholder: | + ```yaml + # your config + ``` + + - type: textarea + id: logs + attributes: + label: Vector Logs + description: Paste any relevant Vector logs or error messages. + placeholder: | + ```sh + Jul 10 14:32:02 vector[1234]: ERROR ... + ``` diff --git a/.github/ISSUE_TEMPLATE/feature.yml b/.github/ISSUE_TEMPLATE/feature.yml index 3675a2173a..ed43f6891b 100644 --- a/.github/ISSUE_TEMPLATE/feature.yml +++ b/.github/ISSUE_TEMPLATE/feature.yml @@ -1,4 +1,5 @@ name: Feature +type: 'Feature' description: 🚀 Suggest a new feature. labels: - 'type: feature' diff --git a/.github/ISSUE_TEMPLATE/minor-release.md b/.github/ISSUE_TEMPLATE/minor-release.md index 50dabfcc72..ccbdbb3472 100644 --- a/.github/ISSUE_TEMPLATE/minor-release.md +++ b/.github/ISSUE_TEMPLATE/minor-release.md @@ -5,25 +5,46 @@ title: "Vector [version] release" labels: "domain: releasing" --- -The week before the release: + +# Setup and Automation + +Note the preparation steps are now automated. First, alter/create release.env + +```shell +export NEW_VECTOR_VERSION= # replace this with the actual new version (e.g.: 0.50.0) +export NEW_VRL_VERSION= # replace this with the actual new VRL version (e.g.: 0.30.0) +export MINOR_VERSION=$(echo "$NEW_VECTOR_VERSION" | cut -d. -f2) +export PREP_BRANCH=prepare-v-0-"${MINOR_VERSION}"-"${NEW_VECTOR_VERSION}"-website +export RELEASE_BRANCH=v0."${MINOR_VERSION}" +``` + +and then source it by running `source ./release.env` + +# The week before the release + +## 1. Manual Steps - [ ] Cut a new release of [VRL](https://github.com/vectordotdev/vrl) if needed -- [ ] Check for any outstanding deprecation actions in [DEPRECATIONS.md](https://github.com/vectordotdev/vector/blob/master/docs/DEPRECATIONS.md) and - take them (or have someone help you take them) + - VRL release steps: https://github.com/vectordotdev/vrl/blob/main/release/README.md + +## 2. Automated Steps + +Run the following: + +```shell +cargo vdev release prepare --version "${NEW_VECTOR_VERSION}" --vrl-version "${NEW_VRL_VERSION}" +``` + +Automated steps include: - [ ] Create a new release branch from master to freeze commits - - `git fetch && git checkout origin/master && git checkout -b v0. && git push -u` + - `git fetch && git checkout origin/master && git checkout -b "{RELEASE_BRANCH}" && git push -u` - [ ] Create a new release preparation branch from `master` - - `git checkout -b website-prepare-v0. && git push -u` + - `git checkout -b "${PREP_BRANCH}" && git push -u` - [ ] Pin VRL to latest released version rather than `main` - [ ] Check if there is a newer version of [Alpine](https://alpinelinux.org/releases/) or [Debian](https://www.debian.org/releases/) available to update the release images in `distribution/docker/`. Update if so. - [ ] Run `cargo vdev build release-cue` to generate a new cue file for the release - - [ ] Add description key to the generated cue file with a description of the release (see - previous releases for examples). - - [ ] Ensure any breaking changes are highlighted in the release upgrade guide - - [ ] Ensure any deprecations are highlighted in the release upgrade guide - - [ ] Review generated changelog entries to ensure they are understandable to end-users - [ ] Copy VRL changelogs from the VRL version in the last Vector release as a new changelog entry ([example](https://github.com/vectordotdev/vector/blob/9c67bba358195f5018febca2f228dfcb2be794b5/website/cue/reference/releases/0.41.0.cue#L33-L64)) - [ ] Update version number in `website/cue/reference/administration/interfaces/kubectl.cue` @@ -32,35 +53,55 @@ The week before the release: - [ ] Create new release md file by copying an existing one in `./website/content/en/releases/` and updating version number - [ ] Commit these changes -- [ ] Open PR against the release branch (`v0.`) for review -- [ ] PR approval +- [ ] Open PR against the release branch (`"${RELEASE_BRANCH}"`) for review + +## 3. Manual Steps + +- [ ] Edit `website/cue/reference/releases/"${NEW_VECTOR_VERSION}".cue` + - [ ] Add description key to the generated cue file with a description of the release (see + previous releases for examples). + - [ ] Ensure any breaking changes are highlighted in the release upgrade guide + - [ ] Ensure any deprecations are highlighted in the release upgrade guide + - [ ] Review generated changelog entries to ensure they are understandable to end-users +- [ ] Check for any outstanding deprecation actions in [DEPRECATIONS.md](https://github.com/vectordotdev/vector/blob/master/docs/DEPRECATIONS.md) and + take them (or have someone help you take them) +- [ ] PR review & approval + +# On the day of release -On the day of release: +- [ ] Make sure the release branch is in sync with origin/master and has only one squashed commit with all commits from the prepare branch. If you made a PR from the prepare branch into the release branch this should already be the case + - [ ] `git checkout "${RELEASE_BRANCH}"` + - [ ] `git show --stat HEAD` - This should show the squashed prepare commit + - [ ] `git diff HEAD~1 origin/master --quiet && echo "Same" || echo "Different"` - Should output `Same` + - Follow these steps if the release branch needs to be updated + - [ ] Rebase the release preparation branch on the release branch + - [ ] Squash the release preparation commits (but not the cherry-picked commits!) to a single + commit. This makes it easier to cherry-pick to master after the release. + - [ ] Ensure release date in `website/cue/reference/releases/0.XX.Y.cue` matches current date. + - If this needs to be updated commit and squash it in the release branch + - [ ] Merge release preparation branch into the release branch + - `git switch "${RELEASE_BRANCH}" && git merge --ff-only "${PREP_BRANCH}"` -- [ ] Rebase the release preparation branch on the release branch - - [ ] Squash the release preparation commits (but not the cherry-picked commits!) to a single - commit. This makes it easier to cherry-pick to master after the release.  - - [ ] Ensure release date in cue matches current date. -- [ ] Merge release preparation branch into the release branch - - `git co v0. && git merge --ff-only prepare-v0.` - [ ] Tag new release - - [ ] `git tag v0..0 -a -m v0..0` - - [ ] `git push origin v0..0` + - [ ] `git tag v"${NEW_VECTOR_VERSION}" -a -m v"${NEW_VECTOR_VERSION}"` + - [ ] `git push origin v"${NEW_VECTOR_VERSION}"` - [ ] Wait for release workflow to complete - - Discoverable via [https://github.com/timberio/vector/actions/workflows/release.yml](https://github.com/timberio/vector/actions/workflows/release.yml) + - Discoverable via [release.yml](https://github.com/vectordotdev/vector/actions/workflows/release.yml) - [ ] Reset the `website` branch to the `HEAD` of the release branch to update https://vector.dev - - [ ] `git checkout website && git reset --hard origin/v0. && git push` + - [ ] `git switch website && git reset --hard origin/"${RELEASE_BRANCH}" && git push` - [ ] Confirm that the release changelog was published to https://vector.dev/releases/ - The deployment is done by Amplify. You can see the [deployment logs here](https://dd-corpsite.datadoghq.com/logs?query=service%3Awebsites-vector%20branch%3Awebsite&agg_m=count&agg_m_source=base&agg_t=count&cols=host%2Cservice&fromUser=true&messageDisplay=inline&refresh_mode=sliding&storage=hot&stream_sort=time%2Casc&viz=stream). - [ ] Release Linux packages. See [`vector-release` usage](https://github.com/DataDog/vector-release#usage). - - Note: the pipeline inputs are the version number `v0.` and a personal GitHub token. + - Note: the pipeline inputs are the version number `v"${NEW_VECTOR_VERSION}"` and a personal GitHub token. - [ ] Manually trigger the `trigger-package-release-pipeline-prod-stable` job. - [ ] Release updated Helm chart. See [releasing Helm chart](https://github.com/vectordotdev/helm-charts#releasing). - [ ] Once Helm chart is released, updated Vector manifests - Run `cargo vdev build manifests` and open a PR with changes -- [ ] Add docker images to [https://github.com/DataDog/images](https://github.com/DataDog/images/tree/master/vector) to have them available internally. -- [ ] Cherry-pick any release commits from the release branch that are not on `master`, to `master` -- [ ] Bump the release number in the `Cargo.toml` on master to the next major release. - - Also, update `Cargo.lock` with: `cargo update -p vector` +- [ ] Add docker images to [https://github.com/DataDog/images](https://github.com/DataDog/images/tree/master/vector) to have them available internally. ([Example PR](https://github.com/DataDog/images/pull/7104)) +- [ ] Create a new PR with title starting as `chore(releasing):` + - [ ] Cherry-pick any release commits from the release branch that are not on `master`, to `master` + - [ ] Bump the release number in the `Cargo.toml` on master to the next minor release. + - [ ] Also, update `Cargo.lock` with: `cargo update -p vector` + - [ ] If there is a VRL version update, revert it and make it track the git `main` branch and then run `cargo update -p vrl`. - [ ] Kick-off post-mortems for any regressions resolved by the release diff --git a/.github/ISSUE_TEMPLATE/patch-release.md b/.github/ISSUE_TEMPLATE/patch-release.md index 440f7a24bb..afc5d8d826 100644 --- a/.github/ISSUE_TEMPLATE/patch-release.md +++ b/.github/ISSUE_TEMPLATE/patch-release.md @@ -5,10 +5,22 @@ title: "Vector [version] release" labels: "domain: releasing" --- -Before the release: +# Setup environment + +```shell +export CURRENT_MINOR_VERSION = # e.g. 47 +export CURRENT_PATCH_VERSION = # e.g. 0 +export CURRENT_VERSION="${RELEASE_BRANCH}"."${CURRENT_PATCH_VERSION}" +export NEW_PATCH_VERSION = # e.g. 1 +export NEW_VERSION="${RELEASE_BRANCH}"."${NEW_PATCH_VERSION}" +export RELEASE_BRANCH=v0."${CURRENT_MINOR_VERSION}" +export PREP_BRANCH=prepare-v-0-"${CURRENT_MINOR_VERSION}"-"${NEW_PATCH_VERSION}"-website +``` + +# Before the release - [ ] Create a new release preparation branch from the current release branch - - `git fetch && git checkout v0. && git checkout -b website-prepare-v0.` + - `git fetch --all && git checkout "${RELEASE_BRANCH}" && git checkout -b "${PREP_BRANCH}""` - [ ] Cherry-pick in all commits to be released from the associated release milestone - If any merge conflicts occur, attempt to solve them and if needed enlist the aid of those familiar with the conflicting commits. - [ ] Bump the release number in the `Cargo.toml` to the current version number @@ -17,32 +29,32 @@ Before the release: previous releases for examples). - [ ] Update version number in `distribution/install.sh` - [ ] Add new version to `website/cue/reference/versions.cue` -- [ ] Create new release md file by copying an existing one in `./website/content/en/releases/` and - updating version number +- [ ] Create new release md file by copying an existing one in `./website/content/en/releases/`. + - Update the version number to `"${NEW_VERSION}"` and increase the `weight` by 1. - [ ] Run `cargo check` to regenerate `Cargo.lock` file - [ ] Commit these changes -- [ ] Open PR against the release branch (`v0.`) for review +- [ ] Open PR against the release branch (`"${RELEASE_BRANCH}"`) for review - [ ] PR approval -On the day of release: +# On the day of release - [ ] Ensure release date in cue matches current date. - [ ] Rebase the release preparation branch on the release branch - Squash the release preparation commits (but not the cherry-picked commits!) to a single commit. This makes it easier to cherry-pick to master after the release. - - `git checkout prepare-v0. && git rebase -i v0.` + - `git fetch --all && git checkout website-prepare-v0-"${CURRENT_MINOR_VERSION}"-"${NEW_PATCH_VERSION}" && git rebase -i "${RELEASE_BRANCH}"` - [ ] Merge release preparation branch into the release branch - - `git co v0. && git merge --ff-only prepare-v0..` + - `git checkout "${RELEASE_BRANCH}" && git merge --ff-only website-prepare-v0-"${CURRENT_MINOR_VERSION}"-"${NEW_PATCH_VERSION}"` - [ ] Tag new release - - [ ] `git tag v0.. -a -m v0..` - - [ ] `git push origin v0..` + - [ ] `git tag "${NEW_VERSION}" -a -m "${NEW_VERSION}"` + - [ ] `git push origin "${NEW_VERSION}"` - [ ] Wait for release workflow to complete - Discoverable via [https://github.com/timberio/vector/actions/workflows/release.yml](https://github.com/timberio/vector/actions/workflows/release.yml) - [ ] Release Linux packages. See [`vector-release` usage](https://github.com/DataDog/vector-release#usage). - - Note: the pipeline inputs are the version number `v0.` and a personal GitHub token. + - Note: the pipeline inputs are the version number `"${CURRENT_VERSION}"` and a personal GitHub token. - [ ] Manually trigger the `trigger-package-release-pipeline-prod-stable` job. - [ ] Push the release branch to update the remote (This should close the preparation branch PR). - - `git checkout v0. && git push` + - `git checkout "${RELEASE_BRANCH}" && git push` - [ ] Release updated Helm chart. See [releasing Helm chart](https://github.com/vectordotdev/helm-charts#releasing). - [ ] Once Helm chart is released, updated Vector manifests - Run `cargo vdev build manifests` and open a PR with changes @@ -50,5 +62,5 @@ On the day of release: - Follow the [instructions at the top of the mirror.yaml file](https://github.com/DataDog/images/blob/fbf12868e90d52e513ebca0389610dea8a3c7e1a/mirror.yaml#L33-L49). - [ ] Cherry-pick any release commits from the release branch that are not on `master`, to `master` - [ ] Reset the `website` branch to the `HEAD` of the release branch to update https://vector.dev - - [ ] `git checkout website && git reset --hard origin/v0.. && git push` + - [ ] `git checkout website && git reset --hard origin/"${RELEASE_BRANCH}" && git push` - [ ] Kick-off post-mortems for any regressions resolved by the release diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 043530c977..76781a822e 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,26 +1,13 @@ - - ## Summary +## Vector configuration + + +## How did you test this PR? + + ## Change Type - [ ] Bug fix - [ ] New feature @@ -31,33 +18,50 @@ This should help the reviewers give feedback faster and with higher quality. --> - [ ] Yes - [ ] No -## How did you test this PR? - - ## Does this PR include user facing changes? - [ ] Yes. Please add a changelog fragment based on our [guidelines](https://github.com/vectordotdev/vector/blob/master/changelog.d/README.md). -- [ ] No. A maintainer will apply the "no-changelog" label to this PR. +- [ ] No. A maintainer will apply the `no-changelog` label to this PR. -## Checklist -- [ ] Please read our [Vector contributor resources](https://github.com/vectordotdev/vector/tree/master/docs#getting-started). - - `make check-all` is a good command to run locally. This check is - defined [here](https://github.com/vectordotdev/vector/blob/1ef01aeeef592c21d32ba4d663e199f0608f615b/Makefile#L450-L454). Some of these - checks might not be relevant to your PR. For Rust changes, at the very least you should run: +## References + + + +## Notes +- Please read our [Vector contributor resources](https://github.com/vectordotdev/vector/tree/master/docs#getting-started). +- Do not hesitate to use `@vectordotdev/vector` to reach out to us regarding this PR. +- Some CI checks run only after we manually approve them. + - We recommend adding a `pre-push` hook, please see [this template](https://github.com/vectordotdev/vector/blob/master/CONTRIBUTING.md#Pre-push). + - Alternatively, we recommend running the following locally before pushing to the remote branch: - `cargo fmt --all` - `cargo clippy --workspace --all-targets -- -D warnings` - `cargo nextest run --workspace` (alternatively, you can run `cargo test --all`) -- [ ] If this PR introduces changes Vector dependencies (modifies `Cargo.lock`), please - run `dd-rust-license-tool write` to regenerate the [license inventory](https://github.com/vectordotdev/vrl/blob/main/LICENSE-3rdparty.csv) and commit the changes (if any). More details [here](https://crates.io/crates/dd-rust-license-tool). - -## References +- After a review is requested, please avoid force pushes to help us review incrementally. + - Feel free to push as many commits as you want. They will be squashed into one before merging. + - For example, you can run `git merge origin master` and `git push`. +- If this PR introduces changes Vector dependencies (modifies `Cargo.lock`), please + run `cargo vdev build licenses` to regenerate the [license inventory](https://github.com/vectordotdev/vrl/blob/main/LICENSE-3rdparty.csv) and commit the changes (if any). More details [here](https://crates.io/crates/dd-rust-license-tool). - + Your PR title must conform to the conventional commit spec: + https://www.conventionalcommits.org/en/v1.0.0/ + + ()!: - + * `type` = chore, enhancement, feat, fix, docs, revert + * `!` = OPTIONAL: signals a breaking change + * `scope` = Optional when `type` is "chore" or "docs", available scopes https://github.com/vectordotdev/vector/blob/master/.github/workflows/semantic.yml#L31 + * `description` = short description of the change + +Examples: + + * enhancement(file source): Add `sort` option to sort discovered files + * feat(new source): Initial `statsd` source + * fix(file source): Fix a bug discovering new files + * chore(external docs): Clarify `batch_size` option +--> diff --git a/.github/actions/spelling/allow.txt b/.github/actions/spelling/allow.txt index f0216c0af3..cb1c04f008 100644 --- a/.github/actions/spelling/allow.txt +++ b/.github/actions/spelling/allow.txt @@ -1,25 +1,43 @@ -ANSIX -APAC -APPSIGNAL +acmecorp Acro +addonmanager Ainol +aiohttp Airis Airpad +AIXM Alcatel +alertmanager Alexey +algoliasearch Alibaba Allfine Allview Allwinner +alpinejs +altostrat Amarok Amaway +amazonlinux Amoi +ansible +ANSIX Aoc Aoson +APAC +apachectl +apachepulsar Apanda +apikey +apimachinery +apiserver +APPSIGNAL Appsignal +appsignal +archlinux Archos Arival +armhf Arnova Asus Atlassian @@ -27,221 +45,17 @@ atleastonce atmostonce Attab Audiosonic +AUTOSAR +aviv avsc Axioo Azend -Bedove -Benss -Blaupunkt -Blusens -Casio -Cantarell -Celkon -Ceph -Chromecast -Citrix -Cloudflare -Cloudfone -Cmx -Coby -Collectd -Comcast -Consolas -Coolpad -BADDCAFE -DEBHELPER -Danew -dkr -Dockerfiles -DOOV -Douban -emqx -eventloop -Enot -Evercoss -exactlyonce -Explay -FAQs -FDO -FQDNs -Fabro -Figma -Flipboard -Foto -Freescale -Galapad -Garmin -Geeksphone -Gfive -Ghemawat -Gionee -HMACs -HTTPDATE -Haier -Haipad -Hannspree -Hena -Hisense -Huawei -Hyundai -Ideapad -Infinix -Instacart -Intenso -Itamar -Ivio -JXD -Jacq -Jameel -Jaytech -Jia -Jiayu -Joda -KBytes -KDL -KTtech -Karbonn -Kingcom -Kolkata -Kruno -Ktouch -Kurio -Kyros -LGE -LYF -Lenco -Lexibook -Lifetab -Lifetouch -Lumia -Malata -manden -Maxthon -Mediacom -Medion -Meizu -Mertz -metakey -Metakey -Micromax -Mito -Mobistel -Modecom -Moto -Mpman -Multilaser -Mumbai -musleabi -Mytab -NLB -Nabi -Netflix -Neue -Nextbook -Nextcloud -OVH -Odys -Openpeak -Oppo -Ovi -Owncloud -PHILIPSTV -POCs -Panasonic -Papyre -pbs -Phicomm -Phototechnik -Pingdom -Pinterest -Pinterestbot -Pipo -Ployer -Podcast -Podkicker -Positivo -Prestigio -pront -Proscan -Qmobilevn -RPZ -RRSIGs -Rackspace -Rathi -Regza -Rijs -Roboto -Rockchip -Roku -Roundcube -Rowling -rumqttc -SBT -SKtelesys -Salesforce -Samsung -Sega -Segoe -Shopify -SIGINTs -Simvalley -Skype -Skytex -Smartbitt -Snapchat -Softbank -Sogou -Soref -Tagi -Tecmobile -Telstra -Tencent -Texet -Thl -Tizen -Tmobile -Tomtec -Tooky -Touchmate -Traefik -Trekstor -Treq -Umeox -Verizon -Videocon -Viewsonic -Wellcom -Wii -Wiko -Woxter -Xeon -Xianghe -Xolo -Xoro -Xperia -Yarvik -Yifang -ZTE -Zopo -Zync -acmecorp -addonmanager -aiohttp -algoliasearch -alpinejs -altostrat -amazonlinux -ansible -apachectl -apachepulsar -apikey -apimachinery -apiserver -appsignal -archlinux -armhf backpressure backticks +BADDCAFE +BADVERS +Bedove +Benss bigendian bindir binfmt @@ -251,16 +65,36 @@ bitflags bitnami bitwidth blackbox +Blaupunkt +Blusens buildname buildroot bytestream callsites +Cantarell +Casio +cbor +CDMA cdnskey +Celkon +Ceph +Chromecast +Citrix +cksum +Cloudflare +Cloudfone +Cmx cncf +Coby codepath codepaths +Collectd +Comcast commandline compiletime +Consolas +contactless +Coolpad coredns corejs coreutils @@ -268,46 +102,83 @@ csync curta daemonset dalek +dammam +Danew databend datacenter datadog datadoghq datanode ddog +DEBHELPER debian +DECT demuxing dfs discriminants distro distroless +dkr +DNP dnslookup dnssec dnstap dnsutils dockercmd +Dockerfiles +doha +DOOV +Douban downsides downwardapi +DVB ede emoji +emqx +enableable +Enot +EPC esbuild etld +eventloop +Evercoss +exactlyonce +Explay +Fabro fakeintake +FAQs fargate +FDO fibonacci +Figma fileapi filebeat finalizers findstring firefox +FLEXRAY +Flipboard fluentbit fluentd +Foto +FQDNs +Freescale +FUJITSU fuzzers +Galapad +Garmin gce gcloud gcp gcr gcs gdpr +Geeksphone +GENIBUS +Gfive +Ghemawat +gifs +Gionee github gnueabi gnueabihf @@ -326,41 +197,79 @@ graphiql greptime greptimecloud greptimedb +GSM gvisor gws hadoop +Haier +Haipad +Hannspree hdfs healthcheck healthchecks +Hena heroicon heroicons heroku herokuapp +Hisense +hitag hitech +HMACs hogwarts htc htmltest +HTTPDATE https +Huawei humungus +Hyundai icecream +Ideapad idn ifeq ifneq imobile +Infinix influxd +Instacart +Intenso +INTERLAKEN ionik ipallowlist ipod ircd +Itamar +Ivio +Jacq +JAMCRC +Jameel +Jaytech jemalloc jemallocator +jetbrains +JetBrains +Jia +Jiayu jndi +Joda journalctl jsonnet jsontag jvm +JXD +Karbonn +KBytes +KDL +keepappkey +keephq kenton +Kingcom +Kolkata konqueror +Kruno +Ktouch +KTtech kube kubeadm kubeconfig @@ -368,152 +277,269 @@ kubectl kubelet kubernetes kubeval +Kurio kustomization kustomize kyocera +Kyros +Lenco lenovo levenstein +Lexibook +LGE +Lifetab +Lifetouch linkerd localdomain localstack +lookback lucene +Lumia +LYF macbook +Malata +manden maxmind maxminddb +Maxthon +MCRF +Mediacom +Medion +MEF +Meizu +meme +Mertz messagebird +metakey +Metakey +microcontroller +Micromax +MIFARE minikube minimalistic minio minishift +miniz +Mito mkfile +Mobistel +modbus +Modecom mongod +Moto motorola mountpoint mozilla +Mpman msiexec +Multilaser +Mumbai +mumbai +musleabi +Mytab +Nabi namenode netcat netdata +Netflix netlify +Neue +Nextbook +Nextcloud nintendo nio nixos nixpkg nixpkgs +NLB nokia +NRSC nslookup nsupdate ntapi ntfs +Odys +onig opendal +Openpeak +OPENPGP +OPENSAFETY opensearch opentelemetry +Oppo oss +otel +otelcol +OVH +Ovi +Owncloud pacman +Panasonic pantech papertrail papertrailapp +Papyre +paulo +pbs petabytes +Phicomm philips +PHILIPSTV +Phototechnik +Pingdom +Pinterest +Pinterestbot +Pipo +Ployer +POCs +Podcast podinfo +Podkicker podman +Positivo postgresql +Prestigio +PROFIBUS +pront +Proscan proxyuser pseudocode psl pushgateway +QA'd +Qmobilevn qwerty rabbitmq +Rackspace +Rathi rclone +Regza +requestline rfcs +RFID +riello +Rijs roadmap +Roboto +Rockchip +ROHC +Roku rootfs +Roundcube roundrobin +Rowling rpmbuild rpms +RPZ +RRSIGs rstest rsyslog rsyslogd +rumqttc +Salesforce +Samsung +SBT scriptblock +Sega +Segoe servlet +shannon +Shopify +SIGINTs +simdutf +Simvalley Sinjo -sublocation -sundar -svcb +siv +SKtelesys +Skype +Skytex +Smartbitt +SMBUS +Snapchat snyk socketaddr +Softbank +Sogou solarwinds +Soref +sortedset splunk ssh staticuser statsd +sublocation +sundar +svcb symbian +Tagi tanushri +Tecmobile +teledisk +Telstra +Tencent +Texet +Thl timeframe timeseries timespan timestamped +Tizen +Tmobile +Tomtec +Tooky +Touchmate +Traefik +Trekstor +Treq +turin typesense tzdata ubuntu +Umeox +UMTS unchunked upstreaminfo +urlencoding useragents usergroups userguide -urlencoding +Verizon +vhosts +Videocon +Viewsonic +WCDMA webhdfs +Wellcom +Wii +Wiko winapi workarounds +Woxter XCHACHA +Xeon +Xianghe +XMODEM +Xolo +Xoro +Xperia XSALSA yandex +Yarvik +Yifang +zadd zeek zookeeper +Zopo zst zstandard -otel -otelcol -siv -onig -aviv -dammam -doha -mumbai -paulo -turin -AIXM -AUTOSAR -CDMA -cksum -contactless -DECT -DNP -DVB -EPC -FLEXRAY -FUJITSU -GENIBUS -GSM -hitag -INTERLAKEN -JAMCRC -MCRF -MEF -microcontroller -MIFARE -modbus -NRSC -OPENPGP -OPENSAFETY -PROFIBUS -RFID -riello -ROHC -SMBUS -teledisk -UMTS -WCDMA -XMODEM -cbor -requestline +ZTE +Zync +envsubst +jimmystewpot +esensar +jchap-pnnl +jhbigler-pnnl +jlambatl +jorgehermo9 +notchairmk +yjagdale diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt index 5eb074f227..e48644308c 100644 --- a/.github/actions/spelling/expect.txt +++ b/.github/actions/spelling/expect.txt @@ -78,7 +78,6 @@ bytesize califrag califragilistic CAROOT -CBOR cddl cdylib cef @@ -175,7 +174,6 @@ eeyun efgh Elhage emerg -Enableable endianess endler eni @@ -256,7 +254,6 @@ hostpath hoverable hoverbear httpdeliveryrequestresponse -httpdump httpevent hugepages hugops @@ -467,6 +464,7 @@ rcode rdkafka rdparty readnone +recordset rediss redoctober regexes @@ -541,6 +539,7 @@ spencergilbert spinlock SPOF spog +sqlx srcaddr srcport SREs @@ -592,6 +591,7 @@ threatmanager Throughputs Tiltfile timberio +TIMESTAMPTZ TKEY tlh tmpfs @@ -672,7 +672,7 @@ wtimeout WTS xact xlarge -XMODEM +XXH xxs YAMLs YBv diff --git a/.github/audit.yml b/.github/audit.yml index 83db5d6015..2544a1bad0 100644 --- a/.github/audit.yml +++ b/.github/audit.yml @@ -9,9 +9,10 @@ on: jobs: security_audit: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v1 - - uses: actions-rs/audit-check@v1 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + # TODO: replace this action - abandoned since 2020 + - uses: actions-rs/audit-check@35b7b53b1e25b55642157ac01b4adceb5b9ebef3 # v1.2.0 with: token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/dependabot.yml b/.github/dependabot.yml index a69c5aaf4d..5e2208e0b7 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -3,7 +3,7 @@ updates: - package-ecosystem: "cargo" directory: "/" schedule: - interval: "weekly" + interval: "monthly" time: "04:00" # UTC labels: - "domain: deps" @@ -73,7 +73,7 @@ updates: - package-ecosystem: "docker" directory: "/distribution/docker/" schedule: - interval: "daily" + interval: "monthly" time: "04:00" # UTC labels: - "domain: releasing" @@ -84,7 +84,7 @@ updates: - package-ecosystem: "github-actions" directory: "/" schedule: - interval: "weekly" + interval: "monthly" labels: - "domain: ci" - "no-changelog" @@ -95,3 +95,19 @@ updates: patterns: - "actions/download-artifact" - "actions/upload-artifact" + - package-ecosystem: "npm" + directory: "website/" + schedule: + interval: "monthly" + time: "05:00" # UTC + labels: + - "dependencies" + - "javascript" + - "no-changelog" + commit-message: + prefix: "chore(website deps)" + open-pull-requests-limit: 50 + groups: + npm_and_yarn: + patterns: + - "*" diff --git a/.github/workflows/build_preview_sites.yml b/.github/workflows/build_preview_sites.yml index ea3c8b2035..aa802412e0 100644 --- a/.github/workflows/build_preview_sites.yml +++ b/.github/workflows/build_preview_sites.yml @@ -7,6 +7,7 @@ on: - completed permissions: + actions: read issues: write pull-requests: write statuses: write diff --git a/.github/workflows/changelog.yaml b/.github/workflows/changelog.yaml index 57d7013f3b..a3534947a1 100644 --- a/.github/workflows/changelog.yaml +++ b/.github/workflows/changelog.yaml @@ -6,63 +6,66 @@ # - new commits pushed # - label is added or removed # Runs on merge queues, but just passes, because it is a required check. - name: Changelog on: - pull_request: + pull_request_target: types: [opened, synchronize, reopened, labeled, unlabeled] - # Due to merge queue requiring same status checks as PRs, must pass by default + + # Required by GitHub merge queue due to branch protection rules. Should always be successful merge_group: types: [checks_requested] jobs: validate-changelog: + permissions: + contents: read + pull-requests: none + runs-on: ubuntu-24.04 - if: github.event_name == 'pull_request' + env: - PR_HAS_LABEL: ${{ contains( github.event.pull_request.labels.*.name, 'no-changelog') }} - steps: - # Checkout full depth because in the check_changelog_fragments script, we need to specify a - # merge base. If we only shallow clone the repo, git may not have enough history to determine - # the base. - - uses: actions/checkout@v4 - with: - fetch-depth: 0 + NO_CHANGELOG: ${{ contains(github.event.pull_request.labels.*.name, 'no-changelog') }} + SHOULD_RUN: ${{ !contains(github.event.pull_request.labels.*.name, 'no-changelog') && github.event_name != 'merge_group' }} - - name: Generate authentication token - # Don't run this step if the PR is from a fork or dependabot since the secrets won't exist - if: ${{ github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]' }} - id: generate_token - uses: tibdex/github-app-token@3beb63f4bd073e61482598c45c71c1019b59b73a - with: - app_id: ${{ secrets.GH_APP_DATADOG_VECTOR_CI_APP_ID }} - private_key: ${{ secrets.GH_APP_DATADOG_VECTOR_CI_APP_PRIVATE_KEY }} + steps: + - name: Bypass when no‑changelog label is present + if: env.NO_CHANGELOG == 'true' + run: | + echo "'no-changelog' label detected – skipping changelog validation." + exit 0 - - run: | - if [[ $PR_HAS_LABEL == 'true' ]] ; then - echo "'no-changelog' label detected." - exit 0 - fi + - name: Merge queue + if: ${{ github.event_name == 'merge_group' }} + run: | + echo "merge_group event – passing without running changelog validation." + exit 0 - # Helper script needs to reference the master branch to compare against - git fetch origin master:refs/remotes/origin/master + # Checkout changelog script and changelog.d/ from master + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + if: env.SHOULD_RUN == 'true' + with: + ref: master + sparse-checkout: | + scripts/check_changelog_fragments.sh + changelog.d/ + sparse-checkout-cone-mode: false - ./scripts/check_changelog_fragments.sh + # Checkout PR's changelog.d/ into tmp/ + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + if: env.SHOULD_RUN == 'true' + with: + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.sha }} + path: tmp + sparse-checkout: changelog.d/ - check-changelog: - name: Changelog - runs-on: ubuntu-24.04 - needs: validate-changelog - if: always() - env: - FAILED: ${{ contains(needs.*.result, 'failure') }} - steps: - - name: exit + - name: Run changelog fragment checker + if: env.SHOULD_RUN == 'true' run: | - echo "failed=${{ env.FAILED }}" - if [[ "$FAILED" == "true" ]] ; then - exit 1 - else - exit 0 - fi + # Overwrite changelog.d/*.md + rm -rf changelog.d/*.md && mv tmp/changelog.d/*.md changelog.d/ + + # Add files and then compare with HEAD instead of origin/master + git add changelog.d/ + MERGE_BASE=HEAD ./scripts/check_changelog_fragments.sh diff --git a/.github/workflows/changes.yml b/.github/workflows/changes.yml index d82a3e79d0..1c470d80f5 100644 --- a/.github/workflows/changes.yml +++ b/.github/workflows/changes.yml @@ -10,10 +10,10 @@ on: # merge_group context. inputs: base_ref: - required: true + required: false type: string head_ref: - required: true + required: false type: string int_tests: required: false @@ -40,12 +40,12 @@ on: value: ${{ jobs.source.outputs.component_docs }} markdown: value: ${{ jobs.source.outputs.markdown }} + website_only: + value: ${{ jobs.source.outputs.website_only }} install: value: ${{ jobs.source.outputs.install }} k8s: value: ${{ jobs.source.outputs.k8s }} - all-int: - value: ${{ jobs.int_tests.outputs.all-tests }} amqp: value: ${{ jobs.int_tests.outputs.amqp }} appsignal: @@ -119,17 +119,20 @@ on: all-changes-json: value: ${{ jobs.int_tests.outputs.all-changes-json }} # e2e tests - all-e2e: - value: ${{ jobs.e2e_tests.outputs.all-tests }} e2e-datadog-logs: value: ${{ jobs.e2e_tests.outputs.datadog-logs }} e2e-datadog-metrics: value: ${{ jobs.e2e_tests.outputs.datadog-metrics }} + e2e-opentelemetry-logs: + value: ${{ jobs.e2e_tests.outputs.opentelemetry-logs }} +env: + BASE_SHA: ${{ inputs.base_ref || (github.event_name == 'merge_group' && github.event.merge_group.base_sha) || github.event.pull_request.base.sha }} + HEAD_SHA: ${{ inputs.head_ref || (github.event_name == 'merge_group' && github.event.merge_group.head_sha) || github.event.pull_request.head.sha }} jobs: # Detects changes that are not specific to integration tests source: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 timeout-minutes: 5 if: ${{ inputs.source }} outputs: @@ -141,14 +144,15 @@ jobs: markdown: ${{ steps.filter.outputs.markdown }} install: ${{ steps.filter.outputs.install }} k8s: ${{ steps.filter.outputs.k8s }} + website_only: ${{ steps.filter.outputs.website == 'true' && steps.filter.outputs.not_website == 'false' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - uses: dorny/paths-filter@v3 + - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 id: filter with: - base: ${{ inputs.base_ref }} - ref: ${{ inputs.head_ref }} + base: ${{ env.BASE_SHA }} + ref: ${{ env.HEAD_SHA }} filters: | source: - ".github/workflows/test.yml" @@ -170,13 +174,12 @@ jobs: - 'Cargo.toml' - 'Cargo.lock' - 'rust-toolchain.toml' - - '.github/workflows/pr.yml' - 'Makefile' - 'scripts/cross/**' - "vdev/**" cue: - 'website/cue/**' - - "vdev" + - "vdev/**" component_docs: - 'scripts/generate-component-docs.rb' - "vdev/**" @@ -195,6 +198,12 @@ jobs: - "distribution/install.sh" k8s: - "src/sources/kubernetes_logs/**" + - "lib/k8s-e2e-tests/**" + - "lib/k8s-test-framework/**" + website: + - "website/**" + not_website: + - "!website/**" # Detects changes that are specific to integration tests int_tests: @@ -202,7 +211,6 @@ jobs: timeout-minutes: 5 if: ${{ inputs.int_tests }} outputs: - all-tests: ${{ steps.filter.outputs.all-tests}} amqp: ${{ steps.filter.outputs.amqp }} appsignal: ${{ steps.filter.outputs.appsignal}} aws: ${{ steps.filter.outputs.aws }} @@ -240,7 +248,7 @@ jobs: webhdfs: ${{ steps.filter.outputs.webhdfs }} all-changes-json: ${{ steps.aggregate.outputs.all-changes-json }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 # creates a yaml file that contains the filters for each integration, # extracted from the output of the `vdev int ci-paths` command, which @@ -248,11 +256,11 @@ jobs: - name: Create filter rules for integrations run: cargo vdev int ci-paths > int_test_filters.yaml - - uses: dorny/paths-filter@v3 + - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 id: filter with: - base: ${{ inputs.base_ref }} - ref: ${{ inputs.head_ref }} + base: ${{ env.BASE_SHA }} + ref: ${{ env.HEAD_SHA }} filters: int_test_filters.yaml # This JSON hack was introduced because GitHub Actions does not support dynamic expressions in the @@ -304,7 +312,7 @@ jobs: echo "$json" > int_tests_changes.json - name: Upload JSON artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: int_tests_changes path: int_tests_changes.json @@ -315,11 +323,10 @@ jobs: timeout-minutes: 5 if: ${{ inputs.e2e_tests }} outputs: - all-tests: ${{ steps.filter.outputs.all-tests}} datadog-logs: ${{ steps.filter.outputs.datadog-logs }} datadog-metrics: ${{ steps.filter.outputs.datadog-metrics }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 # creates a yaml file that contains the filters for each test, # extracted from the output of the `vdev int ci-paths` command, which @@ -327,9 +334,9 @@ jobs: - name: Create filter rules for e2e tests run: cargo vdev e2e ci-paths > int_test_filters.yaml - - uses: dorny/paths-filter@v3 + - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 id: filter with: - base: ${{ inputs.base_ref }} - ref: ${{ inputs.head_ref }} + base: ${{ env.BASE_SHA }} + ref: ${{ env.HEAD_SHA }} filters: int_test_filters.yaml diff --git a/.github/workflows/ci-integration-review.yml b/.github/workflows/ci-integration-review.yml index dd8297f9da..b8cbb2ace1 100644 --- a/.github/workflows/ci-integration-review.yml +++ b/.github/workflows/ci-integration-review.yml @@ -30,14 +30,12 @@ name: CI Integration Review Trigger on: pull_request_review: - types: [submitted] + types: [ submitted ] permissions: statuses: write env: - AWS_ACCESS_KEY_ID: "dummy" - AWS_SECRET_ACCESS_KEY: "dummy" AXIOM_TOKEN: ${{ secrets.AXIOM_TOKEN }} TEST_APPSIGNAL_PUSH_API_KEY: ${{ secrets.TEST_APPSIGNAL_PUSH_API_KEY }} TEST_DATADOG_API_KEY: ${{ secrets.CI_TEST_DATADOG_API_KEY }} @@ -55,7 +53,7 @@ jobs: name: (PR review) Signal pending to PR runs-on: ubuntu-24.04 timeout-minutes: 5 - if: startsWith(github.event.review.body, '/ci-run-integration') || contains(github.event.review.body, '/ci-run-all') + if: startsWith(github.event.review.body, '/ci-run-integration') || startsWith(github.event.review.body, '/ci-run-e2e') || contains(github.event.review.body, '/ci-run-all') steps: - name: Generate authentication token id: generate_token @@ -65,7 +63,7 @@ jobs: private_key: ${{ secrets.GH_APP_DATADOG_VECTOR_CI_APP_PRIVATE_KEY }} - name: Get PR review author id: comment - uses: tspascoal/get-user-teams-membership@v3 + uses: tspascoal/get-user-teams-membership@57e9f42acd78f4d0f496b3be4368fc5f62696662 # v3.0.0 with: username: ${{ github.actor }} team: 'Vector' @@ -76,7 +74,7 @@ jobs: run: exit 1 - name: (PR review) Set latest commit status as pending - uses: myrotvorets/set-commit-status-action@v2.0.1 + uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 with: sha: ${{ github.event.review.commit_id }} token: ${{ secrets.GITHUB_TOKEN }} @@ -84,415 +82,75 @@ jobs: integration-tests: needs: prep-pr - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 timeout-minutes: 90 + strategy: + matrix: + service: [ + "amqp", "appsignal", "aws", "axiom", "azure", "clickhouse", "databend", "datadog-agent", + "datadog-logs", "datadog-metrics", "datadog-traces", "dnstap", "docker-logs", "elasticsearch", + "eventstoredb", "fluent", "gcp", "greptimedb", "http-client", "influxdb", "kafka", "logstash", + "loki", "mongodb", "nats", "nginx", "opentelemetry", "postgres", "prometheus", "pulsar", + "redis", "splunk", "webhdfs" + ] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: submodules: "recursive" ref: ${{ github.event.review.commit_id }} - - run: sudo npm -g install @datadog/datadog-ci + - run: bash scripts/environment/prepare.sh --modules=datadog-ci - run: docker image prune -af ; docker container prune -f - - name: amqp - if: ${{ startsWith(github.event.review.body, '/ci-run-integration-amqp') - || startsWith(github.event.review.body, '/ci-run-integration-all') - || startsWith(github.event.review.body, '/ci-run-all') }} - uses: nick-fields/retry@v3 - with: - # First one requires more time, as we need to build the image from scratch - timeout_minutes: 30 - max_attempts: 3 - command: bash scripts/ci-int-e2e-test.sh int amqp - - - name: appsignal - if: ${{ startsWith(github.event.review.body, '/ci-run-integration-appsignal') - || startsWith(github.event.review.body, '/ci-run-integration-all') - || startsWith(github.event.review.body, '/ci-run-all') }} - uses: nick-fields/retry@v3 - with: - timeout_minutes: 30 - max_attempts: 3 - command: bash scripts/ci-int-e2e-test.sh int appsignal - - - name: aws - if: ${{ startsWith(github.event.review.body, '/ci-run-integration-aws') - || startsWith(github.event.review.body, '/ci-run-integration-all') - || startsWith(github.event.review.body, '/ci-run-all') }} - uses: nick-fields/retry@v3 - with: - timeout_minutes: 30 - max_attempts: 3 - command: bash scripts/ci-int-e2e-test.sh int aws - - - name: axiom - if: ${{ startsWith(github.event.review.body, '/ci-run-integration-axiom') - || startsWith(github.event.review.body, '/ci-run-integration-all') - || startsWith(github.event.review.body, '/ci-run-all') }} - uses: nick-fields/retry@v3 - with: - timeout_minutes: 30 - max_attempts: 3 - command: bash scripts/ci-int-e2e-test.sh int axiom - - - name: azure - if: ${{ startsWith(github.event.review.body, '/ci-run-integration-azure') - || startsWith(github.event.review.body, '/ci-run-integration-all') - || startsWith(github.event.review.body, '/ci-run-all') }} - uses: nick-fields/retry@v3 - with: - timeout_minutes: 30 - max_attempts: 3 - command: bash scripts/ci-int-e2e-test.sh int azure - - - name: clickhouse - if: ${{ startsWith(github.event.review.body, '/ci-run-integration-clickhouse') - || startsWith(github.event.review.body, '/ci-run-integration-all') - || startsWith(github.event.review.body, '/ci-run-all') }} - uses: nick-fields/retry@v3 - with: - timeout_minutes: 30 - max_attempts: 3 - command: bash scripts/ci-int-e2e-test.sh int clickhouse - - - name: databend - if: ${{ startsWith(github.event.review.body, '/ci-run-integration-databend') - || startsWith(github.event.review.body, '/ci-run-integration-all') - || startsWith(github.event.review.body, '/ci-run-all') }} - uses: nick-fields/retry@v3 + - name: Integration Tests - ${{ matrix.service }} + if: ${{ startsWith(github.event.review.body, '/ci-run-integration-all') + || startsWith(github.event.review.body, '/ci-run-all') + || startsWith(github.event.review.body, format('/ci-run-integration-{0}', matrix.service)) }} + uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/ci-int-e2e-test.sh int databend - - - name: datadog-agent - if: ${{ startsWith(github.event.review.body, '/ci-run-integration-datadog-agent') - || startsWith(github.event.review.body, '/ci-run-integration-all') - || startsWith(github.event.review.body, '/ci-run-all') }} - uses: nick-fields/retry@v3 - with: - timeout_minutes: 30 - max_attempts: 3 - command: bash scripts/ci-int-e2e-test.sh int datadog-agent - - - name: datadog-logs - if: ${{ startsWith(github.event.review.body, '/ci-run-integration-datadog-logs') - || startsWith(github.event.review.body, '/ci-run-integration-all') - || startsWith(github.event.review.body, '/ci-run-all') }} - uses: nick-fields/retry@v3 - with: - timeout_minutes: 30 - max_attempts: 3 - command: bash scripts/ci-int-e2e-test.sh int datadog-logs - - - name: datadog-metrics - if: ${{ startsWith(github.event.review.body, '/ci-run-integration-datadog-metrics') - || startsWith(github.event.review.body, '/ci-run-integration-all') - || startsWith(github.event.review.body, '/ci-run-all') }} - uses: nick-fields/retry@v3 - with: - timeout_minutes: 30 - max_attempts: 3 - command: bash scripts/ci-int-e2e-test.sh int datadog-metrics - - - name: datadog-traces - if: ${{ startsWith(github.event.review.body, '/ci-run-integration-datadog-traces') - || startsWith(github.event.review.body, '/ci-run-integration-all') - || startsWith(github.event.review.body, '/ci-run-all') }} - uses: nick-fields/retry@v3 - with: - timeout_minutes: 30 - max_attempts: 3 - command: bash scripts/ci-int-e2e-test.sh int datadog-traces - - - name: dnstap - if: ${{ startsWith(github.event.review.body, '/ci-run-integration-dnstap') - || startsWith(github.event.review.body, '/ci-run-integration-all') - || startsWith(github.event.review.body, '/ci-run-all') }} - uses: nick-fields/retry@v3 - with: - timeout_minutes: 30 - max_attempts: 3 - command: bash scripts/ci-int-e2e-test.sh int dnstap - - - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f - - - name: docker-logs - if: ${{ startsWith(github.event.review.body, '/ci-run-integration-docker-logs') - || startsWith(github.event.review.body, '/ci-run-integration-all') - || startsWith(github.event.review.body, '/ci-run-all') }} - uses: nick-fields/retry@v3 - with: - timeout_minutes: 30 - max_attempts: 3 - command: bash scripts/ci-int-e2e-test.sh int docker-logs - - - name: elasticsearch - if: ${{ startsWith(github.event.review.body, '/ci-run-integration-elasticsearch') - || startsWith(github.event.review.body, '/ci-run-integration-all') - || startsWith(github.event.review.body, '/ci-run-all') }} - uses: nick-fields/retry@v3 - with: - timeout_minutes: 30 - max_attempts: 3 - command: bash scripts/ci-int-e2e-test.sh int elasticsearch - - - name: eventstoredb - if: ${{ startsWith(github.event.review.body, '/ci-run-integration-eventstoredb') - || startsWith(github.event.review.body, '/ci-run-integration-all') - || startsWith(github.event.review.body, '/ci-run-all') }} - uses: nick-fields/retry@v3 - with: - timeout_minutes: 30 - max_attempts: 3 - command: bash scripts/ci-int-e2e-test.sh int eventstoredb - - - name: fluent - if: ${{ startsWith(github.event.review.body, '/ci-run-integration-fluent') - || startsWith(github.event.review.body, '/ci-run-integration-all') - || startsWith(github.event.review.body, '/ci-run-all') }} - uses: nick-fields/retry@v3 - with: - timeout_minutes: 30 - max_attempts: 3 - command: bash scripts/ci-int-e2e-test.sh int fluent - - - name: gcp - if: ${{ startsWith(github.event.review.body, '/ci-run-integration-gcp') - || startsWith(github.event.review.body, '/ci-run-integration-all') - || startsWith(github.event.review.body, '/ci-run-all') }} - uses: nick-fields/retry@v3 - with: - timeout_minutes: 30 - max_attempts: 3 - command: bash scripts/ci-int-e2e-test.sh int gcp - - - name: greptimedb - if: ${{ startsWith(github.event.review.body, '/ci-run-integration-greptimedb') - || startsWith(github.event.review.body, '/ci-run-integration-all') - || startsWith(github.event.review.body, '/ci-run-all') }} - uses: nick-fields/retry@v3 - with: - timeout_minutes: 30 - max_attempts: 3 - command: bash scripts/ci-int-e2e-test.sh int greptimedb - - - name: humio - if: ${{ startsWith(github.event.review.body, '/ci-run-integration-humio') - || startsWith(github.event.review.body, '/ci-run-integration-all') - || startsWith(github.event.review.body, '/ci-run-all') }} - uses: nick-fields/retry@v3 - with: - timeout_minutes: 30 - max_attempts: 3 - command: bash scripts/ci-int-e2e-test.sh int humio - - - name: http-client - if: ${{ startsWith(github.event.review.body, '/ci-run-integration-http-client') - || startsWith(github.event.review.body, '/ci-run-integration-all') - || startsWith(github.event.review.body, '/ci-run-all') }} - uses: nick-fields/retry@v3 - with: - timeout_minutes: 30 - max_attempts: 3 - command: bash scripts/ci-int-e2e-test.sh int http-client - - - name: influxdb - if: ${{ startsWith(github.event.review.body, '/ci-run-integration-influxdb') - || startsWith(github.event.review.body, '/ci-run-integration-all') - || startsWith(github.event.review.body, '/ci-run-all') }} - uses: nick-fields/retry@v3 - with: - timeout_minutes: 30 - max_attempts: 3 - command: bash scripts/ci-int-e2e-test.sh int influxdb - - - name: kafka - if: ${{ startsWith(github.event.review.body, '/ci-run-integration-kafka') - || startsWith(github.event.review.body, '/ci-run-integration-all') - || startsWith(github.event.review.body, '/ci-run-all') }} - uses: nick-fields/retry@v3 - with: - timeout_minutes: 30 - max_attempts: 3 - command: bash scripts/ci-int-e2e-test.sh int kafka - - - name: logstash - if: ${{ startsWith(github.event.review.body, '/ci-run-integration-logstash') - || startsWith(github.event.review.body, '/ci-run-integration-all') - || startsWith(github.event.review.body, '/ci-run-all') }} - uses: nick-fields/retry@v3 - with: - timeout_minutes: 30 - max_attempts: 3 - command: bash scripts/ci-int-e2e-test.sh int logstash - - - name: loki - if: ${{ startsWith(github.event.review.body, '/ci-run-integration-loki') - || startsWith(github.event.review.body, '/ci-run-integration-all') - || startsWith(github.event.review.body, '/ci-run-all') }} - uses: nick-fields/retry@v3 - with: - timeout_minutes: 30 - max_attempts: 3 - command: bash scripts/ci-int-e2e-test.sh int loki - - - name: mongodb - if: ${{ startsWith(github.event.review.body, '/ci-run-integration-mongodb') - || startsWith(github.event.review.body, '/ci-run-integration-all') - || startsWith(github.event.review.body, '/ci-run-all') }} - uses: nick-fields/retry@v3 - with: - timeout_minutes: 30 - max_attempts: 3 - command: bash scripts/ci-int-e2e-test.sh int mongodb - - - name: mqtt - if: ${{ startsWith(github.event.review.body, '/ci-run-integration-mqtt') - || startsWith(github.event.review.body, '/ci-run-integration-all') - || startsWith(github.event.review.body, '/ci-run-all') }} - uses: nick-fields/retry@v3 - with: - timeout_minutes: 30 - max_attempts: 3 - command: bash scripts/ci-int-e2e-test.sh int mqtt - - - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f - - - name: nats - if: ${{ startsWith(github.event.review.body, '/ci-run-integration-nats') - || startsWith(github.event.review.body, '/ci-run-integration-all') - || startsWith(github.event.review.body, '/ci-run-all') }} - uses: nick-fields/retry@v3 - with: - timeout_minutes: 30 - max_attempts: 3 - command: bash scripts/ci-int-e2e-test.sh int nats - - - name: nginx - if: ${{ startsWith(github.event.review.body, '/ci-run-integration-nginx') - || startsWith(github.event.review.body, '/ci-run-integration-all') - || startsWith(github.event.review.body, '/ci-run-all') }} - uses: nick-fields/retry@v3 - with: - timeout_minutes: 30 - max_attempts: 3 - command: bash scripts/ci-int-e2e-test.sh int nginx - - - name: opentelemetry - if: ${{ startsWith(github.event.review.body, '/ci-run-integration-opentelemetry') - || startsWith(github.event.review.body, '/ci-run-integration-all') - || startsWith(github.event.review.body, '/ci-run-all') }} - uses: nick-fields/retry@v3 - with: - timeout_minutes: 30 - max_attempts: 3 - command: bash scripts/ci-int-e2e-test.sh int opentelemetry - - - name: postgres - if: ${{ startsWith(github.event.review.body, '/ci-run-integration-postgres') - || startsWith(github.event.review.body, '/ci-run-integration-all') - || startsWith(github.event.review.body, '/ci-run-all') }} - uses: nick-fields/retry@v3 - with: - timeout_minutes: 30 - max_attempts: 3 - command: bash scripts/ci-int-e2e-test.sh int postgres - - - name: prometheus - if: ${{ startsWith(github.event.review.body, '/ci-run-integration-prometheus') - || startsWith(github.event.review.body, '/ci-run-integration-all') - || startsWith(github.event.review.body, '/ci-run-all') }} - uses: nick-fields/retry@v3 - with: - timeout_minutes: 30 - max_attempts: 3 - command: bash scripts/ci-int-e2e-test.sh int prometheus - - - name: pulsar - if: ${{ startsWith(github.event.review.body, '/ci-run-integration-pulsar') - || startsWith(github.event.review.body, '/ci-run-integration-all') - || startsWith(github.event.review.body, '/ci-run-all') }} - uses: nick-fields/retry@v3 - with: - timeout_minutes: 30 - max_attempts: 3 - command: bash scripts/ci-int-e2e-test.sh int pulsar - - - name: redis - if: ${{ startsWith(github.event.review.body, '/ci-run-integration-redis') - || startsWith(github.event.review.body, '/ci-run-integration-all') - || startsWith(github.event.review.body, '/ci-run-all') }} - uses: nick-fields/retry@v3 - with: - timeout_minutes: 30 - max_attempts: 3 - command: bash scripts/ci-int-e2e-test.sh int redis - - - name: shutdown - if: ${{ startsWith(github.event.review.body, '/ci-run-integration-shutdown') - || startsWith(github.event.review.body, '/ci-run-integration-all') - || startsWith(github.event.review.body, '/ci-run-all') }} - uses: nick-fields/retry@v3 - with: - timeout_minutes: 30 - max_attempts: 3 - command: bash scripts/ci-int-e2e-test.sh int shutdown - - - name: splunk - if: ${{ startsWith(github.event.review.body, '/ci-run-integration-splunk') - || startsWith(github.event.review.body, '/ci-run-integration-all') - || startsWith(github.event.review.body, '/ci-run-all') }} - uses: nick-fields/retry@v3 - with: - timeout_minutes: 30 - max_attempts: 3 - command: bash scripts/ci-int-e2e-test.sh int splunk - - - name: webhdfs - if: ${{ startsWith(github.event.review.body, '/ci-run-integration-webhdfs') - || startsWith(github.event.review.body, '/ci-run-integration-all') - || startsWith(github.event.review.body, '/ci-run-all') }} - uses: nick-fields/retry@v3 - with: - timeout_minutes: 30 - max_attempts: 3 - command: bash scripts/ci-int-e2e-test.sh int webhdfs + command: bash scripts/run-integration-test.sh int ${{ matrix.service }} e2e-tests: needs: prep-pr - runs-on: ubuntu-20.04-8core + runs-on: ubuntu-24.04-8core timeout-minutes: 30 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: submodules: "recursive" ref: ${{ github.event.review.commit_id }} - - run: sudo npm -g install @datadog/datadog-ci + - run: bash scripts/environment/prepare.sh --modules=datadog-ci - run: docker image prune -af ; docker container prune -f - name: e2e-datadog-logs if: ${{ startsWith(github.event.review.body, '/ci-run-e2e-datadog-logs') - || startsWith(github.event.review.body, '/ci-run-integration-all') + || startsWith(github.event.review.body, '/ci-run-e2e-all') || startsWith(github.event.review.body, '/ci-run-all') }} - uses: nick-fields/retry@v3 + uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 with: timeout_minutes: 35 max_attempts: 3 - command: bash scripts/ci-int-e2e-test.sh e2e datadog-logs + command: bash scripts/run-integration-test.sh e2e datadog-logs - name: datadog-e2e-metrics if: ${{ startsWith(github.event.review.body, '/ci-run-e2e-datadog-metrics') - || startsWith(github.event.review.body, '/ci-run-integration-all') + || startsWith(github.event.review.body, '/ci-run-e2e-all') || startsWith(github.event.review.body, '/ci-run-all') }} - uses: nick-fields/retry@v3 + uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 with: timeout_minutes: 35 max_attempts: 3 - command: bash scripts/ci-int-e2e-test.sh e2e datadog-metrics + command: bash scripts/run-integration-test.sh e2e datadog-metrics + + - name: e2e-opentelemetry-logs + if: ${{ startsWith(github.event.review.body, '/ci-run-e2e-opentelemetry-logs') + || startsWith(github.event.review.body, '/ci-run-e2e-all') + || startsWith(github.event.review.body, '/ci-run-all') }} + run: bash scripts/run-integration-test.sh e2e opentelemetry-logs update-pr-status: name: Signal result to PR @@ -503,7 +161,7 @@ jobs: - e2e-tests if: always() && (startsWith(github.event.review.body, '/ci-run-integration') || contains(github.event.review.body, '/ci-run-all')) env: - FAILED: ${{ contains(needs.*.result, 'failure') }} + FAILED: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }} steps: - name: Generate authentication token id: generate_token @@ -514,7 +172,7 @@ jobs: - name: Validate issue comment if: github.event_name == 'pull_request_review' - uses: tspascoal/get-user-teams-membership@v3 + uses: tspascoal/get-user-teams-membership@57e9f42acd78f4d0f496b3be4368fc5f62696662 # v3.0.0 with: username: ${{ github.actor }} team: 'Vector' @@ -522,16 +180,17 @@ jobs: - name: (PR review) Submit PR result as success if: github.event_name == 'pull_request_review' && env.FAILED != 'true' - uses: myrotvorets/set-commit-status-action@v2.0.1 + uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 with: sha: ${{ github.event.review.commit_id }} token: ${{ secrets.GITHUB_TOKEN }} status: 'success' - - run: | - echo "failed=${{ env.FAILED }}" - if [[ "$FAILED" == "true" ]] ; then + - name: Check all jobs status + run: | + if [[ "${{ env.FAILED }}" == "true" ]]; then + echo "One or more jobs failed or were cancelled" exit 1 else - exit 0 + echo "All jobs completed successfully" fi diff --git a/.github/workflows/ci-review-trigger.yml b/.github/workflows/ci-review-trigger.yml index 06f638da5f..6d2ef24c4c 100644 --- a/.github/workflows/ci-review-trigger.yml +++ b/.github/workflows/ci-review-trigger.yml @@ -73,7 +73,7 @@ jobs: private_key: ${{ secrets.GH_APP_DATADOG_VECTOR_CI_APP_PRIVATE_KEY }} - name: Get PR review author id: comment - uses: tspascoal/get-user-teams-membership@v3 + uses: tspascoal/get-user-teams-membership@57e9f42acd78f4d0f496b3be4368fc5f62696662 # v3.0.0 with: username: ${{ github.actor }} team: 'Vector' diff --git a/.github/workflows/cli.yml b/.github/workflows/cli.yml index e797a7abdd..ca5a926e1a 100644 --- a/.github/workflows/cli.yml +++ b/.github/workflows/cli.yml @@ -8,14 +8,14 @@ permissions: jobs: test-cli: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 timeout-minutes: 30 env: CARGO_INCREMENTAL: 0 steps: - name: (PR review) Set latest commit status as pending if: ${{ github.event_name == 'pull_request_review' }} - uses: myrotvorets/set-commit-status-action@v2.0.1 + uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 with: sha: ${{ github.event.review.commit_id }} token: ${{ secrets.GITHUB_TOKEN }} @@ -24,16 +24,16 @@ jobs: - name: (PR review) Checkout review SHA if: ${{ github.event_name == 'pull_request_review' }} - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ github.event.review.commit_id }} - name: Checkout branch if: ${{ github.event_name != 'pull_request_review' }} - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Cache Cargo registry + index - uses: actions/cache@v4 + uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 with: path: | ~/.cargo/bin/ @@ -44,8 +44,8 @@ jobs: restore-keys: | ${{ runner.os }}-cargo- - - run: sudo -E bash scripts/environment/bootstrap-ubuntu-20.04.sh - - run: bash scripts/environment/prepare.sh + - run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh + - run: bash scripts/environment/prepare.sh --modules=cargo-nextest,datadog-ci - run: echo "::add-matcher::.github/matchers/rust.json" - run: make test-cli - name: Upload test results @@ -53,7 +53,7 @@ jobs: if: always() - name: (PR review) Set latest commit status as ${{ job.status }} - uses: myrotvorets/set-commit-status-action@v2.0.1 + uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 if: always() && github.event_name == 'pull_request_review' with: sha: ${{ github.event.review.commit_id }} diff --git a/.github/workflows/compilation-timings.yml b/.github/workflows/compilation-timings.yml index 1404abb64c..2b1c95d3a6 100644 --- a/.github/workflows/compilation-timings.yml +++ b/.github/workflows/compilation-timings.yml @@ -13,50 +13,50 @@ env: jobs: release-build-optimized: name: "Release Build (optimized)" - runs-on: ubuntu-20.04-8core + runs-on: ubuntu-24.04-8core steps: - - uses: colpal/actions-clean@v1 - - uses: actions/checkout@v4 - - run: sudo -E bash scripts/environment/bootstrap-ubuntu-20.04.sh - - run: bash scripts/environment/prepare.sh + - uses: colpal/actions-clean@36e6ca1abd35efe61cb60f912bd7837f67887c8a # v1.1.1 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh + - run: bash scripts/environment/prepare.sh --modules=rustup - run: cargo clean - run: cargo build --release release-build-normal: name: "Release Build (normal)" - runs-on: ubuntu-20.04-8core + runs-on: ubuntu-24.04-8core env: # We're not actually doing a debug build, we're just turning off the logic # in release-flags.sh so that we don't override the Cargo "release" profile # with full LTO / single codegen unit. PROFILE: debug steps: - - uses: colpal/actions-clean@v1 - - uses: actions/checkout@v4 - - run: sudo -E bash scripts/environment/bootstrap-ubuntu-20.04.sh - - run: bash scripts/environment/prepare.sh + - uses: colpal/actions-clean@36e6ca1abd35efe61cb60f912bd7837f67887c8a # v1.1.1 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh + - run: bash scripts/environment/prepare.sh --modules=rustup - run: cargo clean - run: cargo build --release debug-build: name: "Debug Build" - runs-on: ubuntu-20.04-8core + runs-on: ubuntu-24.04-8core steps: - - uses: colpal/actions-clean@v1 - - uses: actions/checkout@v4 - - run: sudo -E bash scripts/environment/bootstrap-ubuntu-20.04.sh - - run: bash scripts/environment/prepare.sh + - uses: colpal/actions-clean@36e6ca1abd35efe61cb60f912bd7837f67887c8a # v1.1.1 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh + - run: bash scripts/environment/prepare.sh --modules=rustup - run: cargo clean - run: cargo build debug-rebuild: name: "Debug Rebuild" - runs-on: ubuntu-20.04-8core + runs-on: ubuntu-24.04-8core steps: - - uses: colpal/actions-clean@v1 - - uses: actions/checkout@v4 - - run: sudo -E bash scripts/environment/bootstrap-ubuntu-20.04.sh - - run: bash scripts/environment/prepare.sh + - uses: colpal/actions-clean@36e6ca1abd35efe61cb60f912bd7837f67887c8a # v1.1.1 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh + - run: bash scripts/environment/prepare.sh --modules=rustup - run: cargo clean - run: cargo build - run: touch src/app.rs @@ -64,11 +64,11 @@ jobs: check: name: "Cargo Check" - runs-on: ubuntu-20.04-8core + runs-on: ubuntu-24.04-8core steps: - - uses: colpal/actions-clean@v1 - - uses: actions/checkout@v4 - - run: sudo -E bash scripts/environment/bootstrap-ubuntu-20.04.sh - - run: bash scripts/environment/prepare.sh + - uses: colpal/actions-clean@36e6ca1abd35efe61cb60f912bd7837f67887c8a # v1.1.1 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh + - run: bash scripts/environment/prepare.sh --modules=rustup - run: cargo clean - run: cargo check diff --git a/.github/workflows/component_features.yml b/.github/workflows/component_features.yml index a8a05af06d..a42080077a 100644 --- a/.github/workflows/component_features.yml +++ b/.github/workflows/component_features.yml @@ -22,12 +22,12 @@ permissions: jobs: check-component-features: # use free tier on schedule and 8 core to expedite results on demand invocation - runs-on: ${{ github.event_name == 'schedule' && 'ubuntu-20.04' || 'ubuntu-20.04-8core' }} + runs-on: ${{ github.event_name == 'schedule' && 'ubuntu-24.04' || 'ubuntu-24.04-8core' }} if: github.event_name == 'pull_request_review' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' steps: - name: (PR review) Set latest commit status as pending if: github.event_name == 'pull_request_review' - uses: myrotvorets/set-commit-status-action@v2.0.1 + uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 with: sha: ${{ github.event.review.commit_id }} token: ${{ secrets.GITHUB_TOKEN }} @@ -36,22 +36,22 @@ jobs: - name: (PR review) Checkout PR branch if: github.event_name == 'pull_request_review' - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ github.event.review.commit_id }} - name: Checkout branch if: github.event_name != 'pull_request_review' - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - run: sudo -E bash scripts/environment/bootstrap-ubuntu-20.04.sh - - run: bash scripts/environment/prepare.sh + - run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh + - run: bash scripts/environment/prepare.sh --modules=rustup - run: echo "::add-matcher::.github/matchers/rust.json" - run: make check-component-features - name: (PR review) Set latest commit status as ${{ job.status }} if: always() && github.event_name == 'pull_request_review' - uses: myrotvorets/set-commit-status-action@v2.0.1 + uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 with: sha: ${{ github.event.review.commit_id }} token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/create_preview_sites.yml b/.github/workflows/create_preview_sites.yml index 1e79c621a7..47ce7bced9 100644 --- a/.github/workflows/create_preview_sites.yml +++ b/.github/workflows/create_preview_sites.yml @@ -26,84 +26,159 @@ permissions: issues: write pull-requests: write statuses: write + actions: read jobs: create_preview_site: runs-on: ubuntu-24.04 timeout-minutes: 5 steps: + # Get the artifacts with the PR number and branch name + - name: Download artifact + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + script: | + const fs = require('fs'); + const artifacts = await github.rest.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: ${{ github.event.workflow_run.id }}, + }); + const matchArtifact = artifacts.data.artifacts.filter(artifact => artifact.name == "pr")[0]; + const download = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: matchArtifact.id, + archive_format: 'zip', + }); + fs.writeFileSync('${{ github.workspace }}/pr.zip', Buffer.from(download.data)); - # Get the artifacts with the PR number and branch name - - name: Download artifact - uses: actions/github-script@v7.0.1 - with: - script: | - const fs = require('fs'); - const artifacts = await github.rest.actions.listWorkflowRunArtifacts({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: ${{ github.event.workflow_run.id }}, - }); - const matchArtifact = artifacts.data.artifacts.filter(artifact => artifact.name == "pr")[0]; - const download = await github.rest.actions.downloadArtifact({ - owner: context.repo.owner, - repo: context.repo.repo, - artifact_id: matchArtifact.id, - archive_format: 'zip', - }); - fs.writeFileSync('${{ github.workspace }}/pr.zip', Buffer.from(download.data)); - - # Extract the info from the artifact and set variables - - name: Extract PR info from artifact - run: | - unzip pr.zip -d pr - BRANCH_NAME=$(cat ./pr/branch) - SANITIZED_BRANCH_NAME=$(echo "$BRANCH_NAME" | sed 's/[\/\.]/-/g') - echo "SANITIZED_BRANCH_NAME=$SANITIZED_BRANCH_NAME" >> $GITHUB_ENV - echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV - - # Kick off the job in amplify - - name: Deploy Site - env: - APP_ID: ${{ inputs.APP_ID }} - APP_NAME: ${{ inputs.APP_NAME }} - REQUEST_TOKEN: ${{ secrets.REQUEST_TOKEN }} - REQUEST_MESSAGE: ${{ secrets.REQUEST_MESSAGE }} - ENDPOINT: ${{ secrets.ENDPOINT }} - run: | - sleep 20 - HMAC_KEY=$(echo -n $REQUEST_MESSAGE | openssl dgst -sha256 -hmac "$REQUEST_TOKEN" | cut -d" " -f2) - SIGNATURE="sha256=$HMAC_KEY" - RESPONSE_CODE=$(curl -s -o /dev/null -w "%{http_code}" -X POST \ - -H "Content-Type: application/json" \ - -H "X-Hub-Signature: $SIGNATURE" \ - -d "{\"app_id\": \"$APP_ID\", \"branch_name\": \"$BRANCH_NAME\"}" \ - "$ENDPOINT") - - # check the response code and fail if not 200 - if [ "$RESPONSE_CODE" != "200" ]; then - echo "Request failed with response code $RESPONSE_CODE" - exit 1 - fi - - # Add preview link to comment if all 3 sites successfully start - - name: Comment Preview Link - if: success() - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - APP_ID: ${{ inputs.APP_ID }} - APP_NAME: ${{ inputs.APP_NAME }} - uses: actions/github-script@v7.0.1 - with: - script: | - const fs = require('fs'); - const prNumber = fs.readFileSync('./pr/number', 'utf8'); - const issueNumber = parseInt(prNumber); - const { APP_ID, APP_NAME, SANITIZED_BRANCH_NAME } = process.env - - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: issueNumber, - body: `Your preview site for the **${APP_NAME}** will be ready in a few minutes, please allow time for it to build. \n \n Heres your preview link: \n [${APP_NAME} preview](https://${SANITIZED_BRANCH_NAME}.${APP_ID}.amplifyapp.com)` - }); + # Extract the info from the artifact and set variables + - name: Extract PR info from artifact + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + script: | + const fs = require('fs'); + const { execSync } = require('child_process'); + const path = require('path'); + + execSync('unzip pr.zip -d pr'); + const branchName = fs.readFileSync(path.join('pr', 'branch'), 'utf8').trim(); + const prNumber = fs.readFileSync(path.join('pr', 'number'), 'utf8').trim(); + const integrity = fs.readFileSync(path.join('pr', 'integrity'), 'utf8').trim(); + + // Validate branch name again (only allow alphanumeric, dash, and underscore) + const branchNameRegex = /^[a-zA-Z0-9_\-]+$/; + if (!branchNameRegex.test(branchName)) { + core.setFailed(`Invalid branch name detected: ${branchName}`); + return; + } + + const sanitizedBranchName = branchName.replace(/[\/\.]/g, '-'); + core.exportVariable('SANITIZED_BRANCH_NAME', sanitizedBranchName); + core.exportVariable('BRANCH_NAME', branchName); + core.exportVariable('PR_NUMBER', prNumber); + core.exportVariable('INTEGRITY', integrity); + + # Validate the integrity of the artifact + - name: Validate Artifact Integrity + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + script: | + const crypto = require('crypto'); + + const prNumber = process.env.PR_NUMBER; + const branchName = process.env.BRANCH_NAME; + const integrity = process.env.INTEGRITY; + + const numberHash = crypto.createHash('sha256').update(prNumber).digest('hex'); + const branchHash = crypto.createHash('sha256').update(branchName).digest('hex'); + const expectedIntegrity = `${numberHash}:${branchHash}`; + + if (expectedIntegrity !== integrity) { + core.setFailed('Artifact integrity validation failed'); + } + + # Kick off the job in amplify + - name: Deploy Site + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + env: + APP_ID: ${{ inputs.APP_ID }} + APP_NAME: ${{ inputs.APP_NAME }} + REQUEST_TOKEN: ${{ secrets.REQUEST_TOKEN }} + REQUEST_MESSAGE: ${{ secrets.REQUEST_MESSAGE }} + ENDPOINT: ${{ secrets.ENDPOINT }} + with: + script: | + const crypto = require('crypto'); + const https = require('https'); + + // Access secrets through environment variables + const appId = process.env.APP_ID; + const appName = process.env.APP_NAME; + const requestToken = process.env.REQUEST_TOKEN; + const requestMessage = process.env.REQUEST_MESSAGE; + const endpoint = process.env.ENDPOINT; + const sanitizedBranchName = process.env.SANITIZED_BRANCH_NAME; + + const hmacKey = crypto.createHmac('sha256', requestToken).update(requestMessage).digest('hex'); + const signature = `sha256=${hmacKey}`; + + const makeRequest = () => { + return new Promise((resolve, reject) => { + const options = { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-Hub-Signature': signature, + } + }; + + const req = https.request(endpoint, options, (res) => { + let data = ''; + res.on('data', (chunk) => { data += chunk; }); + res.on('end', () => { + resolve({ ok: res.statusCode >= 200 && res.statusCode < 300, status: res.statusCode }); + }); + }); + + req.on('error', (error) => { + reject(error); + }); + + req.write(JSON.stringify({ + app_id: appId, + branch_name: sanitizedBranchName, + })); + + req.end(); + }); + }; + + const response = await makeRequest(); + + if (!response.ok) { + core.setFailed(`Request failed with response code ${response.status}`); + } + + # Add preview link to comment if all 3 sites successfully start + - name: Comment Preview Link + if: success() + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + APP_ID: ${{ inputs.APP_ID }} + APP_NAME: ${{ inputs.APP_NAME }} + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + script: | + const fs = require('fs'); + const prNumber = fs.readFileSync('./pr/number', 'utf8'); + const issueNumber = parseInt(prNumber); + const { APP_ID, APP_NAME, SANITIZED_BRANCH_NAME } = process.env; + + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issueNumber, + body: `Your preview site for the **${APP_NAME}** will be ready in a few minutes, please allow time for it to build. \n \n Heres your preview link: \n [${APP_NAME} preview](https://${SANITIZED_BRANCH_NAME}.${APP_ID}.amplifyapp.com)` + }); diff --git a/.github/workflows/cross.yml b/.github/workflows/cross.yml index d9040040ff..1fb58eda09 100644 --- a/.github/workflows/cross.yml +++ b/.github/workflows/cross.yml @@ -9,7 +9,7 @@ permissions: jobs: cross-linux: name: Cross - ${{ matrix.target }} - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 timeout-minutes: 45 env: CARGO_INCREMENTAL: 0 @@ -27,7 +27,7 @@ jobs: steps: - name: (PR review) Set latest commit status as pending if: ${{ github.event_name == 'pull_request_review' }} - uses: myrotvorets/set-commit-status-action@v2.0.1 + uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 with: sha: ${{ github.event.review.commit_id }} token: ${{ secrets.GITHUB_TOKEN }} @@ -36,15 +36,15 @@ jobs: - name: (PR review) Checkout PR branch if: ${{ github.event_name == 'pull_request_review' }} - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ github.event.review.commit_id }} - name: Checkout branch if: ${{ github.event_name != 'pull_request_review' }} - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - uses: actions/cache@v4 + - uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 name: Cache Cargo registry + index with: path: | @@ -57,18 +57,20 @@ jobs: ${{ runner.os }}-cargo- - run: echo "::add-matcher::.github/matchers/rust.json" - - run: 'cargo install cross --version 0.2.4 --force --locked' + - run: | + rustup target add x86_64-unknown-linux-gnu + cargo install cross --version 0.2.4 --force --locked # Why is this build, not check? Because we need to make sure the linking phase works. # aarch64 and musl in particular are notoriously hard to link. # While it may be tempting to slot a `check` in here for quickness, please don't. - run: make cross-build-${{ matrix.target }} - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: "vector-debug-${{ matrix.target }}" path: "./target/${{ matrix.target }}/debug/vector" - name: (PR review) Set latest commit status as failed - uses: myrotvorets/set-commit-status-action@v2.0.1 + uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 if: failure() && github.event_name == 'pull_request_review' with: sha: ${{ steps.comment-branch.outputs.head_sha }} @@ -78,13 +80,13 @@ jobs: update-pr-status: name: (PR review) Signal result to PR - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 timeout-minutes: 5 needs: cross-linux if: needs.cross-linux.result == 'success' && github.event_name == 'pull_request_review' steps: - name: (PR review) Submit PR result as success - uses: myrotvorets/set-commit-status-action@v2.0.1 + uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 with: sha: ${{ github.event.review.commit_id }} token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/deny.yml b/.github/workflows/deny.yml index c69828a537..ac2f323b4e 100644 --- a/.github/workflows/deny.yml +++ b/.github/workflows/deny.yml @@ -21,14 +21,14 @@ permissions: jobs: test-deny: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 timeout-minutes: 30 env: CARGO_INCREMENTAL: 0 steps: - name: (PR review) Set latest commit status as pending if: ${{ github.event_name == 'pull_request_review' }} - uses: myrotvorets/set-commit-status-action@v2.0.1 + uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 with: sha: ${{ github.event.review.commit_id }} token: ${{ secrets.GITHUB_TOKEN }} @@ -37,15 +37,15 @@ jobs: - name: (PR review) Checkout PR branch if: ${{ github.event_name == 'pull_request_review' }} - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ github.event.review.commit_id }} - name: Checkout branch if: ${{ github.event_name != 'pull_request_review' }} - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - uses: actions/cache@v4 + - uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 name: Cache Cargo registry + index with: path: | @@ -57,14 +57,14 @@ jobs: restore-keys: | ${{ runner.os }}-cargo- - - run: sudo -E bash scripts/environment/bootstrap-ubuntu-20.04.sh - - run: bash scripts/environment/prepare.sh + - run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh + - run: bash scripts/environment/prepare.sh --modules=cargo-deny - run: echo "::add-matcher::.github/matchers/rust.json" - name: Check cargo deny advisories/licenses run: make check-deny - name: (PR review) Set latest commit status as ${{ job.status }} - uses: myrotvorets/set-commit-status-action@v2.0.1 + uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 if: always() && github.event_name == 'pull_request_review' with: sha: ${{ github.event.review.commit_id }} diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index a660339f27..3e1dfa9834 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -4,14 +4,16 @@ # - PRs if there are code changes to the source files that are noted in `vdev e2e ci_paths` # - MQ (always pass) # - Scheduled: at midnight UTC Tues-Sat +# - Manual trigger via Actions UI. name: E2E Test Suite on: + workflow_dispatch: pull_request: # Needs to pass by default in MQ merge_group: - types: [checks_requested] + types: [ checks_requested ] schedule: # At midnight UTC Tue-Sat - cron: '0 0 * * 2-6' @@ -37,37 +39,34 @@ env: # observing issues fetching boringssl via HTTPS in the OSX build, seeing if this helps # can be removed when we switch back to the upstream openssl-sys crate CARGO_NET_GIT_FETCH_WITH_CLI: true - jobs: changes: if: github.event_name == 'pull_request' uses: ./.github/workflows/changes.yml with: - base_ref: ${{ github.event.pull_request.base.ref }} - head_ref: ${{ github.event.pull_request.head.ref }} source: false e2e_tests: true secrets: inherit e2e-tests: name: E2E Tests - runs-on: ubuntu-20.04-8core + runs-on: ubuntu-24.04-8core timeout-minutes: 45 needs: changes if: always() && ( - github.event_name == 'schedule' || ( - needs.changes.outputs.all-e2e == 'true' - || needs.changes.outputs.e2e-datadog-logs == 'true' - || needs.changes.outputs.e2e-datadog-metrics == 'true' - ) + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + needs.changes.outputs.e2e-datadog-logs == 'true' || + needs.changes.outputs.e2e-datadog-metrics == 'true' || + needs.changes.outputs.e2e-opentelemetry-logs == 'true' ) steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: submodules: "recursive" - - run: sudo npm -g install @datadog/datadog-ci + - run: bash scripts/environment/prepare.sh --modules=datadog-ci - run: docker image prune -af ; docker container prune -f @@ -82,23 +81,34 @@ jobs: echo "PR_HAS_ACCESS_TO_SECRETS=false" >> "$GITHUB_ENV" fi - - if: (github.event_name == 'schedule' || needs.changes.outputs.all-e2e == 'true' || needs.changes.outputs.e2e-datadog-logs == 'true') && + - if: (github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + needs.changes.outputs.e2e-datadog-logs == 'true') && (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') name: e2e-datadog-logs - uses: nick-fields/retry@v3 + uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 with: timeout_minutes: 35 max_attempts: 3 - command: bash scripts/ci-int-e2e-test.sh e2e datadog-logs + command: bash scripts/run-integration-test.sh e2e datadog-logs - - if: (github.event_name == 'schedule' || needs.changes.outputs.all-e2e == 'true' || needs.changes.outputs.e2e-datadog-metrics == 'true') && + - if: (github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + needs.changes.outputs.e2e-datadog-metrics == 'true') && (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') name: e2e-datadog-metrics - uses: nick-fields/retry@v3 + uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 with: timeout_minutes: 35 max_attempts: 3 - command: bash scripts/ci-int-e2e-test.sh e2e datadog-metrics + command: bash scripts/run-integration-test.sh e2e datadog-metrics + + - if: (github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + needs.changes.outputs.e2e-opentelemetry-logs == 'true') && + (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') + name: e2e-opentelemetry-logs + run: bash scripts/run-integration-test.sh e2e opentelemetry-logs e2e-test-suite: @@ -107,13 +117,12 @@ jobs: timeout-minutes: 5 if: always() needs: e2e-tests - env: - FAILED: ${{ contains(needs.*.result, 'failure') }} steps: - - run: | - echo "failed=${{ env.FAILED }}" - if [[ "$FAILED" == "true" ]] ; then + - name: Check all jobs status + run: | + if [[ "${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}" == "true" ]]; then + echo "One or more jobs failed or were cancelled" exit 1 else - exit 0 + echo "All jobs completed successfully" fi diff --git a/.github/workflows/environment.yml b/.github/workflows/environment.yml index e155eb811f..e979f8bd47 100644 --- a/.github/workflows/environment.yml +++ b/.github/workflows/environment.yml @@ -16,12 +16,12 @@ permissions: jobs: publish-new-environment: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 timeout-minutes: 30 steps: - name: (PR review) Set latest commit status as pending if: ${{ github.event_name == 'pull_request_review' }} - uses: myrotvorets/set-commit-status-action@v2.0.1 + uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 with: sha: ${{ github.event.review.commit_id }} token: ${{ secrets.GITHUB_TOKEN }} @@ -30,27 +30,27 @@ jobs: - name: (PR review) Checkout PR branch if: ${{ github.event_name == 'pull_request_review' }} - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ github.event.review.commit_id }} - name: Checkout branch if: ${{ github.event_name != 'pull_request_review' }} - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Set up QEMU - uses: docker/setup-qemu-action@v3.3.0 + uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3.8.0 + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 - name: Login to DockerHub - uses: docker/login-action@v3 + uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0 if: github.ref == 'refs/heads/master' with: username: ${{ secrets.CI_DOCKER_USERNAME }} password: ${{ secrets.CI_DOCKER_PASSWORD }} - name: Extract metadata (tags, labels) for Docker id: meta - uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 + uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5.8.0 with: images: timberio/vector-dev flavor: | @@ -62,7 +62,7 @@ jobs: org.opencontainers.image.title=Vector development environment org.opencontainers.image.url=https://github.com/vectordotdev/vector - name: Build and push - uses: docker/build-push-action@v6.11.0 + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 with: context: . file: ./scripts/environment/Dockerfile @@ -71,7 +71,7 @@ jobs: labels: ${{ steps.meta.outputs.labels }} - name: (PR review) Set latest commit status as ${{ job.status }} - uses: myrotvorets/set-commit-status-action@v2.0.1 + uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 if: always() && github.event_name == 'pull_request_review' with: sha: ${{ github.event.review.commit_id }} diff --git a/.github/workflows/gardener_issue_comment.yml b/.github/workflows/gardener_issue_comment.yml index 1e38590b7c..b9936a1ede 100644 --- a/.github/workflows/gardener_issue_comment.yml +++ b/.github/workflows/gardener_issue_comment.yml @@ -26,7 +26,7 @@ jobs: - name: Get PR comment author id: comment - uses: tspascoal/get-user-teams-membership@v3 + uses: tspascoal/get-user-teams-membership@57e9f42acd78f4d0f496b3be4368fc5f62696662 # v3.0.0 with: username: ${{ github.actor }} team: 'Vector' diff --git a/.github/workflows/gardener_open_issue.yml b/.github/workflows/gardener_open_issue.yml index 303e8d5147..c8ba082d3e 100644 --- a/.github/workflows/gardener_open_issue.yml +++ b/.github/workflows/gardener_open_issue.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 5 steps: - - uses: actions/add-to-project@v1.0.2 + - uses: actions/add-to-project@244f685bbc3b7adfa8466e08b698b5577571133e # v1.0.2 with: project-url: https://github.com/orgs/vectordotdev/projects/49 github-token: ${{ secrets.GH_PROJECT_PAT }} diff --git a/.github/workflows/gardener_open_pr.yml b/.github/workflows/gardener_open_pr.yml index c299c9318a..873c9da7e2 100644 --- a/.github/workflows/gardener_open_pr.yml +++ b/.github/workflows/gardener_open_pr.yml @@ -20,13 +20,13 @@ jobs: with: app_id: ${{ secrets.GH_APP_DATADOG_VECTOR_CI_APP_ID }} private_key: ${{ secrets.GH_APP_DATADOG_VECTOR_CI_APP_PRIVATE_KEY }} - - uses: tspascoal/get-user-teams-membership@v3 + - uses: tspascoal/get-user-teams-membership@57e9f42acd78f4d0f496b3be4368fc5f62696662 # v3.0.0 id: checkVectorMember with: username: ${{ github.actor }} team: vector GITHUB_TOKEN: ${{ steps.generate_token.outputs.token }} - - uses: actions/add-to-project@v1.0.2 + - uses: actions/add-to-project@244f685bbc3b7adfa8466e08b698b5577571133e # v1.0.2 if: ${{ steps.checkVectorMember.outputs.isTeamMember == 'false' }} with: project-url: https://github.com/orgs/vectordotdev/projects/49 @@ -37,7 +37,7 @@ jobs: timeout-minutes: 5 if: ${{ github.actor == 'dependabot[bot]' }} steps: - - uses: actions/add-to-project@v1.0.2 + - uses: actions/add-to-project@244f685bbc3b7adfa8466e08b698b5577571133e # v1.0.2 with: project-url: https://github.com/orgs/vectordotdev/projects/49 github-token: ${{ secrets.GH_PROJECT_PAT }} diff --git a/.github/workflows/gardener_remove_waiting_author.yml b/.github/workflows/gardener_remove_waiting_author.yml index a9e7f52cb3..ce80b9c0d0 100644 --- a/.github/workflows/gardener_remove_waiting_author.yml +++ b/.github/workflows/gardener_remove_waiting_author.yml @@ -1,15 +1,23 @@ name: Remove Awaiting Author Label +permissions: + pull-requests: write + on: + pull_request_target: + types: [synchronize, review_requested] + pull_request: types: [synchronize, review_requested] jobs: remove_label: + if: "contains(github.event.pull_request.labels.*.name, 'meta: awaiting author')" runs-on: ubuntu-24.04 timeout-minutes: 5 steps: - - uses: actions/checkout@v4 - - uses: actions-ecosystem/action-remove-labels@v1 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0 # v1.3.0 with: labels: "meta: awaiting author" + fail_on_error: true diff --git a/.github/workflows/install-sh.yml b/.github/workflows/install-sh.yml index d33a10e4d0..1722044391 100644 --- a/.github/workflows/install-sh.yml +++ b/.github/workflows/install-sh.yml @@ -9,17 +9,17 @@ permissions: jobs: test-install: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 timeout-minutes: 5 steps: - name: (PR comment) Get PR branch if: ${{ github.event_name == 'issue_comment' }} - uses: xt0rted/pull-request-comment-branch@v3 + uses: xt0rted/pull-request-comment-branch@e8b8daa837e8ea7331c0003c9c316a64c6d8b0b1 # v3.0.0 id: comment-branch - name: (PR comment) Set latest commit status as pending if: ${{ github.event_name == 'issue_comment' }} - uses: myrotvorets/set-commit-status-action@v2.0.1 + uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 with: sha: ${{ steps.comment-branch.outputs.head_sha }} token: ${{ secrets.GITHUB_TOKEN }} @@ -28,13 +28,13 @@ jobs: - name: (PR comment) Checkout PR branch if: ${{ github.event_name == 'issue_comment' }} - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ steps.comment-branch.outputs.head_ref }} - name: Checkout branch if: ${{ github.event_name != 'issue_comment' }} - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - run: sudo apt-get install --yes bc - run: bash distribution/install.sh -- -y @@ -42,7 +42,7 @@ jobs: - name: (PR comment) Set latest commit status as ${{ job.status }} if: github.event_name == 'issue_comment' - uses: myrotvorets/set-commit-status-action@v2.0.1 + uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 with: sha: ${{ steps.comment-branch.outputs.head_sha }} token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 737884eb37..61d37674fd 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -15,8 +15,6 @@ on: type: string env: - AWS_ACCESS_KEY_ID: "dummy" - AWS_SECRET_ACCESS_KEY: "dummy" AXIOM_TOKEN: ${{ secrets.AXIOM_TOKEN }} TEST_APPSIGNAL_PUSH_API_KEY: ${{ secrets.TEST_APPSIGNAL_PUSH_API_KEY }} CONTAINER_TOOL: "docker" @@ -30,26 +28,26 @@ env: jobs: test-integration: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 timeout-minutes: 40 if: inputs.if || github.event_name == 'workflow_dispatch' steps: - name: (PR comment) Get PR branch if: ${{ github.event_name == 'issue_comment' }} - uses: xt0rted/pull-request-comment-branch@v3 + uses: xt0rted/pull-request-comment-branch@e8b8daa837e8ea7331c0003c9c316a64c6d8b0b1 # v3.0.0 id: comment-branch - name: (PR comment) Checkout PR branch if: ${{ github.event_name == 'issue_comment' }} - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ steps.comment-branch.outputs.head_ref }} - name: Checkout branch if: ${{ github.event_name != 'issue_comment' }} - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - run: sudo npm -g install @datadog/datadog-ci + - run: bash scripts/environment/prepare.sh --modules=rustup,datadog-ci - run: make test-integration-${{ inputs.test_name }} env: diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index eca811b3a2..5d46c5c1db 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -7,9 +7,10 @@ name: Integration Test Suite on: + workflow_dispatch: pull_request: merge_group: - types: [checks_requested] + types: [ checks_requested ] concurrency: # `github.event.number` exists for pull requests, otherwise fall back to SHA for merge queue @@ -17,8 +18,6 @@ concurrency: cancel-in-progress: true env: - AWS_ACCESS_KEY_ID: "dummy" - AWS_SECRET_ACCESS_KEY: "dummy" CONTAINER_TOOL: "docker" DD_ENV: "ci" DD_API_KEY: ${{ secrets.DD_API_KEY }} @@ -36,106 +35,89 @@ env: jobs: changes: - if: github.event_name == 'pull_request' + if: github.event_name == 'pull_request' || github.event_name == 'merge_group' uses: ./.github/workflows/changes.yml with: - base_ref: ${{ github.event.pull_request.base.ref }} - head_ref: ${{ github.event.pull_request.head.ref }} - source: false + source: true int_tests: true secrets: inherit - check-all: + check-secrets: runs-on: ubuntu-latest - needs: changes outputs: - should_run: ${{ steps.check.outputs.should_run }} + can_access_secrets: ${{ steps.secret_check.outputs.can_access_secrets }} steps: - - name: Download JSON artifact from changes.yml - uses: actions/download-artifact@v4 - with: - name: int_tests_changes - - - name: Check if all values are false - id: check - run: | - # Check if all values are 'false' - json=$(cat int_tests_changes.json) - all_false=$(echo "$json" | jq -r 'to_entries | all(.value == false)') - - if [[ "$all_false" == "true" ]]; then - echo "No changes detected. Skipping integration tests." - echo "should_run=false" >> $GITHUB_OUTPUT - else - echo "Detected changes. Proceeding with integration tests." - echo "should_run=true" >> $GITHUB_OUTPUT - fi - - setup: - runs-on: ubuntu-latest - needs: check-all - if: ${{ needs.check-all.outputs.should_run == 'true' }} - steps: - - uses: actions/checkout@v4 - - - run: sudo npm -g install @datadog/datadog-ci - - run: sudo -E bash scripts/ci-free-disk-space.sh - - run: docker image prune -af ; docker container prune -f + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Determine if secrets are defined (PR author is team member) - if: github.event_name == 'pull_request' + id: secret_check env: GH_APP_DATADOG_VECTOR_CI_APP_ID: ${{ secrets.GH_APP_DATADOG_VECTOR_CI_APP_ID }} run: | - if [[ "$GH_APP_DATADOG_VECTOR_CI_APP_ID" != "" ]] ; then - echo "PR_HAS_ACCESS_TO_SECRETS=true" >> "$GITHUB_ENV" + if [[ "$GH_APP_DATADOG_VECTOR_CI_APP_ID" != "" ]]; then + echo "can_access_secrets=true" >> $GITHUB_OUTPUT else - echo "PR_HAS_ACCESS_TO_SECRETS=false" >> "$GITHUB_ENV" + echo "can_access_secrets=false" >> $GITHUB_OUTPUT fi integration-tests: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 needs: - changes - - setup + - check-secrets + + if: ${{ !failure() && !cancelled() && (needs.check-secrets.outputs.can_access_secrets == 'true' || github.event_name == 'merge_group') }} strategy: matrix: + # TODO: Add "splunk" back once https://github.com/vectordotdev/vector/issues/23474 is fixed. # If you modify this list, please also update the `int_tests` job in changes.yml. service: [ - "amqp", "appsignal", "aws", "axiom", "azure", "clickhouse", "databend", "datadog-agent", + "amqp", "appsignal", "axiom", "aws", "azure", "clickhouse", "databend", "datadog-agent", "datadog-logs", "datadog-metrics", "datadog-traces", "dnstap", "docker-logs", "elasticsearch", "eventstoredb", "fluent", "gcp", "greptimedb", "http-client", "influxdb", "kafka", "logstash", "loki", "mongodb", "nats", "nginx", "opentelemetry", "postgres", "prometheus", "pulsar", - "redis", "splunk", "webhdfs" + "redis", "webhdfs" ] timeout-minutes: 90 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: submodules: "recursive" - - run: docker image prune -af ; docker container prune -f - - name: Download JSON artifact from changes.yml - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + if: github.event_name == 'pull_request' || github.event_name == 'merge_group' with: name: int_tests_changes - name: Run Integration Tests for ${{ matrix.service }} - uses: nick-fields/retry@v3 + uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 with: timeout_minutes: 30 max_attempts: 3 command: | - # Parse the JSON and check if the specific integration test should run. - should_run=$(jq '."${{ matrix.service }}"' int_tests_changes.json) + if [[ -f int_tests_changes.json ]]; then + # Parse the JSON and check if the specific integration test should run. + should_run=$(jq -r '."${{ matrix.service }}" // false' int_tests_changes.json) + else + # The `changes` job did not run (manual run) or the file is missing, default to false. + should_run=false + fi + + if [[ "${{ needs.changes.outputs.website_only }}" == "true" ]]; then + echo "Skipping ${{ matrix.service }} test since only website changes were detected" + exit 0 + fi # Check if any of the three conditions is true if [[ "${{ github.event_name }}" == "merge_group" || \ - "${{ needs.changes.outputs.all-int }}" == "true" || \ + "${{ github.event_name }}" == "workflow_dispatch" || \ + "${{ needs.changes.outputs.dependencies }}" == "true" || \ "$should_run" == "true" ]]; then + # Only install dep if test runs + bash scripts/environment/prepare.sh --modules=datadog-ci echo "Running test for ${{ matrix.service }}" - bash scripts/ci-int-e2e-test.sh int ${{ matrix.service }} + bash scripts/run-integration-test.sh int ${{ matrix.service }} else echo "Skipping ${{ matrix.service }} test as the value is false or conditions not met." fi @@ -144,20 +126,14 @@ jobs: integration-test-suite: name: Integration Test Suite runs-on: ubuntu-24.04 - timeout-minutes: 5 if: always() needs: - - changes - - check-all - - setup - integration-tests - env: - FAILED: ${{ contains(needs.*.result, 'failure') }} steps: - run: | - echo "failed=${{ env.FAILED }}" - if [[ "$FAILED" == "true" ]] ; then + if [[ "${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}" == "true" ]]; then + echo "One or more jobs failed or were cancelled" exit 1 else - exit 0 + echo "All jobs completed successfully" fi diff --git a/.github/workflows/k8s_e2e.yml b/.github/workflows/k8s_e2e.yml index cd8085b3f2..7165bf63d5 100644 --- a/.github/workflows/k8s_e2e.yml +++ b/.github/workflows/k8s_e2e.yml @@ -40,8 +40,6 @@ concurrency: cancel-in-progress: true env: - AWS_ACCESS_KEY_ID: "dummy" - AWS_SECRET_ACCESS_KEY: "dummy" CONTAINER_TOOL: "docker" RUST_BACKTRACE: full TEST_LOG: vector=debug @@ -53,20 +51,17 @@ env: jobs: changes: # Only evaluate files changed on pull request trigger - if: github.event_name == 'pull_request' + if: ${{ github.event_name == 'pull_request' || github.event_name == 'merge_group' }} uses: ./.github/workflows/changes.yml - with: - base_ref: ${{ github.event.pull_request.base.ref }} - head_ref: ${{ github.event.pull_request.head.ref }} secrets: inherit build-x86_64-unknown-linux-gnu: name: Build - x86_64-unknown-linux-gnu - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 timeout-minutes: 45 needs: changes - # Run this job even if `changes` job is skipped (non- pull request trigger) - if: ${{ !failure() && !cancelled() && (github.event_name != 'pull_request' || needs.changes.outputs.k8s == 'true') }} + # Run this job even if `changes` job is skipped + if: ${{ !failure() && !cancelled() && needs.changes.outputs.website_only != 'true' && needs.changes.outputs.k8s != 'false' }} # cargo-deb requires a release build, but we don't need optimizations for tests env: CARGO_PROFILE_RELEASE_OPT_LEVEL: 0 @@ -75,7 +70,7 @@ jobs: steps: - name: (PR review) Set latest commit status as pending if: ${{ github.event_name == 'pull_request_review' }} - uses: myrotvorets/set-commit-status-action@v2.0.1 + uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 with: sha: ${{ github.event.review.commit_id }} token: ${{ secrets.GITHUB_TOKEN }} @@ -83,15 +78,15 @@ jobs: - name: (PR review) Checkout PR branch if: ${{ github.event_name == 'pull_request_review' }} - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ github.event.review.commit_id }} - name: Checkout branch if: ${{ github.event_name != 'pull_request_review' }} - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - uses: actions/cache@v4 + - uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 with: path: | ~/.cargo/registry @@ -99,18 +94,19 @@ jobs: key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - run: sudo -E bash scripts/ci-free-disk-space.sh - - run: sudo -E bash scripts/environment/bootstrap-ubuntu-20.04.sh - - run: bash scripts/environment/prepare.sh + - run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh + - run: bash scripts/environment/prepare.sh --modules=rustup,cross + - run: rustup target add x86_64-unknown-linux-gnu - run: echo "::add-matcher::.github/matchers/rust.json" - run: VECTOR_VERSION="$(cargo vdev version)" make package-deb-x86_64-unknown-linux-gnu - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: e2e-test-deb-package path: target/artifacts/* - name: (PR review) Set latest commit status as 'failure' - uses: myrotvorets/set-commit-status-action@v2.0.1 + uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 if: failure() && github.event_name == 'pull_request_review' with: sha: ${{ github.event.review.commit_id }} @@ -129,11 +125,11 @@ jobs: timeout-minutes: 5 needs: changes # Run this job even if `changes` job is skipped - if: ${{ !failure() && !cancelled() && (github.event_name != 'pull_request' || needs.changes.outputs.k8s == 'true') }} + if: ${{ !failure() && !cancelled() && needs.changes.outputs.website_only != 'true' && needs.changes.outputs.k8s != 'false' }} outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} steps: - - uses: actions/github-script@v7.0.1 + - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 id: set-matrix with: script: | @@ -180,7 +176,7 @@ jobs: test-e2e-kubernetes: name: K8s ${{ matrix.kubernetes_version.version }} / ${{ matrix.container_runtime }} (${{ matrix.kubernetes_version.role }}) - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 timeout-minutes: 45 needs: - build-x86_64-unknown-linux-gnu @@ -193,20 +189,20 @@ jobs: steps: - name: (PR review) Get PR branch if: ${{ github.event_name == 'pull_request_review' }} - uses: xt0rted/pull-request-comment-branch@v3 + uses: xt0rted/pull-request-comment-branch@e8b8daa837e8ea7331c0003c9c316a64c6d8b0b1 # v3.0.0 id: comment-branch - name: (PR review) Checkout PR branch if: ${{ github.event_name == 'pull_request_review' }} - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ steps.comment-branch.outputs.head_ref }} - name: Checkout branch if: ${{ github.event_name != 'pull_request_review' }} - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: e2e-test-deb-package path: target/artifacts @@ -218,14 +214,20 @@ jobs: MINIKUBE_VERSION: ${{ matrix.minikube_version }} CONTAINER_RUNTIME: ${{ matrix.container_runtime }} - - run: make test-e2e-kubernetes + # TODO: This job has been quite flakey. Need to investigate further and then remove the retries. + - name: Run tests + uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 env: USE_MINIKUBE_CACHE: "true" SKIP_PACKAGE_DEB: "true" CARGO_INCREMENTAL: 0 + with: + timeout_minutes: 45 + max_attempts: 3 + command: make test-e2e-kubernetes - name: (PR review) Set latest commit status as failure - uses: myrotvorets/set-commit-status-action@v2.0.1 + uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 if: failure() && github.event_name == 'pull_request_review' with: sha: ${{ github.event.review.commit_id }} @@ -243,25 +245,26 @@ jobs: - test-e2e-kubernetes if: always() env: - FAILED: ${{ contains(needs.*.result, 'failure') }} + FAILED: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }} steps: - name: (PR review) Get PR branch if: github.event_name == 'pull_request_review' && env.FAILED != 'true' - uses: xt0rted/pull-request-comment-branch@v3 + uses: xt0rted/pull-request-comment-branch@e8b8daa837e8ea7331c0003c9c316a64c6d8b0b1 # v3.0.0 id: comment-branch - name: (PR review) Submit PR result as success if: github.event_name == 'pull_request_review' && env.FAILED != 'true' - uses: myrotvorets/set-commit-status-action@v2.0.1 + uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 with: sha: ${{ github.event.review.commit_id }} token: ${{ secrets.GITHUB_TOKEN }} status: 'success' - - run: | - echo "failed=${{ env.FAILED }}" - if [[ "$FAILED" == "true" ]] ; then + - name: Check all jobs status + run: | + if [[ "${{ env.FAILED }}" == "true" ]]; then + echo "One or more jobs failed or were cancelled" exit 1 else - exit 0 + echo "All jobs completed successfully" fi diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 68baa1a388..44171cc374 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -4,13 +4,13 @@ on: jobs: label: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 timeout-minutes: 5 permissions: contents: read pull-requests: write steps: - - uses: actions/labeler@v5 + - uses: actions/labeler@8558fd74291d67161a8a78ce36a881fa63b766a9 # v5.0.0 with: repo-token: "${{ secrets.GITHUB_TOKEN }}" sync-labels: true diff --git a/.github/workflows/master_merge_queue.yml b/.github/workflows/master_merge_queue.yml index c09a317bd2..b6cf9486e1 100644 --- a/.github/workflows/master_merge_queue.yml +++ b/.github/workflows/master_merge_queue.yml @@ -28,8 +28,6 @@ concurrency: cancel-in-progress: true env: - AWS_ACCESS_KEY_ID: "dummy" - AWS_SECRET_ACCESS_KEY: "dummy" CONTAINER_TOOL: "docker" DD_ENV: "ci" DD_API_KEY: ${{ secrets.DD_API_KEY }} @@ -49,9 +47,6 @@ jobs: changes: if: ${{ github.event_name == 'merge_group' }} uses: ./.github/workflows/changes.yml - with: - base_ref: ${{ github.event.merge_group.base_ref }} - head_ref: ${{ github.event.merge_group.head_ref }} secrets: inherit test-cli: @@ -107,7 +102,7 @@ jobs: name: Master Merge Queue Suite # Always run this so that pull_request triggers are marked as success. if: always() - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 timeout-minutes: 5 needs: - changes @@ -119,14 +114,12 @@ jobs: - unit-mac - unit-windows - install-sh - env: - FAILED: ${{ contains(needs.*.result, 'failure') }} steps: - - name: exit + - name: Check all jobs status run: | - echo "failed=${{ env.FAILED }}" - if [[ "$FAILED" == "true" ]] ; then + if [[ "${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}" == "true" ]]; then + echo "One or more jobs failed or were cancelled" exit 1 else - exit 0 + echo "All jobs completed successfully" fi diff --git a/.github/workflows/misc.yml b/.github/workflows/misc.yml index 271893191a..c1cd250939 100644 --- a/.github/workflows/misc.yml +++ b/.github/workflows/misc.yml @@ -8,14 +8,14 @@ permissions: jobs: test-misc: - runs-on: ubuntu-20.04 - timeout-minutes: 60 + runs-on: ubuntu-24.04 + timeout-minutes: 90 env: CARGO_INCREMENTAL: 0 steps: - name: (PR review) Set latest commit status as pending if: ${{ github.event_name == 'pull_request_review' }} - uses: myrotvorets/set-commit-status-action@v2.0.1 + uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 with: sha: ${{ github.event.review.commit_id }} token: ${{ secrets.GITHUB_TOKEN }} @@ -24,15 +24,15 @@ jobs: - name: (PR review) Checkout review SHA if: ${{ github.event_name == 'pull_request_review' }} - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ github.event.review.commit_id }} - name: Checkout branch if: ${{ github.event_name != 'pull_request_review' }} - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - uses: actions/cache@v4 + - uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 name: Cache Cargo registry + index with: path: | @@ -45,15 +45,15 @@ jobs: ${{ runner.os }}-cargo- - run: sudo -E bash scripts/ci-free-disk-space.sh - - run: sudo -E bash scripts/environment/bootstrap-ubuntu-20.04.sh - - run: bash scripts/environment/prepare.sh + - run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh + - run: bash scripts/environment/prepare.sh --modules=rustup - run: echo "::add-matcher::.github/matchers/rust.json" - run: make test-behavior - run: make check-examples - run: make test-docs - name: (PR review) Set latest commit status as ${{ job.status }} - uses: myrotvorets/set-commit-status-action@v2.0.1 + uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 if: always() && github.event_name == 'pull_request_review' with: sha: ${{ github.event.review.commit_id }} diff --git a/.github/workflows/msrv.yml b/.github/workflows/msrv.yml index baa2b5c2ed..25c94ed8da 100644 --- a/.github/workflows/msrv.yml +++ b/.github/workflows/msrv.yml @@ -1,7 +1,13 @@ -name: Check minimum supported Rust version +name: Check minimum supported Rust version (MSRV) on: workflow_call: + workflow_dispatch: + inputs: + checkout_ref: + description: "Git ref (branch, tag or SHA) to check out" + required: false + type: string env: RUST_BACKTRACE: full @@ -13,10 +19,12 @@ env: jobs: check-msrv: - runs-on: ubuntu-20.04 - timeout-minutes: 20 + runs-on: ubuntu-24.04 + timeout-minutes: 45 steps: - - uses: actions/checkout@v4 - - run: sudo -E bash scripts/environment/bootstrap-ubuntu-20.04.sh - - run: cargo install cargo-msrv --version 0.15.1 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + ref: ${{ inputs.checkout_ref }} + - run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh + - run: bash scripts/environment/prepare.sh --modules=cargo-msrv - run: cargo msrv verify diff --git a/.github/workflows/preview_site_trigger.yml b/.github/workflows/preview_site_trigger.yml index 3ec4da8d03..811f5925ae 100644 --- a/.github/workflows/preview_site_trigger.yml +++ b/.github/workflows/preview_site_trigger.yml @@ -1,29 +1,52 @@ name: Call Build Preview - on: pull_request: types: [opened, reopened, synchronize] - jobs: approval_check: runs-on: ubuntu-24.04 timeout-minutes: 5 + # Only run for PRs with 'website' in the branch name if: ${{ contains(github.head_ref, 'website') }} steps: - - name: Echo approval + # Validate branch name + - name: Validate branch name and set output + id: validate run: | - echo "Workflow has been allowed to run for PR ${{ github.event.number }}. Setting artifacts and then continuing workflow runs" + BRANCH="${{ github.head_ref }}" + if [[ ! "$BRANCH" =~ ^[a-zA-Z0-9_-]+$ ]]; then + echo "valid=false" >> $GITHUB_OUTPUT + else + echo "valid=true" >> $GITHUB_OUTPUT + fi - # Save the PR number and branch name to an artifact for use in subsequent jobs - - name: Save PR number and Branch name - run: | - mkdir -p ./pr - echo "${{ github.event.number }}" > ./pr/number - echo "${{ github.head_ref }}" >> ./pr/branch + # Save PR information (only if branch is valid) + - name: Validate and save PR information + if: steps.validate.outputs.valid == 'true' + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + script: | + const fs = require('fs').promises; + const path = require('path'); + const crypto = require('crypto'); + const prNumber = context.payload.number; + const branchName = context.payload.pull_request.head.ref; + + await fs.mkdir('./pr', { recursive: true }); + await fs.writeFile('./pr/number', prNumber.toString()); + await fs.writeFile('./pr/branch', branchName); + + const numberHash = crypto.createHash('sha256').update(prNumber.toString()).digest('hex'); + const branchHash = crypto.createHash('sha256').update(branchName).digest('hex'); + await fs.writeFile('./pr/integrity', `${numberHash}:${branchHash}`); + + core.info(`Saved PR #${prNumber} and branch ${branchName}`); - # Upload the artifact + # Upload the artifact using latest version (only if branch is valid) - name: Upload PR information artifact - uses: actions/upload-artifact@v4 + if: steps.validate.outputs.valid == 'true' + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: pr path: pr/ + retention-days: 1 diff --git a/.github/workflows/protobuf.yml b/.github/workflows/protobuf.yml index 1899b78441..b261f18b43 100644 --- a/.github/workflows/protobuf.yml +++ b/.github/workflows/protobuf.yml @@ -1,5 +1,8 @@ name: Protobuf Compatibility +permissions: + contents: read + on: pull_request: paths: @@ -19,10 +22,12 @@ jobs: timeout-minutes: 5 steps: # Run `git checkout` - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 # Install the `buf` CLI - - uses: bufbuild/buf-setup-action@v1.49.0 + - uses: bufbuild/buf-setup-action@a47c93e0b1648d5651a065437926377d060baa99 # v1.50.0 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} # Perform breaking change detection against the `master` branch - - uses: bufbuild/buf-breaking-action@v1.1.4 + - uses: bufbuild/buf-breaking-action@c57b3d842a5c3f3b454756ef65305a50a587c5ba # v1.1.4 with: against: "https://github.com/vectordotdev/vector.git#branch=master" diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index ac5c07f437..aacf36bd0a 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -32,7 +32,7 @@ env: jobs: generate-publish-metadata: name: Generate Publish-related Metadata - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 timeout-minutes: 5 outputs: vector_version: ${{ steps.generate-publish-metadata.outputs.vector_version }} @@ -40,7 +40,7 @@ jobs: vector_release_channel: ${{ steps.generate-publish-metadata.outputs.vector_release_channel }} steps: - name: Checkout Vector - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ inputs.git_ref }} - name: Generate publish metadata @@ -58,17 +58,17 @@ jobs: CHANNEL: ${{ needs.generate-publish-metadata.outputs.vector_release_channel }} steps: - name: Checkout Vector - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ inputs.git_ref }} - name: Bootstrap runner environment (Ubuntu-specific) - run: sudo -E bash scripts/environment/bootstrap-ubuntu-20.04.sh + run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh - name: Bootstrap runner environment (generic) - run: bash scripts/environment/prepare.sh + run: bash scripts/environment/prepare.sh --modules=rustup,cross - name: Build Vector run: make package-x86_64-unknown-linux-musl-all - name: Stage package artifacts for publish - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-unknown-linux-musl path: target/artifacts/vector* @@ -76,25 +76,25 @@ jobs: build-x86_64-unknown-linux-gnu-packages: name: Build Vector for x86_64-unknown-linux-gnu (.tar.gz, DEB, RPM) runs-on: release-builder-linux - needs: generate-publish-metadata timeout-minutes: 60 + needs: generate-publish-metadata env: VECTOR_VERSION: ${{ needs.generate-publish-metadata.outputs.vector_version }} VECTOR_BUILD_DESC: ${{ needs.generate-publish-metadata.outputs.vector_build_desc }} CHANNEL: ${{ needs.generate-publish-metadata.outputs.vector_release_channel }} steps: - name: Checkout Vector - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ inputs.git_ref }} - name: Bootstrap runner environment (Ubuntu-specific) - run: sudo -E bash scripts/environment/bootstrap-ubuntu-20.04.sh + run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh - name: Bootstrap runner environment (generic) - run: bash scripts/environment/prepare.sh + run: bash scripts/environment/prepare.sh --modules=rustup,cross - name: Build Vector run: make package-x86_64-unknown-linux-gnu-all - name: Stage package artifacts for publish - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-unknown-linux-gnu path: target/artifacts/vector* @@ -110,19 +110,19 @@ jobs: CHANNEL: ${{ needs.generate-publish-metadata.outputs.vector_release_channel }} steps: - name: Checkout Vector - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ inputs.git_ref }} - name: Bootstrap runner environment (Ubuntu-specific) - run: sudo -E bash scripts/environment/bootstrap-ubuntu-20.04.sh + run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh - name: Bootstrap runner environment (generic) - run: bash scripts/environment/prepare.sh + run: bash scripts/environment/prepare.sh --modules=rustup,cross - name: Build Vector env: DOCKER_PRIVILEGED: "true" run: make package-aarch64-unknown-linux-musl-all - name: Stage package artifacts for publish - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: vector-${{ env.VECTOR_VERSION }}-aarch64-unknown-linux-musl path: target/artifacts/vector* @@ -138,19 +138,19 @@ jobs: CHANNEL: ${{ needs.generate-publish-metadata.outputs.vector_release_channel }} steps: - name: Checkout Vector - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ inputs.git_ref }} - name: Bootstrap runner environment (Ubuntu-specific) - run: sudo -E bash scripts/environment/bootstrap-ubuntu-20.04.sh + run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh - name: Bootstrap runner environment (generic) - run: bash scripts/environment/prepare.sh + run: bash scripts/environment/prepare.sh --modules=rustup,cross - name: Build Vector env: DOCKER_PRIVILEGED: "true" run: make package-aarch64-unknown-linux-gnu-all - name: Stage package artifacts for publish - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: vector-${{ env.VECTOR_VERSION }}-aarch64-unknown-linux-gnu path: target/artifacts/vector* @@ -166,19 +166,19 @@ jobs: CHANNEL: ${{ needs.generate-publish-metadata.outputs.vector_release_channel }} steps: - name: Checkout Vector - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ inputs.git_ref }} - name: Bootstrap runner environment (Ubuntu-specific) - run: sudo -E bash scripts/environment/bootstrap-ubuntu-20.04.sh + run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh - name: Bootstrap runner environment (generic) - run: bash scripts/environment/prepare.sh + run: bash scripts/environment/prepare.sh --modules=rustup,cross - name: Build Vector env: DOCKER_PRIVILEGED: "true" run: make package-armv7-unknown-linux-gnueabihf-all - name: Stage package artifacts for publish - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: vector-${{ env.VECTOR_VERSION }}-armv7-unknown-linux-gnueabihf path: target/artifacts/vector* @@ -194,19 +194,19 @@ jobs: CHANNEL: ${{ needs.generate-publish-metadata.outputs.vector_release_channel }} steps: - name: Checkout Vector - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ inputs.git_ref }} - name: Bootstrap runner environment (Ubuntu-specific) - run: sudo -E bash scripts/environment/bootstrap-ubuntu-20.04.sh + run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh - name: Bootstrap runner environment (generic) - run: bash scripts/environment/prepare.sh + run: bash scripts/environment/prepare.sh --modules=rustup,cross - name: Build Vector env: DOCKER_PRIVILEGED: "true" run: make package-armv7-unknown-linux-musleabihf - name: Stage package artifacts for publish - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: vector-${{ env.VECTOR_VERSION }}-armv7-unknown-linux-musleabihf path: target/artifacts/vector* @@ -222,19 +222,19 @@ jobs: CHANNEL: ${{ needs.generate-publish-metadata.outputs.vector_release_channel }} steps: - name: Checkout Vector - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ inputs.git_ref }} - name: Bootstrap runner environment (Ubuntu-specific) - run: sudo -E bash scripts/environment/bootstrap-ubuntu-20.04.sh + run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh - name: Bootstrap runner environment (generic) - run: bash scripts/environment/prepare.sh + run: bash scripts/environment/prepare.sh --modules=rustup,cross - name: Build Vector env: DOCKER_PRIVILEGED: "true" run: make package-arm-unknown-linux-gnueabi-all - name: Stage package artifacts for publish - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: vector-${{ env.VECTOR_VERSION }}-arm-unknown-linux-gnueabi path: target/artifacts/vector* @@ -242,6 +242,7 @@ jobs: build-arm-unknown-linux-musleabi-packages: name: Build Vector for arm-unknown-linux-musleabi (.tar.gz) runs-on: release-builder-linux + timeout-minutes: 60 needs: generate-publish-metadata env: VECTOR_VERSION: ${{ needs.generate-publish-metadata.outputs.vector_version }} @@ -249,19 +250,19 @@ jobs: CHANNEL: ${{ needs.generate-publish-metadata.outputs.vector_release_channel }} steps: - name: Checkout Vector - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ inputs.git_ref }} - name: Bootstrap runner environment (Ubuntu-specific) - run: sudo -E bash scripts/environment/bootstrap-ubuntu-20.04.sh + run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh - name: Bootstrap runner environment (generic) - run: bash scripts/environment/prepare.sh + run: bash scripts/environment/prepare.sh --modules=rustup,cross - name: Build Vector env: DOCKER_PRIVILEGED: "true" run: make package-arm-unknown-linux-musleabi - name: Stage package artifacts for publish - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: vector-${{ env.VECTOR_VERSION }}-arm-unknown-linux-musleabi path: target/artifacts/vector* @@ -279,10 +280,11 @@ jobs: matrix: include: # Refer to https://docs.github.com/en/actions/using-github-hosted-runners/using-larger-runners/about-larger-runners#about-macos-larger-runners. + # and to https://github.com/actions/runner-images - architecture: x86_64 - runner: macos-latest-large + runner: macos-14-large - architecture: arm64 - runner: macos-latest-xlarge + runner: macos-14-xlarge steps: - name: Verify Runner Architecture run: | @@ -293,11 +295,13 @@ jobs: exit 1 fi - name: Checkout Vector - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ inputs.git_ref }} - name: Bootstrap runner environment (macOS-specific) - run: bash scripts/environment/bootstrap-macos.sh + run: | + bash scripts/environment/bootstrap-macos.sh + bash scripts/environment/prepare.sh --modules=rustup - name: Build Vector env: TARGET: "${{ matrix.architecture }}-apple-darwin" @@ -306,7 +310,7 @@ jobs: export PATH="$HOME/.cargo/bin:$PATH" make package - name: Stage package artifacts for publish - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: vector-${{ env.VECTOR_VERSION }}-${{ matrix.architecture }}-apple-darwin path: target/artifacts/vector* @@ -314,7 +318,7 @@ jobs: build-x86_64-pc-windows-msvc-packages: name: Build Vector for x86_64-pc-windows-msvc (.zip) - runs-on: release-builder-windows-2022 + runs-on: windows-2025-8core timeout-minutes: 90 needs: generate-publish-metadata env: @@ -325,11 +329,11 @@ jobs: RELEASE_BUILDER: "true" steps: - name: Checkout Vector - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ inputs.git_ref }} - name: Bootstrap runner environment (Windows-specific) - run: .\scripts\environment\bootstrap-windows-2022.ps1 + run: .\scripts\environment\bootstrap-windows-2025.ps1 - name: Install Wix shell: bash run: | @@ -354,14 +358,14 @@ jobs: export PATH="/c/wix:$PATH" ./scripts/package-msi.sh - name: Stage package artifacts for publish - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-pc-windows-msvc path: target/artifacts/vector* deb-verify: name: Verify DEB Packages - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 timeout-minutes: 5 needs: - generate-publish-metadata @@ -375,7 +379,6 @@ jobs: - ubuntu:20.04 - ubuntu:22.04 - ubuntu:24.04 - - debian:10 - debian:11 - debian:12 container: @@ -395,11 +398,11 @@ jobs: - name: Fix Git safe directories issue when in containers (actions/checkout#760) run: git config --global --add safe.directory /__w/vector/vector - name: Checkout Vector - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ inputs.git_ref }} - name: Download staged package artifacts (x86_64-unknown-linux-gnu) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-unknown-linux-gnu path: target/artifacts @@ -409,7 +412,7 @@ jobs: rpm-verify: name: Verify RPM Packages - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 timeout-minutes: 5 needs: - generate-publish-metadata @@ -446,11 +449,11 @@ jobs: - name: Fix Git safe directories issue when in containers (actions/checkout#760) run: git config --global --add safe.directory /__w/vector/vector - name: Checkout Vector - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ inputs.git_ref }} - name: Download staged package artifacts (x86_64-unknown-linux-gnu) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-unknown-linux-gnu path: target/artifacts @@ -470,17 +473,18 @@ jobs: strategy: matrix: include: + # Refer to https://github.com/actions/runner-images - target: x86_64-apple-darwin - runner: macos-latest-large + runner: macos-14-large - target: arm64-apple-darwin - runner: macos-latest-xlarge + runner: macos-14 steps: - name: Checkout Vector - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ inputs.git_ref }} - name: Download staged package artifacts (${{ matrix.target }}) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-${{ matrix.target }} path: target/artifacts @@ -491,7 +495,7 @@ jobs: publish-docker: name: Publish to Docker - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 timeout-minutes: 15 needs: - generate-publish-metadata @@ -510,68 +514,75 @@ jobs: CHANNEL: ${{ needs.generate-publish-metadata.outputs.vector_release_channel }} steps: - name: Checkout Vector - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ inputs.git_ref }} - name: Login to DockerHub - uses: docker/login-action@v3 + uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0 with: username: ${{ secrets.CI_DOCKER_USERNAME }} password: ${{ secrets.CI_DOCKER_PASSWORD }} + - name: Login to GitHub Container Registry + uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Set up QEMU - uses: docker/setup-qemu-action@v3.3.0 + uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0 with: platforms: all - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@v3.8.0 + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 with: version: latest install: true - name: Download staged package artifacts (aarch64-unknown-linux-gnu) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-aarch64-unknown-linux-gnu path: target/artifacts - name: Download staged package artifacts (aarch64-unknown-linux-musl) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-aarch64-unknown-linux-musl path: target/artifacts - name: Download staged package artifacts (x86_64-unknown-linux-gnu) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-unknown-linux-gnu path: target/artifacts - name: Download staged package artifacts (x86_64-unknown-linux-musl) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-unknown-linux-musl path: target/artifacts - name: Download staged package artifacts (armv7-unknown-linux-gnueabihf) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-armv7-unknown-linux-gnueabihf path: target/artifacts - name: Download staged package artifacts (armv7-unknown-linux-musleabihf) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-armv7-unknown-linux-musleabihf path: target/artifacts - name: Download staged package artifacts (arm-unknown-linux-gnueabi) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-arm-unknown-linux-gnueabi path: target/artifacts - name: Download staged package artifacts (arm-unknown-linux-musleabi) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-arm-unknown-linux-musleabi path: target/artifacts - name: Build and publish Docker images env: PLATFORM: "linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v6" - uses: nick-fields/retry@v3 + REPOS: "timberio/vector,ghcr.io/vectordotdev/vector" + uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 with: timeout_minutes: 15 max_attempts: 3 @@ -580,7 +591,7 @@ jobs: publish-s3: name: Publish to S3 - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 timeout-minutes: 10 needs: - generate-publish-metadata @@ -602,61 +613,61 @@ jobs: CHANNEL: ${{ needs.generate-publish-metadata.outputs.vector_release_channel }} steps: - name: Checkout Vector - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ inputs.git_ref }} - name: Download staged package artifacts (aarch64-unknown-linux-gnu) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-aarch64-unknown-linux-gnu path: target/artifacts - name: Download staged package artifacts (aarch64-unknown-linux-musl) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-aarch64-unknown-linux-musl path: target/artifacts - name: Download staged package artifacts (x86_64-unknown-linux-gnu) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-unknown-linux-gnu path: target/artifacts - name: Download staged package artifacts (x86_64-unknown-linux-musl) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-unknown-linux-musl path: target/artifacts - name: Download staged package artifacts (x86_64-apple-darwin) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-apple-darwin path: target/artifacts - name: Download staged package artifacts (arm64-apple-darwin) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-arm64-apple-darwin path: target/artifacts - name: Download staged package artifacts (x86_64-pc-windows-msvc) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-pc-windows-msvc path: target/artifacts - name: Download staged package artifacts (armv7-unknown-linux-gnueabihf) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-armv7-unknown-linux-gnueabihf path: target/artifacts - name: Download staged package artifacts (armv7-unknown-linux-musleabihf) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-armv7-unknown-linux-musleabihf path: target/artifacts - name: Download staged package artifacts (arm-unknown-linux-gnueabi) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-arm-unknown-linux-gnueabi path: target/artifacts - name: Download staged package artifacts (arm-unknown-linux-musleabi) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-arm-unknown-linux-musleabi path: target/artifacts @@ -667,10 +678,10 @@ jobs: run: make release-s3 publish-github: - name: Publish to GitHub + name: Publish release to GitHub # We only publish to GitHub for versioned releases, not nightlies. if: inputs.channel == 'release' - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 timeout-minutes: 10 needs: - generate-publish-metadata @@ -692,66 +703,66 @@ jobs: VECTOR_VERSION: ${{ needs.generate-publish-metadata.outputs.vector_version }} steps: - name: Checkout Vector - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ inputs.git_ref }} - name: Download staged package artifacts (aarch64-unknown-linux-gnu) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-aarch64-unknown-linux-gnu path: target/artifacts - name: Download staged package artifacts (aarch64-unknown-linux-musl) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-aarch64-unknown-linux-musl path: target/artifacts - name: Download staged package artifacts (x86_64-unknown-linux-gnu) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-unknown-linux-gnu path: target/artifacts - name: Download staged package artifacts (x86_64-unknown-linux-musl) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-unknown-linux-musl path: target/artifacts - name: Download staged package artifacts (x86_64-apple-darwin) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-apple-darwin path: target/artifacts - name: Download staged package artifacts (arm64-apple-darwin) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-arm64-apple-darwin path: target/artifacts - name: Download staged package artifacts (x86_64-pc-windows-msvc) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-pc-windows-msvc path: target/artifacts - name: Download staged package artifacts (armv7-unknown-linux-gnueabihf) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-armv7-unknown-linux-gnueabihf path: target/artifacts - name: Download staged package artifacts (armv7-unknown-linux-musleabihf) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-armv7-unknown-linux-musleabihf path: target/artifacts - name: Download artifact checksums - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-SHA256SUMS path: target/artifacts - name: Download staged package artifacts (arm-unknown-linux-gnueabi) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-arm-unknown-linux-gnueabi path: target/artifacts - name: Download staged package artifacts (arm-unknown-linux-musleabi) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-arm-unknown-linux-musleabi path: target/artifacts @@ -764,7 +775,7 @@ jobs: name: Publish to Homebrew # We only publish to Homebrew for versioned releases, not nightlies. if: inputs.channel == 'release' - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 timeout-minutes: 10 needs: - generate-publish-metadata @@ -773,17 +784,17 @@ jobs: VECTOR_VERSION: ${{ needs.generate-publish-metadata.outputs.vector_version }} steps: - name: Checkout Vector - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ inputs.git_ref }} - name: Publish update to Homebrew tap env: - GITHUB_TOKEN: ${{ secrets.GH_PACKAGE_PUBLISHER_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: make release-homebrew generate-sha256sum: name: Generate SHA256 checksums - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 timeout-minutes: 5 needs: - generate-publish-metadata @@ -801,68 +812,68 @@ jobs: VECTOR_VERSION: ${{ needs.generate-publish-metadata.outputs.vector_version }} steps: - name: Checkout Vector - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ inputs.git_ref }} - name: Download staged package artifacts (aarch64-unknown-linux-gnu) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-aarch64-unknown-linux-gnu path: target/artifacts - name: Download staged package artifacts (aarch64-unknown-linux-musl) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-aarch64-unknown-linux-musl path: target/artifacts - name: Download staged package artifacts (x86_64-unknown-linux-gnu) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-unknown-linux-gnu path: target/artifacts - name: Download staged package artifacts (x86_64-unknown-linux-musl) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-unknown-linux-musl path: target/artifacts - name: Download staged package artifacts (x86_64-apple-darwin) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-apple-darwin path: target/artifacts - name: Download staged package artifacts (arm64-apple-darwin) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-arm64-apple-darwin path: target/artifacts - name: Download staged package artifacts (x86_64-pc-windows-msvc) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-pc-windows-msvc path: target/artifacts - name: Download staged package artifacts (armv7-unknown-linux-gnueabihf) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-armv7-unknown-linux-gnueabihf path: target/artifacts - name: Download staged package artifacts (armv7-unknown-linux-musleabihf) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-armv7-unknown-linux-musleabihf path: target/artifacts - name: Download staged package artifacts (arm-unknown-linux-gnueabi) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-arm-unknown-linux-gnueabi path: target/artifacts - name: Download staged package artifacts (arm-unknown-linux-musleabi) - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-${{ env.VECTOR_VERSION }}-arm-unknown-linux-musleabi path: target/artifacts - name: Generate SHA256 checksums for artifacts run: make sha256sum - name: Stage checksum for publish - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: vector-${{ env.VECTOR_VERSION }}-SHA256SUMS path: target/artifacts/vector-${{ env.VECTOR_VERSION }}-SHA256SUMS diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 1e13adb872..538d5cb824 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -51,7 +51,7 @@ jobs: smp-version: ${{ steps.experimental-meta.outputs.SMP_CRATE_VERSION }} steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: fetch-depth: 0 # need to pull repository history to find merge bases @@ -105,7 +105,7 @@ jobs: - name: Set SMP version id: experimental-meta run: | - export SMP_CRATE_VERSION="0.19.3" + export SMP_CRATE_VERSION="0.24.1" echo "smp crate version: ${SMP_CRATE_VERSION}" echo "SMP_CRATE_VERSION=${SMP_CRATE_VERSION}" >> $GITHUB_OUTPUT @@ -117,11 +117,11 @@ jobs: outputs: source_changed: ${{ steps.filter.outputs.SOURCE_CHANGED }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Collect file changes id: changes - uses: dorny/paths-filter@v3 + uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 with: base: ${{ needs.resolve-inputs.outputs.baseline-sha }} ref: ${{ needs.resolve-inputs.outputs.comparison-sha }} @@ -184,27 +184,27 @@ jobs: build-baseline: name: Build baseline Vector container - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 timeout-minutes: 30 needs: - should-run-gate - resolve-inputs steps: - - uses: colpal/actions-clean@v1 + - uses: colpal/actions-clean@36e6ca1abd35efe61cb60f912bd7837f67887c8a # v1.1.1 - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ needs.resolve-inputs.outputs.baseline-sha }} path: baseline-vector - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@v3.8.0 + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 - name: Build 'vector' target image - uses: docker/build-push-action@v6.11.0 + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 with: context: baseline-vector/ cache-from: type=gha @@ -216,34 +216,34 @@ jobs: vector:${{ needs.resolve-inputs.outputs.baseline-tag }} - name: Upload image as artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: baseline-image path: "${{ runner.temp }}/baseline-image.tar" build-comparison: name: Build comparison Vector container - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 timeout-minutes: 30 needs: - should-run-gate - resolve-inputs steps: - - uses: colpal/actions-clean@v1 + - uses: colpal/actions-clean@36e6ca1abd35efe61cb60f912bd7837f67887c8a # v1.1.1 - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ needs.resolve-inputs.outputs.comparison-sha }} path: comparison-vector - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@v3.8.0 + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 - name: Build 'vector' target image - uses: docker/build-push-action@v6.11.0 + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 with: context: comparison-vector/ cache-from: type=gha @@ -255,7 +255,7 @@ jobs: vector:${{ needs.resolve-inputs.outputs.comparison-tag }} - name: Upload image as artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: comparison-image path: "${{ runner.temp }}/comparison-image.tar" @@ -269,7 +269,7 @@ jobs: - resolve-inputs steps: - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4.0.2 + uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1 with: aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} @@ -277,7 +277,7 @@ jobs: - name: Download SMP binary run: | - aws s3 cp s3://smp-cli-releases/v${{ needs.resolve-inputs.outputs.smp-version }}/x86_64-unknown-linux-gnu/smp ${{ runner.temp }}/bin/smp + aws s3 cp s3://smp-cli-releases/v${{ needs.resolve-inputs.outputs.smp-version }}/x86_64-unknown-linux-musl/smp ${{ runner.temp }}/bin/smp ## ## SUBMIT @@ -294,7 +294,7 @@ jobs: - build-baseline steps: - name: 'Download baseline image' - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: baseline-image @@ -303,7 +303,7 @@ jobs: docker load --input baseline-image.tar - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4.0.2 + uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1 with: aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} @@ -311,10 +311,10 @@ jobs: - name: Login to Amazon ECR id: login-ecr - uses: aws-actions/amazon-ecr-login@v2 + uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2.0.1 - name: Docker Login to ECR - uses: docker/login-action@v3 + uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0 with: registry: ${{ steps.login-ecr.outputs.registry }} @@ -334,7 +334,7 @@ jobs: - build-comparison steps: - name: 'Download comparison image' - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: comparison-image @@ -343,7 +343,7 @@ jobs: docker load --input comparison-image.tar - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4.0.2 + uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1 with: aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} @@ -351,10 +351,10 @@ jobs: - name: Login to Amazon ECR id: login-ecr - uses: aws-actions/amazon-ecr-login@v2 + uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2.0.1 - name: Docker Login to ECR - uses: docker/login-action@v3 + uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0 with: registry: ${{ steps.login-ecr.outputs.registry }} @@ -373,12 +373,12 @@ jobs: - upload-baseline-image-to-ecr - upload-comparison-image-to-ecr steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ needs.resolve-inputs.outputs.comparison-sha }} - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4.0.2 + uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1 with: aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} @@ -386,11 +386,11 @@ jobs: - name: Login to Amazon ECR id: login-ecr - uses: aws-actions/amazon-ecr-login@v2 + uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2.0.1 - name: Download SMP binary run: | - aws s3 cp s3://smp-cli-releases/v${{ needs.resolve-inputs.outputs.smp-version }}/x86_64-unknown-linux-gnu/smp ${{ runner.temp }}/bin/smp + aws s3 cp s3://smp-cli-releases/v${{ needs.resolve-inputs.outputs.smp-version }}/x86_64-unknown-linux-musl/smp ${{ runner.temp }}/bin/smp - name: Submit job env: @@ -408,7 +408,7 @@ jobs: --submission-metadata ${{ runner.temp }}/submission-metadata \ --replicas ${{ env.SMP_REPLICAS }} - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: vector-submission-metadata path: ${{ runner.temp }}/submission-metadata @@ -448,10 +448,10 @@ jobs: - should-run-gate - resolve-inputs steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4.0.2 + uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1 with: aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} @@ -459,10 +459,10 @@ jobs: - name: Download SMP binary run: | - aws s3 cp s3://smp-cli-releases/v${{ needs.resolve-inputs.outputs.smp-version }}/x86_64-unknown-linux-gnu/smp ${{ runner.temp }}/bin/smp + aws s3 cp s3://smp-cli-releases/v${{ needs.resolve-inputs.outputs.smp-version }}/x86_64-unknown-linux-musl/smp ${{ runner.temp }}/bin/smp - name: Download submission metadata - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-submission-metadata path: ${{ runner.temp }}/ @@ -485,12 +485,12 @@ jobs: - submit-job - resolve-inputs steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ needs.resolve-inputs.outputs.comparison-sha }} - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4.0.2 + uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1 with: aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} @@ -498,10 +498,10 @@ jobs: - name: Download SMP binary run: | - aws s3 cp s3://smp-cli-releases/v${{ needs.resolve-inputs.outputs.smp-version }}/x86_64-unknown-linux-gnu/smp ${{ runner.temp }}/bin/smp + aws s3 cp s3://smp-cli-releases/v${{ needs.resolve-inputs.outputs.smp-version }}/x86_64-unknown-linux-musl/smp ${{ runner.temp }}/bin/smp - name: Download submission metadata - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: vector-submission-metadata path: ${{ runner.temp }}/ @@ -518,12 +518,12 @@ jobs: - name: Read regression report id: read-analysis - uses: juliangruber/read-file-action@v1 + uses: juliangruber/read-file-action@b549046febe0fe86f8cb4f93c24e284433f9ab58 # v1.1.7 with: path: ${{ runner.temp }}/outputs/report.md - name: Upload regression report to artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: capture-artifacts path: ${{ runner.temp }}/outputs/* @@ -544,14 +544,28 @@ jobs: - submit-job - detect-regression - analyze-experiment - env: - FAILED: ${{ contains(needs.*.result, 'failure') }} steps: - - name: exit + - name: Download capture-artifacts + continue-on-error: true + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: capture-artifacts + + - name: Display Markdown Summary + continue-on-error: true + run: | + REPORT_MD=report.md + if [ -f ${REPORT_MD} ]; then + cat ${REPORT_MD} >> $GITHUB_STEP_SUMMARY + else + echo "Did not find ${REPORT_MD} file." + fi + + - name: Check all jobs status run: | - echo "failed=${{ env.FAILED }}" - if [[ "$FAILED" == "true" ]] ; then + if [[ "${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}" == "true" ]]; then + echo "One or more jobs failed or were cancelled" exit 1 else - exit 0 + echo "All jobs completed successfully" fi diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 472ed4e9ae..0f7aebbd05 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -32,12 +32,12 @@ jobs: steps: - name: "Checkout code" - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: persist-credentials: false - name: "Run analysis" - uses: ossf/scorecard-action@v2.4.0 + uses: ossf/scorecard-action@05b42c624433fc40578a4040d5cf5e36ddca8cde # v2.4.2 with: results_file: results.sarif results_format: sarif @@ -59,7 +59,7 @@ jobs: # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF # format to the repository Actions tab. - name: "Upload artifact" - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: SARIF file path: results.sarif @@ -68,6 +68,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard (optional). # Commenting out will disable upload of results to your repo's Code Scanning dashboard - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@v3 + uses: github/codeql-action/upload-sarif@2d92b76c45b91eb80fc44c74ce3fce0ee94e8f9d # v3.30.0 with: sarif_file: results.sarif diff --git a/.github/workflows/semantic.yml b/.github/workflows/semantic.yml index 8c15013b40..3fa5e58e6e 100644 --- a/.github/workflows/semantic.yml +++ b/.github/workflows/semantic.yml @@ -4,18 +4,17 @@ name: "PR Title Semantic Check" on: + pull_request_target: + types: [opened, edited, synchronize] pull_request: - types: - - opened - - edited - - synchronize + types: [opened, edited, synchronize] jobs: main: name: Check Semantic PR runs-on: ubuntu-24.04 steps: - - uses: amannn/action-semantic-pull-request@v5 + - uses: amannn/action-semantic-pull-request@0723387faaf9b38adef4775cd42cfd5155ed6017 # v5.5.3 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: @@ -29,12 +28,9 @@ jobs: revert scopes: | - new source - new transform - new sink - ARC administration api + ARC architecture auth buffers @@ -43,14 +39,17 @@ jobs: codecs compression config + config provider core data model delivery deployment + deprecations deps dev durability enriching + enrichment tables enterprise exceptions external @@ -62,6 +61,9 @@ jobs: logs metrics networking + new sink + new source + new transform observability parsing performance @@ -86,7 +88,9 @@ jobs: traces transforms unit tests + vdev vrl + amazon-linux platform apt platform arm platform @@ -111,7 +115,6 @@ jobs: x86_64 platform yum platform - service providers aws service azure service confluent service @@ -127,8 +130,10 @@ jobs: new relic service papertrail service sematext service + service providers splunk service yandex service + apache_metrics source aws_ecs_metrics source aws_kinesis_firehose source @@ -145,7 +150,7 @@ jobs: gcp_pubsub source heroku_logs source host_metrics source - http source + http_client source http_scrape source internal_logs source internal_metrics source @@ -154,8 +159,10 @@ jobs: kubernetes_logs source logstash source mongodb_metrics source + nats source new source nginx_metrics source + okta source opentelemetry source postgresql_metrics source prometheus_remote_write source @@ -168,8 +175,11 @@ jobs: stdin source syslog source vector source + websocket source + aws_ec2_metadata transform dedupe transform + exclusive_route transform filter transform geoip transform log_to_metric transform @@ -180,10 +190,11 @@ jobs: reduce transform remap transform route transform - exclusive_route transform sample transform tag_cardinality_limit transform throttle transform + + amqp sink apex sink aws_cloudwatch_logs sink aws_cloudwatch_metrics sink @@ -191,12 +202,14 @@ jobs: aws_kinesis_streams sink aws_s3 sink aws_sqs sink + axiom sink azure_blob sink azure_monitor_logs sink blackhole sink clickhouse sink console sink datadog_archives sink + datadog_common sink datadog_events sink datadog_logs sink datadog_metrics sink @@ -207,8 +220,8 @@ jobs: gcp_pubsub sink gcp_stackdriver_logs sink gcp_stackdriver_metrics sink - greptimedb_metrics sink greptimedb_logs sink + greptimedb_metrics sink honeycomb sink http sink humio_logs sink @@ -224,6 +237,7 @@ jobs: new_relic_logs sink opentelemetry sink papertrail sink + postgres sink prometheus_exporter sink prometheus_remote_write sink pulsar sink @@ -235,6 +249,8 @@ jobs: statsd sink vector sink websocket sink + websocket_server sink + blog website css website guides website @@ -243,3 +259,6 @@ jobs: search website template website website + website deps + + opentelemetry lib diff --git a/.github/workflows/spelling.yml b/.github/workflows/spelling.yml index 5976f1a9fe..810fb03867 100644 --- a/.github/workflows/spelling.yml +++ b/.github/workflows/spelling.yml @@ -87,7 +87,7 @@ jobs: steps: - name: check-spelling id: spelling - uses: check-spelling/check-spelling@v0.0.24 + uses: check-spelling/check-spelling@c635c2f3f714eec2fcf27b643a1919b9a811ef2e # v0.0.25 with: suppress_push_for_open_pull_request: ${{ github.actor != 'dependabot[bot]' && 1 }} checkout: true diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 418c272c63..d08f370c8d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -11,8 +11,6 @@ concurrency: cancel-in-progress: true env: - AWS_ACCESS_KEY_ID: "dummy" - AWS_SECRET_ACCESS_KEY: "dummy" CONTAINER_TOOL: "docker" DD_ENV: "ci" DD_API_KEY: ${{ secrets.DD_API_KEY }} @@ -29,53 +27,43 @@ jobs: changes: uses: ./.github/workflows/changes.yml secrets: inherit - with: - base_ref: ${{ github.event.merge_group.base_ref || github.event.pull_request.base.ref }} - head_ref: ${{ github.event.merge_group.head_ref || github.event.pull_request.head.ref }} - - checks: - name: Checks - runs-on: ubuntu-20.04-8core - timeout-minutes: 60 + + check-fmt: + name: Check code format + runs-on: ubuntu-24.04 needs: changes - env: - CARGO_INCREMENTAL: 0 steps: - - uses: actions/checkout@v4 - with: - # check-version needs tags - fetch-depth: 0 # fetch everything - - - uses: actions/cache@v4 - name: Cache Cargo registry + index - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-cargo- - - - run: sudo -E bash scripts/environment/bootstrap-ubuntu-20.04.sh - - - uses: ruby/setup-ruby@v1 - - - run: bash scripts/environment/prepare.sh - + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Enable Rust matcher run: echo "::add-matcher::.github/matchers/rust.json" + - run: make check-fmt - - name: Check code format - run: make check-fmt - - - name: Check clippy - if: needs.changes.outputs.source == 'true' - run: make check-clippy + check-clippy: + name: Check clippy + runs-on: ubuntu-24.04-8core + if: needs.changes.outputs.source == 'true' + needs: changes + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - name: Enable Rust matcher + run: echo "::add-matcher::.github/matchers/rust.json" + - run: sudo bash ./scripts/environment/install-protoc.sh + - run: sudo apt-get update && sudo apt-get install -y libsasl2-dev + - run: make check-clippy + + test: + name: Unit and Component Validation tests - x86_64-unknown-linux-gnu + runs-on: ubuntu-24.04-8core + if: needs.changes.outputs.source == 'true' + needs: changes + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - name: Enable Rust matcher + run: echo "::add-matcher::.github/matchers/rust.json" + - run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh + - run: bash ./scripts/environment/prepare.sh --modules=cargo-nextest,datadog-ci - - name: Unit - x86_64-unknown-linux-gnu - if: needs.changes.outputs.source == 'true' + - name: Unit Test run: make test env: CARGO_BUILD_JOBS: 5 @@ -88,60 +76,141 @@ jobs: run: scripts/upload-test-results.sh if: always() - - name: Check version - run: make check-version - - - name: Check scripts - run: make check-scripts - - - name: Check events - if: needs.changes.outputs.source == 'true' - run: make check-events - - - name: Check that the 3rd-party license file is up to date - if: needs.changes.outputs.dependencies == 'true' - run: make check-licenses - - - name: Check Cue docs - if: needs.changes.outputs.cue == 'true' - run: make check-docs - - - name: Check Markdown - if: needs.changes.outputs.markdown == 'true' - run: make check-markdown + check-version: + name: Check version + runs-on: ubuntu-24.04 + needs: changes + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + fetch-depth: 0 + - run: make check-version - - name: Check Component Docs - if: needs.changes.outputs.source == 'true' || needs.changes.outputs.component_docs == 'true' - run: make check-component-docs + check-scripts: + name: Check scripts + runs-on: ubuntu-24.04 + needs: changes + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - run: make check-scripts - - name: Check Rust Docs - if: needs.changes.outputs.source == 'true' - run: cd rust-doc && make docs + check-events: + name: Check events + runs-on: ubuntu-24.04-8core + if: needs.changes.outputs.source == 'true' + needs: changes + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - name: Enable Rust matcher + run: echo "::add-matcher::.github/matchers/rust.json" + - run: make check-events - - name: VRL - Linux - if: needs.changes.outputs.source == 'true' || needs.changes.outputs.cue == 'true' - run: cargo vdev test-vrl + check-licenses: + name: Check that the 3rd-party license file is up to date + runs-on: ubuntu-24.04 + if: needs.changes.outputs.dependencies == 'true' + needs: changes + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - run: bash ./scripts/environment/prepare.sh --modules=dd-rust-license-tool + - run: make check-licenses + + check-docs: + name: Check Cue docs + runs-on: ubuntu-24.04 + if: needs.changes.outputs.cue == 'true' + needs: changes + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh + - run: make check-docs + + check-markdown: + name: Check Markdown + runs-on: ubuntu-24.04 + if: needs.changes.outputs.markdown == 'true' + needs: changes + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - run: bash ./scripts/environment/prepare.sh --modules=markdownlint + - run: make check-markdown + + check-component-docs: + name: Check Component Docs + runs-on: ubuntu-24.04-8core + if: needs.changes.outputs.source == 'true' || needs.changes.outputs.component_docs == 'true' + needs: changes + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - run: sudo bash ./scripts/environment/install-protoc.sh + - run: sudo apt-get update && sudo apt-get install -y libsasl2-dev + - run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh + - run: make check-component-docs + + check-rust-docs: + name: Check Rust Docs + runs-on: ubuntu-24.04-8core + if: needs.changes.outputs.source == 'true' + needs: changes + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - name: Enable Rust matcher + run: echo "::add-matcher::.github/matchers/rust.json" + - run: cd rust-doc && make docs - - name: Build VRL Playground - if: needs.changes.outputs.source == 'true' || needs.changes.outputs.dependencies == 'true' - run: | + test-vrl: + name: VRL - Linux + runs-on: ubuntu-24.04-8core + if: needs.changes.outputs.source == 'true' || needs.changes.outputs.cue == 'true' + needs: changes + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - name: Enable Rust matcher + run: echo "::add-matcher::.github/matchers/rust.json" + - run: sudo bash ./scripts/environment/install-protoc.sh + - run: bash ./scripts/environment/prepare.sh --modules=wasm-pack + - run: make test-vrl + + build-vrl-playground: + name: Build VRL Playground + runs-on: ubuntu-24.04-8core + if: needs.changes.outputs.source == 'true' || needs.changes.outputs.dependencies == 'true' + needs: changes + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - name: Enable Rust matcher + run: echo "::add-matcher::.github/matchers/rust.json" + - run: bash ./scripts/environment/prepare.sh --modules=wasm-pack + - run: | cd lib/vector-vrl/web-playground/ + rustup target add wasm32-unknown-unknown wasm-pack build --target web --out-dir public/pkg - # This is a required status check, so it always needs to run if prior jobs failed, in order to mark the status correctly. all-checks: name: Test Suite - runs-on: ubuntu-20.04 - timeout-minutes: 5 + runs-on: ubuntu-24.04 if: always() - needs: [changes, checks] - env: - FAILED: ${{ contains(needs.*.result, 'failure') }} + needs: + - changes + - check-fmt + - check-clippy + - test + - check-version + - check-scripts + - check-events + - check-licenses + - check-docs + - check-markdown + - check-component-docs + - check-rust-docs + - test-vrl + - build-vrl-playground steps: - - run: | - echo "failed=${{ env.FAILED }}" - if [[ "$FAILED" == "true" ]] ; then + - name: Check all jobs status + run: | + if [[ "${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}" == "true" ]]; then + echo "One or more jobs failed or were cancelled" exit 1 else - exit 0 + echo "All jobs completed successfully" fi diff --git a/.github/workflows/unit_mac.yml b/.github/workflows/unit_mac.yml index 5426f6bc51..e3b364ef0b 100644 --- a/.github/workflows/unit_mac.yml +++ b/.github/workflows/unit_mac.yml @@ -8,14 +8,14 @@ permissions: jobs: unit-mac: - runs-on: macos-13 + runs-on: macos-14-xlarge timeout-minutes: 90 env: CARGO_INCREMENTAL: 0 steps: - name: (PR review) Set latest commit status as pending if: ${{ github.event_name == 'pull_request_review' }} - uses: myrotvorets/set-commit-status-action@v2.0.1 + uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 with: sha: ${{ github.event.review.commit_id }} token: ${{ secrets.GITHUB_TOKEN }} @@ -24,15 +24,15 @@ jobs: - name: (PR review) Checkout PR branch if: ${{ github.event_name == 'pull_request_review' }} - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ github.event.review.commit_id }} - name: Checkout branch if: ${{ github.event_name != 'pull_request_review' }} - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - uses: actions/cache@v4 + - uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 name: Cache Cargo registry + index with: path: | @@ -45,12 +45,12 @@ jobs: ${{ runner.os }}-cargo- - run: bash scripts/environment/bootstrap-macos.sh - - run: bash scripts/environment/prepare.sh + - run: bash scripts/environment/prepare.sh --modules=cargo-nextest - run: echo "::add-matcher::.github/matchers/rust.json" # Some tests e.g. `reader_exits_cleanly_when_writer_done_and_in_flight_acks` are flaky. - name: Run tests - uses: nick-fields/retry@v3 + uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 with: timeout_minutes: 45 max_attempts: 3 @@ -59,7 +59,7 @@ jobs: - run: make test-behavior - name: (PR review) Set latest commit status as ${{ job.status }} - uses: myrotvorets/set-commit-status-action@v2.0.1 + uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 if: always() && github.event_name == 'pull_request_review' with: sha: ${{ github.event.review.commit_id }} diff --git a/.github/workflows/unit_windows.yml b/.github/workflows/unit_windows.yml index 357ce89557..1f1db735fb 100644 --- a/.github/workflows/unit_windows.yml +++ b/.github/workflows/unit_windows.yml @@ -7,14 +7,13 @@ permissions: statuses: write jobs: - test-windows: - runs-on: windows-2022-8core + runs-on: windows-2025-8core timeout-minutes: 60 steps: - name: (PR review) Set latest commit status as pending if: ${{ github.event_name == 'pull_request_review' }} - uses: myrotvorets/set-commit-status-action@v2.0.1 + uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 with: sha: ${{ github.event.review.commit_id }} token: ${{ secrets.GITHUB_TOKEN }} @@ -23,19 +22,22 @@ jobs: - name: (PR review) Checkout PR branch if: ${{ github.event_name == 'pull_request_review' }} - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ github.event.review.commit_id }} - name: Checkout branch if: ${{ github.event_name != 'pull_request_review' }} - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - run: .\scripts\environment\bootstrap-windows-2022.ps1 + - uses: actions/setup-python@v5 + with: + python-version: "3.10" + - run: .\scripts\environment\bootstrap-windows-2025.ps1 - run: make test - name: (PR review) Set latest commit status as ${{ job.status }} - uses: myrotvorets/set-commit-status-action@v2.0.1 + uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1 if: always() && github.event_name == 'pull_request_review' with: sha: ${{ github.event.review.commit_id }} diff --git a/.github/workflows/vdev_publish.yml b/.github/workflows/vdev_publish.yml new file mode 100644 index 0000000000..3659031522 --- /dev/null +++ b/.github/workflows/vdev_publish.yml @@ -0,0 +1,63 @@ +name: Publish vdev +on: + push: + tags: [ "vdev-v*.*.*" ] + +permissions: + contents: write # needed for creating releases + +jobs: + build: + runs-on: ${{ matrix.os }} + strategy: + matrix: + include: + - os: macos-latest + target: aarch64-apple-darwin + - os: ubuntu-24.04 + target: x86_64-unknown-linux-gnu + + steps: + - name: Checkout Vector + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Bootstrap runner environment (Ubuntu) + if: startsWith(matrix.os, 'ubuntu') + run: | + sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh + + - name: Bootstrap runner environment (macOS) + if: startsWith(matrix.os, 'macos') + run: bash scripts/environment/bootstrap-macos.sh + + - run: bash scripts/environment/prepare.sh --modules=rustup + + - name: Build + working-directory: vdev + run: cargo build --release --target ${{ matrix.target }} + + - name: Package + shell: bash + run: | + VERSION=${GITHUB_REF_NAME#vdev-v} + OUTDIR="vdev-${{ matrix.target }}-v${VERSION}" + mkdir -p "$OUTDIR" + + BIN_ROOT="${CARGO_TARGET_DIR:-target}" + BIN="${BIN_ROOT}/${{ matrix.target }}/release/vdev" + if [[ ! -f "$BIN" ]]; then + echo "Binary not found at: $BIN" + find ${BIN_ROOT} -type -d + exit 1 + fi + + cp "$BIN" "$OUTDIR/" + tar -czf "${OUTDIR}.tgz" "$OUTDIR" + echo "ASSET=${OUTDIR}.tgz" >> "$GITHUB_ENV" + + - name: Upload asset to release + uses: softprops/action-gh-release@v2 + with: + files: ${{ env.ASSET }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index 24b0593d51..1963fcca25 100644 --- a/.gitignore +++ b/.gitignore @@ -1,24 +1,66 @@ +# Rust cargo-timing*.html +**/*.rs.bk +target +tests/data/wasm/*/target +bench_output.txt + +# Python *.pyc + +# Ruby +miniodat + +# Node.js / JavaScript / TypeScript +node_modules +scripts/package-lock.json + +# Yarn +yarn.lock + +# JetBrains IDEs +.idea/ *.iml -*.tmp -*~ -**/*.rs.bk + +# macOS .DS_Store + +# Emacs .dir-locals.el -checkpoints/* -miniodat -bench_output.txt + +# Temporary files +tmp/ +*.tmp +*~ sample.log -scripts/package-lock.json -target -node_modules -tests/data/wasm/*/target + +# Profiling tools heaptrack.* massif.* -# tilt +# Checkpoints +checkpoints/* + +# Tilt tilt_modules/ -# Jetbrains -.idea/ +# Docker +**/Dockerfile~ +**/Dockerfile.*~ +.dockerignore + +# Docker Compose +docker-compose.override.yml +docker-compose.override.*.yml +**/compose-temp-*.yaml + +# Compose project volume mounts +volumes/ +*.volume/ + +# Environment variable files (can contain secrets) +*.env +.env.* + +# LLM tools +copilot-instructions.md diff --git a/.rustfmt.toml b/.rustfmt.toml index 3d8af21ab6..94966055e5 100644 --- a/.rustfmt.toml +++ b/.rustfmt.toml @@ -1,4 +1,4 @@ -edition = "2021" +edition = "2024" newline_style = "unix" reorder_imports = true diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d15a6dbdcc..eee956d472 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -116,6 +116,45 @@ Please ensure your commits are small and focused; they should tell a story of your change. This helps reviewers to follow your changes, especially for more complex changes. +#### Pre-push + +To reduce iterations you can create do the following: + +```shell +touch .git/hooks/pre-push +chmod +x .git/hooks/pre-push +``` + +You can use the following as a starting point: + +```shell +#!/bin/sh +set -e +echo "Running pre-push checks..." + +# We recommend always running all the following checks. +make check-licenses +make check-fmt +make check-clippy +make check-component-docs + +# Some other checks that in our experience rarely fail on PRs. +make check-deny +make check-docs +make check-version +make check-examples +make check-scripts + +./scripts/check_changelog_fragments.sh + +# The following check is very slow. +# make check-component-features +``` + +Please note that `make check-all` covers all checks, but it is slow and may runs checks not +relevant to your PR. This command is defined +[here](https://github.com/vectordotdev/vector/blob/1ef01aeeef592c21d32ba4d663e199f0608f615b/Makefile#L450-L454). + ### GitHub Pull Requests Once your changes are ready you must submit your branch as a [pull request](https://github.com/vectordotdev/vector/pulls). @@ -131,7 +170,7 @@ format. Vector performs a pull request check to verify the pull request title in case you forget. A list of allowed sub-categories is defined -[here](https://github.com/vectordotdev/vector/blob/master/.github/semantic.yml#L21). +[here](https://github.com/vectordotdev/vector/blob/master/.github/workflows/semantic.yml#L21). The following are all good examples of pull request titles: @@ -240,7 +279,7 @@ first to ensure they pass. ```sh # Run the Clippy linter to catch common mistakes. -cargo vdev check rust --clippy +cargo vdev check rust --fix # Ensure all code is properly formatted. Code can be run through `rustfmt` using `cargo fmt` to ensure it is properly formatted. cargo vdev check fmt # Ensure the internal metrics that Vector emits conform to standards. diff --git a/Cargo.lock b/Cargo.lock index 4d4e3ad137..96a3b6ec6f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "Inflector" @@ -102,15 +102,16 @@ dependencies = [ "cfg-if", "getrandom 0.2.15", "once_cell", + "serde", "version_check", - "zerocopy", + "zerocopy 0.7.31", ] [[package]] name = "aho-corasick" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" dependencies = [ "memchr", ] @@ -132,9 +133,9 @@ dependencies = [ [[package]] name = "allocator-api2" -version = "0.2.16" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] name = "amq-protocol" @@ -146,7 +147,7 @@ dependencies = [ "amq-protocol-types", "amq-protocol-uri", "cookie-factory", - "nom", + "nom 7.1.3", "serde", ] @@ -168,7 +169,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7412353b58923fa012feb9a64ccc0c811747babee2e5a2fd63eb102dc8054c3" dependencies = [ "cookie-factory", - "nom", + "nom 7.1.3", "serde", "serde_json", ] @@ -264,9 +265,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.95" +version = "1.0.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" +checksum = "b0674a1ddeecb70197781e945de4b3b8ffb61fa939a5597bcf48503737663100" [[package]] name = "apache-avro" @@ -333,7 +334,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c6368f9ae5c6ec403ca910327ae0c9437b0a85255b6950c90d497e6177f6e5e" dependencies = [ "proc-macro-hack", - "quote 1.0.38", + "quote 1.0.40", "syn 1.0.109", ] @@ -351,11 +352,11 @@ checksum = "eab1c04a571841102f5345a8fc0f6bb3d31c315dec879b5c6e42e40ce7ffa34e" [[package]] name = "ascii-canvas" -version = "3.0.0" +version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8824ecca2e851cec16968d54a01dd372ef8f95b244fb84b84e70128be347c3c6" +checksum = "ef1e3e699d84ab1b0911a1010c5c106aa34ae89aeac103be5ce0c3859db1e891" dependencies = [ - "term", + "term 1.0.1", ] [[package]] @@ -370,12 +371,12 @@ dependencies = [ [[package]] name = "assert_cmd" -version = "2.0.16" +version = "2.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1835b7f27878de8525dc71410b5a31cdcc5f230aed5ba5df968e09c201b23d" +checksum = "2bd389a4b2970a01282ee455294913c0a43724daedcd1a24c3eb0ec1c1320b66" dependencies = [ "anstyle", - "bstr 1.11.3", + "bstr 1.12.0", "doc-comment", "libc", "predicates", @@ -407,20 +408,29 @@ dependencies = [ "futures-core", ] +[[package]] +name = "async-channel" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89b47800b0be77592da0afd425cc03468052844aff33b84e33cc696f64e77b6a" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + [[package]] name = "async-compression" -version = "0.4.18" +version = "0.4.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df895a515f70646414f4b45c0b79082783b80552b373a68283012928df56f522" +checksum = "977eb15ea9efd848bb8a4a1a2500347ed7f0bf794edf0dc3ddcf439f43d36b23" dependencies = [ - "brotli", - "flate2", + "compression-codecs", + "compression-core", "futures-core", - "memchr", "pin-project-lite", "tokio", - "zstd 0.13.2", - "zstd-safe 7.2.1", ] [[package]] @@ -432,8 +442,8 @@ dependencies = [ "async-lock 2.8.0", "async-task", "concurrent-queue", - "fastrand 2.1.1", - "futures-lite", + "fastrand 2.3.0", + "futures-lite 1.13.0", "slab", ] @@ -446,7 +456,7 @@ dependencies = [ "async-lock 2.8.0", "autocfg", "blocking", - "futures-lite", + "futures-lite 1.13.0", ] [[package]] @@ -455,12 +465,12 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1b6f5d7df27bd294849f8eec66ecfc63d11814df7a4f5d74168a2394467b776" dependencies = [ - "async-channel", + "async-channel 1.9.0", "async-executor", "async-io 1.13.0", "async-lock 2.8.0", "blocking", - "futures-lite", + "futures-lite 1.13.0", "once_cell", ] @@ -477,9 +487,9 @@ dependencies = [ [[package]] name = "async-graphql" -version = "7.0.7" +version = "7.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b76aba2f176af685c2229633881a3adeae51f87ae1811781e73910b7001c93e" +checksum = "036618f842229ba0b89652ffe425f96c7c16a49f7e3cb23b56fca7f61fd74980" dependencies = [ "async-graphql-derive", "async-graphql-parser", @@ -487,16 +497,16 @@ dependencies = [ "async-stream", "async-trait", "base64 0.22.1", - "bytes 1.9.0", + "bytes 1.10.1", "chrono", "fnv", + "futures-timer", "futures-util", - "http 1.1.0", - "indexmap 2.7.0", + "http 1.3.1", + "indexmap 2.11.0", "mime", "multer", "num-traits", - "once_cell", "pin-project-lite", "regex", "serde", @@ -508,26 +518,26 @@ dependencies = [ [[package]] name = "async-graphql-derive" -version = "7.0.7" +version = "7.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72e2e26a6b44bc61df3ca8546402cf9204c28e30c06084cc8e75cd5e34d4f150" +checksum = "fd45deb3dbe5da5cdb8d6a670a7736d735ba65b455328440f236dfb113727a3d" dependencies = [ "Inflector", "async-graphql-parser", - "darling 0.20.8", + "darling 0.20.11", "proc-macro-crate 3.2.0", - "proc-macro2 1.0.93", - "quote 1.0.38", + "proc-macro2 1.0.101", + "quote 1.0.40", "strum 0.26.3", - "syn 2.0.96", + "syn 2.0.106", "thiserror 1.0.68", ] [[package]] name = "async-graphql-parser" -version = "7.0.7" +version = "7.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f801451484b4977d6fe67b29030f81353cabdcbb754e5a064f39493582dac0cf" +checksum = "60b7607e59424a35dadbc085b0d513aa54ec28160ee640cf79ec3b634eba66d3" dependencies = [ "async-graphql-value", "pest", @@ -537,25 +547,24 @@ dependencies = [ [[package]] name = "async-graphql-value" -version = "7.0.7" +version = "7.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69117c43c01d81a69890a9f5dd6235f2f027ca8d1ec62d6d3c5e01ca0edb4f2b" +checksum = "34ecdaff7c9cffa3614a9f9999bf9ee4c3078fe3ce4d6a6e161736b56febf2de" dependencies = [ - "bytes 1.9.0", - "indexmap 2.7.0", + "bytes 1.10.1", + "indexmap 2.11.0", "serde", "serde_json", ] [[package]] name = "async-graphql-warp" -version = "7.0.7" +version = "7.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c3082de64b6d8e3956fa92e3009c27db209aa17388abf7a7d766adc6bb9b8ba" +checksum = "ff6008a33c32d5a048aa72437821eb864dd56a80c0d80c8df48f11f12154db6c" dependencies = [ "async-graphql", "futures-util", - "http 0.2.9", "serde_json", "warp", ] @@ -570,7 +579,7 @@ dependencies = [ "autocfg", "cfg-if", "concurrent-queue", - "futures-lite", + "futures-lite 1.13.0", "log", "parking", "polling 2.8.0", @@ -582,22 +591,21 @@ dependencies = [ [[package]] name = "async-io" -version = "2.1.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10da8f3146014722c89e7859e1d7bb97873125d7346d10ca642ffab794355828" +checksum = "43a2b323ccce0a1d90b449fd71f2a06ca7faa7c54c2751f06c9bd851fc061059" dependencies = [ - "async-lock 2.8.0", + "async-lock 3.4.0", "cfg-if", "concurrent-queue", "futures-io", - "futures-lite", + "futures-lite 2.6.0", "parking", - "polling 3.3.0", + "polling 3.7.4", "rustix 0.38.40", "slab", "tracing 0.1.41", - "waker-fn", - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] @@ -622,12 +630,12 @@ dependencies = [ [[package]] name = "async-nats" -version = "0.38.0" +version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76433c4de73442daedb3a59e991d94e85c14ebfc33db53dfcd347a21cd6ef4f8" +checksum = "08f6da6d49a956424ca4e28fe93656f790d748b469eaccbc7488fec545315180" dependencies = [ "base64 0.22.1", - "bytes 1.9.0", + "bytes 1.10.1", "futures 0.3.31", "memchr", "nkeys", @@ -648,7 +656,7 @@ dependencies = [ "thiserror 1.0.68", "time", "tokio", - "tokio-rustls 0.26.1", + "tokio-rustls 0.26.2", "tokio-util", "tokio-websockets", "tracing 0.1.41", @@ -664,7 +672,7 @@ checksum = "0434b1ed18ce1cf5769b8ac540e33f01fa9471058b5e89da9e06f3c882a8c12f" dependencies = [ "async-io 1.13.0", "blocking", - "futures-lite", + "futures-lite 1.13.0", ] [[package]] @@ -679,7 +687,7 @@ dependencies = [ "blocking", "cfg-if", "event-listener 3.0.1", - "futures-lite", + "futures-lite 1.13.0", "rustix 0.38.40", "windows-sys 0.48.0", ] @@ -702,9 +710,9 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ - "proc-macro2 1.0.93", - "quote 1.0.38", - "syn 2.0.96", + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", ] [[package]] @@ -713,7 +721,7 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e47d90f65a225c4527103a8d747001fc56e375203592b25ad103e1ca13124c5" dependencies = [ - "async-io 2.1.0", + "async-io 2.4.0", "async-lock 2.8.0", "atomic-waker", "cfg-if", @@ -742,26 +750,35 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ - "proc-macro2 1.0.93", - "quote 1.0.38", - "syn 2.0.96", + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", ] [[package]] name = "async-task" -version = "4.5.0" +version = "4.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4eb2cdb97421e01129ccb49169d8279ed21e829929144f4a22a6e54ac549ca1" +checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" [[package]] name = "async-trait" -version = "0.1.85" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", +] + +[[package]] +name = "atoi" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f934833b4b7233644e5848f235df3f57ed8c80f1528a26c3dfa13d2147fa056" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" dependencies = [ - "proc-macro2 1.0.93", - "quote 1.0.38", - "syn 2.0.96", + "num-traits", ] [[package]] @@ -778,12 +795,11 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "aws-config" -version = "1.0.1" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80c950a809d39bc9480207cb1cfc879ace88ea7e3a4392a8e9999e45d6e5692e" +checksum = "8c39646d1a6b51240a1a23bb57ea4eebede7e16fbc237fdc876980233dcecb4f" dependencies = [ "aws-credential-types", - "aws-http", "aws-runtime", "aws-sdk-sso", "aws-sdk-ssooidc", @@ -795,23 +811,23 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", - "bytes 1.9.0", - "fastrand 2.1.1", + "bytes 1.10.1", + "fastrand 2.3.0", "hex", - "http 0.2.9", - "hyper 0.14.28", + "http 1.3.1", "ring", "time", "tokio", "tracing 0.1.41", + "url", "zeroize", ] [[package]] name = "aws-credential-types" -version = "1.2.1" +version = "1.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60e8f6b615cb5fc60a98132268508ad104310f0cfb25a1c22eee76efdf9154da" +checksum = "d025db5d9f52cbc413b167136afb3d8aeea708c0d8884783cf6253be5e22f6f2" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -819,54 +835,41 @@ dependencies = [ "zeroize", ] -[[package]] -name = "aws-http" -version = "0.60.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "361c4310fdce94328cc2d1ca0c8a48c13f43009c61d3367585685a50ca8c66b6" -dependencies = [ - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-types", - "bytes 1.9.0", - "http 0.2.9", - "http-body 0.4.5", - "pin-project-lite", - "tracing 0.1.41", -] - [[package]] name = "aws-runtime" -version = "1.0.3" +version = "1.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce0953f7fc1c4428511345e28ea3e98c8b59c9e91eafae30bf76d71d70642693" +checksum = "c034a1bc1d70e16e7f4e4caf7e9f7693e4c9c24cd91cf17c2a0b21abaebc7c8b" dependencies = [ "aws-credential-types", - "aws-http", "aws-sigv4", "aws-smithy-async", "aws-smithy-eventstream", "aws-smithy-http", + "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", - "fastrand 2.1.1", + "bytes 1.10.1", + "fastrand 2.3.0", "http 0.2.9", + "http-body 0.4.5", "percent-encoding", + "pin-project-lite", "tracing 0.1.41", "uuid", ] [[package]] name = "aws-sdk-cloudwatch" -version = "1.3.0" +version = "1.70.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "043546afc3d129d3d487c2fd121aabe4208300293f541a5c8adffdc919a603b0" +checksum = "84865623b1276624879f5283029fa13d7e6bfd5d58eb7df4dabd485a2f291b9b" dependencies = [ "aws-credential-types", - "aws-http", "aws-runtime", "aws-smithy-async", + "aws-smithy-compression", "aws-smithy-http", "aws-smithy-json", "aws-smithy-query", @@ -875,42 +878,46 @@ dependencies = [ "aws-smithy-types", "aws-smithy-xml", "aws-types", + "fastrand 2.3.0", + "flate2", "http 0.2.9", - "regex", + "http-body 0.4.5", + "once_cell", + "regex-lite", "tracing 0.1.41", ] [[package]] name = "aws-sdk-cloudwatchlogs" -version = "1.3.0" +version = "1.76.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67a5bcf460e098cf49292d216fe520b28a5d9c8dae10db0ff9a97bb2c95dd386" +checksum = "3b7a8df7f19f2ff90191c905fefb8cf0ff512ad7c2cc92c422240ff5b114750c" dependencies = [ "aws-credential-types", - "aws-http", "aws-runtime", "aws-smithy-async", + "aws-smithy-eventstream", "aws-smithy-http", "aws-smithy-json", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", - "bytes 1.9.0", - "fastrand 2.1.1", + "bytes 1.10.1", + "fastrand 2.3.0", "http 0.2.9", - "regex", + "once_cell", + "regex-lite", "tracing 0.1.41", ] [[package]] name = "aws-sdk-elasticsearch" -version = "1.3.0" +version = "1.67.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3090fdd5bf46d4097af7f560cb7305e1ef6f3f743bb7a4531246113e823309b0" +checksum = "3cd27ce0dea72b9e62417a63e79674c2364579e496dc0d379688077057d3da67" dependencies = [ "aws-credential-types", - "aws-http", "aws-runtime", "aws-smithy-async", "aws-smithy-http", @@ -919,20 +926,21 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", - "bytes 1.9.0", + "bytes 1.10.1", + "fastrand 2.3.0", "http 0.2.9", - "regex", + "once_cell", + "regex-lite", "tracing 0.1.41", ] [[package]] name = "aws-sdk-firehose" -version = "1.3.0" +version = "1.71.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8789c5a2d55cb3ed24d8f0635498c53e0f6413c0f9839e9cb54e130e96a81b53" +checksum = "19ab6082dce5671305954d84698eafe55624010d48eb5711da6c8482c5f119bb" dependencies = [ "aws-credential-types", - "aws-http", "aws-runtime", "aws-smithy-async", "aws-smithy-http", @@ -941,42 +949,67 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", - "bytes 1.9.0", + "bytes 1.10.1", + "fastrand 2.3.0", "http 0.2.9", - "regex", + "once_cell", + "regex-lite", "tracing 0.1.41", ] [[package]] name = "aws-sdk-kinesis" -version = "1.3.0" +version = "1.66.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbcd6e94c56f1b4881b405c0953a82d50e110311100cf2355e50fdab79d73f44" +checksum = "e43e5fb05c78cdad4fef5be4503465e4b42292f472fc991823ea4c50078208e4" dependencies = [ "aws-credential-types", - "aws-http", "aws-runtime", "aws-smithy-async", + "aws-smithy-eventstream", "aws-smithy-http", "aws-smithy-json", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", - "bytes 1.9.0", + "bytes 1.10.1", + "fastrand 2.3.0", "http 0.2.9", - "regex", + "once_cell", + "regex-lite", + "tracing 0.1.41", +] + +[[package]] +name = "aws-sdk-kms" +version = "1.75.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb89d6ae47f03ca664f604571d0f29165112543ba1a39878347815b8028c235b" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes 1.10.1", + "fastrand 2.3.0", + "http 0.2.9", + "regex-lite", "tracing 0.1.41", ] [[package]] name = "aws-sdk-s3" -version = "1.4.0" +version = "1.82.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dcafc2fe52cc30b2d56685e2fa6a879ba50d79704594852112337a472ddbd24" +checksum = "e6eab2900764411ab01c8e91a76fd11a63b4e12bc3da97d9e14a0ce1343d86d3" dependencies = [ "aws-credential-types", - "aws-http", "aws-runtime", "aws-sigv4", "aws-smithy-async", @@ -989,24 +1022,29 @@ dependencies = [ "aws-smithy-types", "aws-smithy-xml", "aws-types", - "bytes 1.9.0", + "bytes 1.10.1", + "fastrand 2.3.0", + "hex", + "hmac", "http 0.2.9", + "http 1.3.1", "http-body 0.4.5", + "lru 0.12.5", "once_cell", "percent-encoding", - "regex", + "regex-lite", + "sha2", "tracing 0.1.41", "url", ] [[package]] name = "aws-sdk-secretsmanager" -version = "1.3.0" +version = "1.76.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faea24d86bcabc65048014abd3ee5763a5e8877f678d9cd688a12e086ebe2dbd" +checksum = "eb99bf4d3be2b4598ad26eed5da8d0c930b8d47d76b279a03e47d160151eb0fb" dependencies = [ "aws-credential-types", - "aws-http", "aws-runtime", "aws-smithy-async", "aws-smithy-http", @@ -1015,21 +1053,20 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", - "bytes 1.9.0", - "fastrand 2.1.1", + "bytes 1.10.1", + "fastrand 2.3.0", "http 0.2.9", - "regex", + "regex-lite", "tracing 0.1.41", ] [[package]] name = "aws-sdk-sns" -version = "1.3.0" +version = "1.73.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48bff824fe28888cc4ce6acb52ba8e5cd9f4f38fc5bd1bb40d916e3000e17b57" +checksum = "9e936a9af3eccbd24452a57bb8206d2f8e1e483d38c52b1a2901fcb892d98866" dependencies = [ "aws-credential-types", - "aws-http", "aws-runtime", "aws-smithy-async", "aws-smithy-http", @@ -1040,19 +1077,19 @@ dependencies = [ "aws-smithy-types", "aws-smithy-xml", "aws-types", + "fastrand 2.3.0", "http 0.2.9", - "regex", + "regex-lite", "tracing 0.1.41", ] [[package]] name = "aws-sdk-sqs" -version = "1.3.0" +version = "1.64.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5736d9255f65f36df4f0812665c33fa36042b96192e6bba843ef5fcc75187cd8" +checksum = "514d007ac4d5b156b408d8dd623a57b37ae77425810e0fedcffab57b0dabaded" dependencies = [ "aws-credential-types", - "aws-http", "aws-runtime", "aws-smithy-async", "aws-smithy-http", @@ -1061,20 +1098,21 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", - "bytes 1.9.0", + "bytes 1.10.1", + "fastrand 2.3.0", "http 0.2.9", - "regex", + "once_cell", + "regex-lite", "tracing 0.1.41", ] [[package]] name = "aws-sdk-sso" -version = "1.6.0" +version = "1.64.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86575c7604dcdb583aba3390200e5333d8e4fe597bad54f57b190aaf4fac9771" +checksum = "02d4bdb0e5f80f0689e61c77ab678b2b9304af329616af38aef5b6b967b8e736" dependencies = [ "aws-credential-types", - "aws-http", "aws-runtime", "aws-smithy-async", "aws-smithy-http", @@ -1083,20 +1121,21 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", - "bytes 1.9.0", + "bytes 1.10.1", + "fastrand 2.3.0", "http 0.2.9", - "regex", + "once_cell", + "regex-lite", "tracing 0.1.41", ] [[package]] name = "aws-sdk-ssooidc" -version = "1.6.0" +version = "1.65.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef0d7c1d0730adb5e85407174483a579e39576e0f4350ecd0fac69ec1217b1b" +checksum = "acbbb3ce8da257aedbccdcb1aadafbbb6a5fe9adf445db0e1ea897bdc7e22d08" dependencies = [ "aws-credential-types", - "aws-http", "aws-runtime", "aws-smithy-async", "aws-smithy-http", @@ -1105,20 +1144,21 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", - "bytes 1.9.0", + "bytes 1.10.1", + "fastrand 2.3.0", "http 0.2.9", - "regex", + "once_cell", + "regex-lite", "tracing 0.1.41", ] [[package]] name = "aws-sdk-sts" -version = "1.6.0" +version = "1.73.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f45778089751d5aa8645a02dd60865fa0eea39f00be5db2c7779bc50b83db19a" +checksum = "f1e9c3c24e36183e2f698235ed38dcfbbdff1d09b9232dc866c4be3011e0b47e" dependencies = [ "aws-credential-types", - "aws-http", "aws-runtime", "aws-smithy-async", "aws-smithy-http", @@ -1129,29 +1169,29 @@ dependencies = [ "aws-smithy-types", "aws-smithy-xml", "aws-types", + "fastrand 2.3.0", "http 0.2.9", - "regex", + "regex-lite", "tracing 0.1.41", ] [[package]] name = "aws-sigv4" -version = "1.2.6" +version = "1.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d3820e0c08d0737872ff3c7c1f21ebbb6693d832312d6152bf18ef50a5471c2" +checksum = "084c34162187d39e3740cb635acd73c4e3a551a36146ad6fe8883c929c9f876c" dependencies = [ "aws-credential-types", "aws-smithy-eventstream", "aws-smithy-http", "aws-smithy-runtime-api", "aws-smithy-types", - "bytes 1.9.0", + "bytes 1.10.1", "form_urlencoded", "hex", "hmac", "http 0.2.9", - "http 1.1.0", - "once_cell", + "http 1.3.1", "percent-encoding", "sha2", "time", @@ -1160,9 +1200,9 @@ dependencies = [ [[package]] name = "aws-smithy-async" -version = "1.2.3" +version = "1.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "427cb637d15d63d6f9aae26358e1c9a9c09d5aa490d64b09354c8217cfef0f28" +checksum = "1e190749ea56f8c42bf15dd76c65e14f8f765233e6df9b0506d9d934ebef867c" dependencies = [ "futures-util", "pin-project-lite", @@ -1171,15 +1211,16 @@ dependencies = [ [[package]] name = "aws-smithy-checksums" -version = "0.60.0" +version = "0.63.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5a373ec01aede3dd066ec018c1bc4e8f5dd11b2c11c59c8eef1a5c68101f397" +checksum = "b65d21e1ba6f2cdec92044f904356a19f5ad86961acf015741106cdfafd747c0" dependencies = [ "aws-smithy-http", "aws-smithy-types", - "bytes 1.9.0", + "bytes 1.10.1", "crc32c", "crc32fast", + "crc64fast-nvme", "hex", "http 0.2.9", "http-body 0.4.5", @@ -1190,47 +1231,94 @@ dependencies = [ "tracing 0.1.41", ] +[[package]] +name = "aws-smithy-compression" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c41172a5393f54e26d6b1bfbfce5d0abaa5c46870a1641c1c1899b527f8b6388" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes 1.10.1", + "flate2", + "futures-util", + "http 0.2.9", + "http-body 0.4.5", + "pin-project-lite", + "tracing 0.1.41", +] + [[package]] name = "aws-smithy-eventstream" -version = "0.60.5" +version = "0.60.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cef7d0a272725f87e51ba2bf89f8c21e4df61b9e49ae1ac367a6d69916ef7c90" +checksum = "604c7aec361252b8f1c871a7641d5e0ba3a7f5a586e51b66bc9510a5519594d9" dependencies = [ "aws-smithy-types", - "bytes 1.9.0", + "bytes 1.10.1", "crc32fast", ] [[package]] name = "aws-smithy-http" -version = "0.60.11" +version = "0.62.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c8bc3e8fdc6b8d07d976e301c02fe553f72a39b7a9fea820e023268467d7ab6" +checksum = "7c4dacf2d38996cf729f55e7a762b30918229917eca115de45dfa8dfb97796c9" dependencies = [ "aws-smithy-eventstream", "aws-smithy-runtime-api", "aws-smithy-types", - "bytes 1.9.0", + "bytes 1.10.1", "bytes-utils", "futures-core", "http 0.2.9", + "http 1.3.1", "http-body 0.4.5", - "once_cell", "percent-encoding", "pin-project-lite", "pin-utils", "tracing 0.1.41", ] +[[package]] +name = "aws-smithy-http-client" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147e8eea63a40315d704b97bf9bc9b8c1402ae94f89d5ad6f7550d963309da1b" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "h2 0.3.26", + "h2 0.4.12", + "http 0.2.9", + "http-body 0.4.5", + "hyper 0.14.28", + "hyper-rustls 0.24.2", + "pin-project-lite", + "rustls 0.21.12", + "tokio", + "tracing 0.1.41", +] + [[package]] name = "aws-smithy-json" -version = "0.60.7" +version = "0.61.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4683df9469ef09468dad3473d129960119a0d3593617542b7d52086c8486f2d6" +checksum = "92144e45819cae7dc62af23eac5a038a58aa544432d2102609654376a900bd07" dependencies = [ "aws-smithy-types", ] +[[package]] +name = "aws-smithy-observability" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9364d5989ac4dd918e5cc4c4bdcc61c9be17dcd2586ea7f69e348fc7c6cab393" +dependencies = [ + "aws-smithy-runtime-api", +] + [[package]] name = "aws-smithy-query" version = "0.60.7" @@ -1243,42 +1331,39 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.7.4" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f20685047ca9d6f17b994a07f629c813f08b5bce65523e47124879e60103d45" +checksum = "d3946acbe1ead1301ba6862e712c7903ca9bb230bdf1fbd1b5ac54158ef2ab1f" dependencies = [ "aws-smithy-async", "aws-smithy-http", + "aws-smithy-http-client", + "aws-smithy-observability", "aws-smithy-runtime-api", "aws-smithy-types", - "bytes 1.9.0", - "fastrand 2.1.1", - "h2 0.3.26", + "bytes 1.10.1", + "fastrand 2.3.0", "http 0.2.9", + "http 1.3.1", "http-body 0.4.5", "http-body 1.0.0", - "httparse", - "hyper 0.14.28", - "hyper-rustls 0.24.2", - "once_cell", "pin-project-lite", "pin-utils", - "rustls 0.21.11", "tokio", "tracing 0.1.41", ] [[package]] name = "aws-smithy-runtime-api" -version = "1.7.3" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92165296a47a812b267b4f41032ff8069ab7ff783696d217f0994a0d7ab585cd" +checksum = "07f5e0fc8a6b3f2303f331b94504bbf754d85488f402d6f1dd7a6080f99afe56" dependencies = [ "aws-smithy-async", "aws-smithy-types", - "bytes 1.9.0", + "bytes 1.10.1", "http 0.2.9", - "http 1.1.0", + "http 1.3.1", "pin-project-lite", "tokio", "tracing 0.1.41", @@ -1287,16 +1372,16 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.2.11" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38ddc9bd6c28aeb303477170ddd183760a956a03e083b3902a990238a7e3792d" +checksum = "d498595448e43de7f4296b7b7a18a8a02c61ec9349128c80a368f7c3b4ab11a8" dependencies = [ "base64-simd", - "bytes 1.9.0", + "bytes 1.10.1", "bytes-utils", "futures-core", "http 0.2.9", - "http 1.1.0", + "http 1.3.1", "http-body 0.4.5", "http-body 1.0.0", "http-body-util", @@ -1322,9 +1407,9 @@ dependencies = [ [[package]] name = "aws-types" -version = "1.3.3" +version = "1.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5221b91b3e441e6675310829fd8984801b772cb1546ef6c0e54dec9f1ac13fef" +checksum = "b069d19bf01e46298eaedd7c6f283fe565a59263e53eebec945f3e6398f42390" dependencies = [ "aws-credential-types", "aws-smithy-async", @@ -1343,7 +1428,7 @@ dependencies = [ "async-trait", "axum-core 0.3.4", "bitflags 1.3.2", - "bytes 1.9.0", + "bytes 1.10.1", "futures-util", "http 0.2.9", "http-body 0.4.5", @@ -1358,7 +1443,7 @@ dependencies = [ "serde", "sync_wrapper 0.1.2", "tokio", - "tower", + "tower 0.4.13", "tower-layer", "tower-service", ] @@ -1371,9 +1456,9 @@ checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf" dependencies = [ "async-trait", "axum-core 0.4.5", - "bytes 1.9.0", + "bytes 1.10.1", "futures-util", - "http 1.1.0", + "http 1.3.1", "http-body 1.0.0", "http-body-util", "itoa", @@ -1385,7 +1470,7 @@ dependencies = [ "rustversion", "serde", "sync_wrapper 1.0.1", - "tower", + "tower 0.4.13", "tower-layer", "tower-service", ] @@ -1397,7 +1482,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" dependencies = [ "async-trait", - "bytes 1.9.0", + "bytes 1.10.1", "futures-util", "http 0.2.9", "http-body 0.4.5", @@ -1414,9 +1499,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" dependencies = [ "async-trait", - "bytes 1.9.0", + "bytes 1.10.1", "futures-util", - "http 1.1.0", + "http 1.3.1", "http-body 1.0.0", "http-body-util", "mime", @@ -1435,7 +1520,7 @@ checksum = "4ccd63c07d1fbfb3d4543d7ea800941bf5a30db1911b9b9e4db3b2c4210a434f" dependencies = [ "async-trait", "base64 0.21.7", - "bytes 1.9.0", + "bytes 1.10.1", "dyn-clone", "futures 0.3.31", "getrandom 0.2.15", @@ -1443,7 +1528,7 @@ dependencies = [ "log", "paste", "pin-project", - "quick-xml", + "quick-xml 0.31.0", "rand 0.8.5", "reqwest 0.11.26", "rustc_version 0.4.1", @@ -1484,7 +1569,7 @@ dependencies = [ "RustyXML", "async-trait", "azure_core", - "bytes 1.9.0", + "bytes 1.10.1", "futures 0.3.31", "hmac", "log", @@ -1506,7 +1591,7 @@ dependencies = [ "RustyXML", "azure_core", "azure_storage", - "bytes 1.9.0", + "bytes 1.10.1", "futures 0.3.31", "log", "serde", @@ -1530,13 +1615,12 @@ dependencies = [ [[package]] name = "backon" -version = "0.4.1" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c1a6197b2120bb2185a267f6515038558b019e92b832bb0320e96d66268dcf9" +checksum = "592277618714fbcecda9a02ba7a8781f319d26532a88553bbacc77ba5d2b3a8d" dependencies = [ - "fastrand 1.9.0", - "futures-core", - "pin-project", + "fastrand 2.3.0", + "gloo-timers", "tokio", ] @@ -1569,9 +1653,12 @@ checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" [[package]] name = "base62" -version = "2.0.3" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48fa474cf7492f9a299ba6019fb99ec673e1739556d48e8a90eabaea282ef0e4" +checksum = "10e52a7bcb1d6beebee21fb5053af9e3cbb7a7ed1a4909e534040e676437ab1f" +dependencies = [ + "rustversion", +] [[package]] name = "base64" @@ -1607,30 +1694,15 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" -[[package]] -name = "bit-set" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" -dependencies = [ - "bit-vec 0.6.3", -] - [[package]] name = "bit-set" version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" dependencies = [ - "bit-vec 0.8.0", + "bit-vec", ] -[[package]] -name = "bit-vec" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" - [[package]] name = "bit-vec" version = "0.8.0" @@ -1645,9 +1717,12 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.6.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" +dependencies = [ + "serde", +] [[package]] name = "bitmask-enum" @@ -1655,8 +1730,8 @@ version = "2.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6cbbb8f56245b5a479b30a62cdc86d26e2f35c2b9f594bc4671654b03851380" dependencies = [ - "quote 1.0.38", - "syn 2.0.96", + "quote 1.0.40", + "syn 2.0.106", ] [[package]] @@ -1695,12 +1770,12 @@ version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c36a4d0d48574b3dd360b4b7d95cc651d2b6557b6402848a27d4b228a473e2a" dependencies = [ - "async-channel", + "async-channel 1.9.0", "async-lock 2.8.0", "async-task", - "fastrand 2.1.1", + "fastrand 2.3.0", "futures-io", - "futures-lite", + "futures-lite 1.13.0", "piper", "tracing 0.1.41", ] @@ -1711,34 +1786,34 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "489d2af57852b78a86478273ac6a1ef912061b6af3a439694c49f309f6ea3bdd" dependencies = [ - "siphasher", + "siphasher 0.3.11", ] [[package]] name = "bollard" -version = "0.16.1" +version = "0.19.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0aed08d3adb6ebe0eff737115056652670ae290f177759aac19c30456135f94c" +checksum = "8796b390a5b4c86f9f2e8173a68c2791f4fa6b038b84e96dbc01c016d1e6722c" dependencies = [ "base64 0.22.1", "bollard-stubs", - "bytes 1.9.0", + "bytes 1.10.1", "chrono", "futures-core", "futures-util", "hex", "home", - "http 1.1.0", + "http 1.3.1", "http-body-util", - "hyper 1.4.1", + "hyper 1.7.0", "hyper-named-pipe", - "hyper-rustls 0.26.0", + "hyper-rustls 0.27.5", "hyper-util", - "hyperlocal-next", + "hyperlocal", "log", "pin-project-lite", - "rustls 0.22.4", - "rustls-native-certs 0.7.0", + "rustls 0.23.23", + "rustls-native-certs 0.8.1", "rustls-pemfile 2.1.0", "rustls-pki-types", "serde", @@ -1746,7 +1821,7 @@ dependencies = [ "serde_json", "serde_repr", "serde_urlencoded", - "thiserror 1.0.68", + "thiserror 2.0.3", "tokio", "tokio-util", "tower-service", @@ -1756,45 +1831,28 @@ dependencies = [ [[package]] name = "bollard-stubs" -version = "1.44.0-rc.2" +version = "1.49.0-rc.28.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "709d9aa1c37abb89d40f19f5d0ad6f0d88cb1581264e571c9350fc5bb89cf1c5" +checksum = "2e7814991259013d5a5bee4ae28657dae0747d843cf06c40f7fc0c2894d6fa38" dependencies = [ "chrono", "serde", + "serde_json", "serde_repr", - "serde_with 3.12.0", -] - -[[package]] -name = "borsh" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf617fabf5cdbdc92f774bfe5062d870f228b80056d41180797abf48bed4056e" -dependencies = [ - "borsh-derive", - "cfg_aliases 0.1.1", + "serde_with 3.14.0", ] [[package]] -name = "borsh-derive" -version = "1.2.0" +name = "borrow-or-share" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f404657a7ea7b5249e36808dff544bc88a28f26e0ac40009f674b7a009d14be3" -dependencies = [ - "once_cell", - "proc-macro-crate 2.0.0", - "proc-macro2 1.0.93", - "quote 1.0.38", - "syn 2.0.96", - "syn_derive", -] +checksum = "3eeab4423108c5d7c744f4d234de88d18d636100093ae04caf4825134b9c3a32" [[package]] name = "brotli" -version = "7.0.0" +version = "8.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc97b8f16f944bba54f0433f07e30be199b6dc2bd25937444bbad560bcea29bd" +checksum = "cf19e729cdbd51af9a397fb9ef8ac8378007b797f8273cfbfdf45dcaa316167b" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -1803,9 +1861,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "4.0.0" +version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6221fe77a248b9117d431ad93761222e1cf8ff282d9d1d5d9f53d6299a1cf76" +checksum = "874bb8112abecc98cbd6d81ea4fa7e94fb9449648c93cc89aa40c81c24d7de03" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -1845,9 +1903,9 @@ dependencies = [ [[package]] name = "bstr" -version = "1.11.3" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "531a9155a481e2ee699d4f98f43c0ca4ff8ee1bfd55c31e9e98fb29d2b176fe0" +checksum = "234113d19d0d7d613b40e86fb654acf958910802bcceab913a4f9e7cda03b1a4" dependencies = [ "memchr", "regex-automata 0.4.8", @@ -1877,11 +1935,17 @@ version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7ec4c6f261935ad534c0c22dbef2201b45918860eb1c574b972bd213a76af61" dependencies = [ - "proc-macro2 1.0.93", - "quote 1.0.38", + "proc-macro2 1.0.101", + "quote 1.0.40", "syn 1.0.109", ] +[[package]] +name = "bytecount" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" + [[package]] name = "bytemuck" version = "1.21.0" @@ -1906,9 +1970,9 @@ dependencies = [ [[package]] name = "bytes" -version = "1.9.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" +checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" dependencies = [ "serde", ] @@ -1919,25 +1983,25 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e47d3a8076e283f3acd27400535992edb3ba4b5bb72f8891ad8fbe7932a7d4b9" dependencies = [ - "bytes 1.9.0", + "bytes 1.10.1", "either", ] [[package]] name = "bytesize" -version = "1.3.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e368af43e418a04d52505cf3dbc23dda4e3407ae2fa99fd0e4f308ce546acc" +checksum = "a3c8f83209414aacf0eeae3cf730b18d6981697fba62f200fcfb92b9f082acba" [[package]] name = "cargo-lock" -version = "10.0.1" +version = "10.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6469776d007022d505bbcc2be726f5f096174ae76d710ebc609eb3029a45b551" +checksum = "c06acb4f71407ba205a07cb453211e0e6a67b21904e47f6ba1f9589e38f2e454" dependencies = [ - "semver 1.0.24", + "semver 1.0.26", "serde", - "toml", + "toml 0.8.23", "url", ] @@ -1973,12 +2037,13 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.83" +version = "1.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +checksum = "c736e259eea577f443d5c86c304f9f4ae0295c43f3ba05c21f1d66b5f06001af" dependencies = [ "jobserver", "libc", + "shlex", ] [[package]] @@ -1998,15 +2063,9 @@ dependencies = [ [[package]] name = "cfg-if" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" - -[[package]] -name = "cfg_aliases" -version = "0.1.1" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" +checksum = "2fd1289c04a9ea8cb22300a459a72a385d7c73d3259e2ed7dcb2af674838cfa9" [[package]] name = "cfg_aliases" @@ -2050,9 +2109,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.39" +version = "0.4.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825" +checksum = "c469d952047f47f91b68d1cba3f10d63c11d73e4636f24f08daf0278abf01c4d" dependencies = [ "android-tzdata", "iana-time-zone", @@ -2060,31 +2119,20 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-targets 0.52.6", + "windows-link", ] [[package]] name = "chrono-tz" -version = "0.10.0" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd6dd8046d00723a59a2f8c5f295c515b9bb9a331ee4f8f3d4dd49e428acd3b6" +checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3" dependencies = [ "chrono", - "chrono-tz-build", - "phf", + "phf 0.12.1", "serde", ] -[[package]] -name = "chrono-tz-build" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94fea34d77a245229e7746bd2beb786cd2a896f306ff491fb8cecb3074b10a7" -dependencies = [ - "parse-zoneinfo", - "phf_codegen", -] - [[package]] name = "ciborium" version = "0.2.2" @@ -2114,26 +2162,15 @@ dependencies = [ [[package]] name = "cidr" -version = "0.2.2" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d18b093eba54c9aaa1e3784d4361eb2ba944cf7d0a932a830132238f483e8d8" +checksum = "bd1b64030216239a2e7c364b13cd96a2097ebf0dfe5025f2dedee14a23f2ab60" [[package]] -name = "cidr-utils" -version = "0.6.1" +name = "cipher" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25c0a9fb70c2c2cc2a520aa259b1d1345650046a07df1b6da1d3cefcd327f43e" -dependencies = [ - "cidr", - "num-bigint", - "num-traits", -] - -[[package]] -name = "cipher" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ "crypto-common", "inout", @@ -2142,9 +2179,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.26" +version = "4.5.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8eb5e908ef3a6efbe1ed62520fb7287959888c88485abe072543190ecc66783" +checksum = "7eac00902d9d136acd712710d71823fb8ac8004ca445a89e73a41d45aa712931" dependencies = [ "clap_builder", "clap_derive", @@ -2152,9 +2189,9 @@ dependencies = [ [[package]] name = "clap-verbosity-flag" -version = "3.0.2" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2678fade3b77aa3a8ff3aae87e9c008d3fb00473a41c71fbf74e91c8c7b37e84" +checksum = "9d92b1fab272fe943881b77cc6e920d6543e5b1bfadbd5ed81c7c5a755742394" dependencies = [ "clap", "log", @@ -2162,36 +2199,36 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.26" +version = "4.5.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96b01801b5fc6a0a232407abc821660c9c6d25a1cafc0d4f85f29fb8d9afc121" +checksum = "2ad9bbf750e73b5884fb8a211a9424a1906c1e156724260fdae972f31d70e1d6" dependencies = [ "anstream", "anstyle", "clap_lex", - "strsim 0.11.0", + "strsim 0.11.1", "terminal_size", ] [[package]] name = "clap_complete" -version = "4.5.42" +version = "4.5.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33a7e468e750fa4b6be660e8b5651ad47372e8fb114030b594c2d75d48c5ffd0" +checksum = "4d9501bd3f5f09f7bbee01da9a511073ed30a80cd7a509f1214bb74eadea71ad" dependencies = [ "clap", ] [[package]] name = "clap_derive" -version = "4.5.24" +version = "4.5.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54b755194d6389280185988721fffba69495eed5ee9feeee9a599b53db80318c" +checksum = "bbfd7eae0b0f1a6e63d4b13c9c478de77c2eb546fba158ad50b4203dc24b9f9c" dependencies = [ "heck 0.5.0", - "proc-macro2 1.0.93", - "quote 1.0.38", - "syn 2.0.96", + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", ] [[package]] @@ -2234,7 +2271,7 @@ name = "codecs" version = "0.1.0" dependencies = [ "apache-avro", - "bytes 1.9.0", + "bytes 1.10.1", "chrono", "csv-core", "derivative", @@ -2247,16 +2284,16 @@ dependencies = [ "ordered-float 4.6.0", "prost 0.12.6", "prost-reflect", - "rand 0.8.5", + "rand 0.9.2", "regex", "rstest", "serde", "serde_json", - "serde_with 3.12.0", + "serde_with 3.14.0", "similar-asserts", "smallvec", - "snafu 0.7.5", - "syslog_loose", + "snafu 0.8.9", + "syslog_loose 0.23.0", "tokio", "tokio-util", "tracing 0.1.41", @@ -2272,12 +2309,13 @@ dependencies = [ [[package]] name = "codespan-reporting" -version = "0.11.1" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" +checksum = "fe6d2e5af09e8c8ad56c969f2157a3d4238cebc7c55f0a517728c38f7b200f81" dependencies = [ + "serde", "termcolor", - "unicode-width 0.1.13", + "unicode-width 0.2.0", ] [[package]] @@ -2314,7 +2352,7 @@ version = "4.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35ed6e9d84f0b51a7f52daf1c7d71dd136fd7a3f41a8462b8cdb8c78d920fad4" dependencies = [ - "bytes 1.9.0", + "bytes 1.10.1", "futures-core", "memchr", "pin-project-lite", @@ -2350,6 +2388,26 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "compression-codecs" +version = "0.4.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "485abf41ac0c8047c07c87c72c8fb3eb5197f6e9d7ded615dfd1a00ae00a0f64" +dependencies = [ + "brotli", + "compression-core", + "flate2", + "memchr", + "zstd 0.13.2", + "zstd-safe 7.2.1", +] + +[[package]] +name = "compression-core" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e47641d3deaf41fb1538ac1f54735925e275eaf3bf4d55c81b137fba797e5cbb" + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -2361,14 +2419,14 @@ dependencies = [ [[package]] name = "confy" -version = "0.6.1" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45b1f4c00870f07dc34adcac82bb6a72cc5aabca8536ba1797e01df51d2ce9a0" +checksum = "f29222b549d4e3ded127989d523da9e928918d0d0d7f7c1690b439d0d538bae9" dependencies = [ "directories", "serde", - "thiserror 1.0.68", - "toml", + "thiserror 2.0.3", + "toml 0.8.23", ] [[package]] @@ -2380,10 +2438,22 @@ dependencies = [ "encode_unicode 0.3.6", "lazy_static", "libc", - "unicode-width 0.1.13", "windows-sys 0.45.0", ] +[[package]] +name = "console" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e09ced7ebbccb63b4c65413d821f2e00ce54c5ca4514ddc6b3c892fdbcbc69d" +dependencies = [ + "encode_unicode 1.0.0", + "libc", + "once_cell", + "unicode-width 0.2.0", + "windows-sys 0.60.2", +] + [[package]] name = "console-api" version = "0.8.1" @@ -2391,8 +2461,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8030735ecb0d128428b64cd379809817e620a40e5001c54465b99ec5feec2857" dependencies = [ "futures-core", - "prost 0.13.3", - "prost-types 0.13.3", + "prost 0.13.5", + "prost-types 0.13.5", "tonic 0.12.3", "tracing-core 0.1.33", ] @@ -2410,8 +2480,8 @@ dependencies = [ "hdrhistogram", "humantime", "hyper-util", - "prost 0.13.3", - "prost-types 0.13.3", + "prost 0.13.5", + "prost-types 0.13.5", "serde", "serde_json", "thread_local", @@ -2431,9 +2501,9 @@ checksum = "28c122c3980598d243d63d9a704629a2d748d101f278052ff068be5a4423ab6f" [[package]] name = "const_fn" -version = "0.4.9" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbdcdcb6d86f71c5e97409ad45898af11cbc995b4ee8112d59095a28d376c935" +checksum = "2f8a2ca5ac02d09563609681103aada9e1777d54fc57a5acd7a41404f9c93b6e" [[package]] name = "convert_case" @@ -2443,19 +2513,57 @@ checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" [[package]] name = "convert_case" -version = "0.6.0" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb402b8d4c85569410425650ce3eddc7d698ed96d39a73f941b08fb63082f1e7" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "convert_case" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca" +checksum = "baaaa0ecca5b51987b9423ccdc971514dd8b0bb7b4060b983d3664dad3f1f89f" dependencies = [ "unicode-segmentation", ] +[[package]] +name = "cookie" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ddef33a339a91ea89fb53151bd0a4689cfce27055c291dfa69945475d22c747" +dependencies = [ + "percent-encoding", + "time", + "version_check", +] + [[package]] name = "cookie-factory" version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "396de984970346b0d9e93d1415082923c679e5ae5c3ee3dcbd104f5610af126b" +[[package]] +name = "cookie_store" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2eac901828f88a5241ee0600950ab981148a18f2f756900ffba1b125ca6a3ef9" +dependencies = [ + "cookie", + "document-features", + "idna 1.0.3", + "log", + "publicsuffix", + "serde", + "serde_derive", + "serde_json", + "time", + "url", +] + [[package]] name = "core-foundation" version = "0.9.3" @@ -2466,11 +2574,21 @@ dependencies = [ "libc", ] +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" -version = "0.8.4" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "core2" @@ -2492,9 +2610,9 @@ dependencies = [ [[package]] name = "crc" -version = "3.2.1" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" +checksum = "9710d3b3739c2e349eb44fe848ad0b7c8cb1e42bd87ee49371df2f7acaf3e675" dependencies = [ "crc-catalog", ] @@ -2507,44 +2625,49 @@ checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" [[package]] name = "crc32c" -version = "0.6.4" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8f48d60e5b4d2c53d5c2b1d8a58c849a70ae5e5509b08a48d047e3b65714a74" +checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" dependencies = [ "rustc_version 0.4.1", ] [[package]] name = "crc32fast" -version = "1.4.2" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" dependencies = [ "cfg-if", ] +[[package]] +name = "crc64fast-nvme" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4955638f00a809894c947f85a024020a20815b65a5eea633798ea7924edab2b3" +dependencies = [ + "crc", +] + [[package]] name = "criterion" -version = "0.5.1" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +checksum = "e1c047a62b0cc3e145fa84415a3191f628e980b194c2755aa12300a4e6cbd928" dependencies = [ "anes", "cast", "ciborium", "clap", "criterion-plot", - "futures 0.3.31", - "is-terminal", - "itertools 0.10.5", + "itertools 0.13.0", "num-traits", - "once_cell", "oorandom", "plotters", "rayon", "regex", "serde", - "serde_derive", "serde_json", "tinytemplate", "tokio", @@ -2553,14 +2676,20 @@ dependencies = [ [[package]] name = "criterion-plot" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +checksum = "9b1bcc0dc7dfae599d84ad0b1a55f80cde8af3725da8313b528da95ef783e338" dependencies = [ "cast", - "itertools 0.10.5", + "itertools 0.13.0", ] +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + [[package]] name = "crossbeam-channel" version = "0.5.8" @@ -2616,17 +2745,34 @@ version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.9.0", "crossterm_winapi", - "futures-core", "mio", - "parking_lot", + "parking_lot 0.12.4", "rustix 0.38.40", "signal-hook", "signal-hook-mio", "winapi", ] +[[package]] +name = "crossterm" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8b9f2e4c67f833b660cdb0a3523065869fb35570177239812ed4c905aeff87b" +dependencies = [ + "bitflags 2.9.0", + "crossterm_winapi", + "document-features", + "futures-core", + "mio", + "parking_lot 0.12.4", + "rustix 1.0.1", + "signal-hook", + "signal-hook-mio", + "winapi", +] + [[package]] name = "crossterm_winapi" version = "0.9.1" @@ -2694,9 +2840,9 @@ dependencies = [ [[package]] name = "csv-core" -version = "0.1.11" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" +checksum = "7d02f3b0da4c6504f86e9cd789d8dbafab48c2321be74e9987593de5a894d93d" dependencies = [ "memchr", ] @@ -2747,9 +2893,9 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ - "proc-macro2 1.0.93", - "quote 1.0.38", - "syn 2.0.96", + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", ] [[package]] @@ -2764,12 +2910,12 @@ dependencies = [ [[package]] name = "darling" -version = "0.20.8" +version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54e36fcd13ed84ffdfda6f5be89b31287cbb80c439841fe69e04841435464391" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" dependencies = [ - "darling_core 0.20.8", - "darling_macro 0.20.8", + "darling_core 0.20.11", + "darling_macro 0.20.11", ] [[package]] @@ -2780,24 +2926,24 @@ checksum = "859d65a907b6852c9361e3185c862aae7fafd2887876799fa55f5f99dc40d610" dependencies = [ "fnv", "ident_case", - "proc-macro2 1.0.93", - "quote 1.0.38", + "proc-macro2 1.0.101", + "quote 1.0.40", "strsim 0.10.0", "syn 1.0.109", ] [[package]] name = "darling_core" -version = "0.20.8" +version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c2cf1c23a687a1feeb728783b993c4e1ad83d99f351801977dd809b48d0a70f" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" dependencies = [ "fnv", "ident_case", - "proc-macro2 1.0.93", - "quote 1.0.38", - "strsim 0.10.0", - "syn 2.0.96", + "proc-macro2 1.0.101", + "quote 1.0.40", + "strsim 0.11.1", + "syn 2.0.106", ] [[package]] @@ -2807,19 +2953,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" dependencies = [ "darling_core 0.13.4", - "quote 1.0.38", + "quote 1.0.40", "syn 1.0.109", ] [[package]] name = "darling_macro" -version = "0.20.8" +version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a668eda54683121533a393014d8692171709ff57a7d61f187b6e782719f8933f" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ - "darling_core 0.20.8", - "quote 1.0.38", - "syn 2.0.96", + "darling_core 0.20.11", + "quote 1.0.40", + "syn 2.0.106", ] [[package]] @@ -2828,19 +2974,6 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7762d17f1241643615821a8455a0b2c3e803784b058693d990b11f2dce25a0ca" -[[package]] -name = "dashmap" -version = "5.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" -dependencies = [ - "cfg-if", - "hashbrown 0.14.5", - "lock_api", - "once_cell", - "parking_lot_core", -] - [[package]] name = "dashmap" version = "6.1.0" @@ -2852,14 +2985,14 @@ dependencies = [ "hashbrown 0.14.5", "lock_api", "once_cell", - "parking_lot_core", + "parking_lot_core 0.9.11", ] [[package]] name = "data-encoding" -version = "2.6.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" +checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476" [[package]] name = "data-url" @@ -2869,16 +3002,17 @@ checksum = "5c297a1c74b71ae29df00c3e22dd9534821d60eb9af5a0192823fa2acea70c2a" [[package]] name = "databend-client" -version = "0.22.2" +version = "0.28.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd8770a1c49fa21e62a768a0de442cc3f77998a357303d56ddd3485cb7c58d3a" +checksum = "5d689ffeaa08b1e4be3f035fcdadd4ea69db3dbf529ec5668c6911b8a301fc06" dependencies = [ - "async-trait", + "cookie", "log", "once_cell", - "parking_lot", + "parking_lot 0.12.4", "percent-encoding", - "reqwest 0.12.4", + "reqwest 0.12.9", + "semver 1.0.26", "serde", "serde_json", "tokio", @@ -2900,11 +3034,10 @@ dependencies = [ [[package]] name = "deadpool" -version = "0.10.0" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb84100978c1c7b37f09ed3ce3e5f843af02c2a2c431bae5b19230dad2c1b490" +checksum = "5ed5957ff93768adf7a65ab167a17835c3d2c3c50d084fe305174c112f468e2f" dependencies = [ - "async-trait", "deadpool-runtime", "num_cpus", "tokio", @@ -2915,6 +3048,9 @@ name = "deadpool-runtime" version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63dfa964fe2a66f3fde91fc70b267fe193d822c7e603e2a675a49a7f46ad3f49" +dependencies = [ + "tokio", +] [[package]] name = "der" @@ -2943,8 +3079,8 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" dependencies = [ - "proc-macro2 1.0.93", - "quote 1.0.38", + "proc-macro2 1.0.101", + "quote 1.0.40", "syn 1.0.109", ] @@ -2954,9 +3090,9 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" dependencies = [ - "proc-macro2 1.0.93", - "quote 1.0.38", - "syn 2.0.96", + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", ] [[package]] @@ -2974,10 +3110,10 @@ version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d48cda787f839151732d396ac69e3473923d54312c070ee21e9effcaa8ca0b1d" dependencies = [ - "darling 0.20.8", - "proc-macro2 1.0.93", - "quote 1.0.38", - "syn 2.0.96", + "darling 0.20.11", + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", ] [[package]] @@ -2987,7 +3123,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "206868b8242f27cecce124c19fd88157fbd0dd334df2587f36417bafbc85097b" dependencies = [ "derive_builder_core", - "syn 2.0.96", + "syn 2.0.106", ] [[package]] @@ -2997,18 +3133,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" dependencies = [ "convert_case 0.4.0", - "proc-macro2 1.0.93", - "quote 1.0.38", + "proc-macro2 1.0.101", + "quote 1.0.40", "rustc_version 0.4.1", "syn 1.0.109", ] -[[package]] -name = "diff" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" - [[package]] name = "difflib" version = "0.4.0" @@ -3029,9 +3159,9 @@ dependencies = [ [[package]] name = "directories" -version = "5.0.1" +version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a49173b84e034382284f27f1af4dcbbd231ffa358c0fe316541a7337f376a35" +checksum = "16f5094c54661b38d03bd7e50df373292118db60b585c08a411c6d840017fe7d" dependencies = [ "dirs-sys", ] @@ -3048,14 +3178,14 @@ dependencies = [ [[package]] name = "dirs-sys" -version = "0.4.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" +checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" dependencies = [ "libc", "option-ext", - "redox_users", - "windows-sys 0.48.0", + "redox_users 0.5.0", + "windows-sys 0.60.2", ] [[package]] @@ -3065,7 +3195,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" dependencies = [ "libc", - "redox_users", + "redox_users 0.4.3", "winapi", ] @@ -3075,9 +3205,9 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ - "proc-macro2 1.0.93", - "quote 1.0.38", - "syn 2.0.96", + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", ] [[package]] @@ -3088,7 +3218,7 @@ checksum = "e5766087c2235fec47fafa4cfecc81e494ee679d0fd4a59887ea0919bfb0e4fc" dependencies = [ "cfg-if", "libc", - "socket2 0.5.8", + "socket2 0.5.10", "windows-sys 0.48.0", ] @@ -3099,7 +3229,7 @@ dependencies = [ "criterion", "data-encoding", "hickory-proto", - "snafu 0.7.5", + "snafu 0.8.9", ] [[package]] @@ -3108,14 +3238,15 @@ version = "0.1.0" dependencies = [ "anyhow", "base64 0.22.1", - "bytes 1.9.0", + "bytes 1.10.1", "chrono", "chrono-tz", "dnsmsg-parser", "hickory-proto", + "paste", "prost 0.12.6", "prost-build 0.12.6", - "snafu 0.7.5", + "snafu 0.8.9", "tracing 0.1.41", "vector-lib", "vrl", @@ -3134,20 +3265,33 @@ dependencies = [ "anyhow", "serde", "serde_json", - "snafu 0.7.5", + "snafu 0.8.9", "tracing 0.1.41", "vector-config", "vector-config-common", ] +[[package]] +name = "document-features" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95249b50c6c185bee49034bcb378a49dc2b5dff0be90ff6616d31d64febab05d" +dependencies = [ + "litrs", +] + [[package]] name = "domain" -version = "0.10.3" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64008666d9f3b6a88a63cd28ad8f3a5a859b8037e11bfb680c1b24945ea1c28d" +checksum = "a11dd7f04a6a6d2aea0153c6e31f5ea7af8b2efdf52cdaeea7a9a592c7fefef9" dependencies = [ - "bytes 1.9.0", + "bumpalo", + "bytes 1.10.1", + "domain-macros", "futures-util", + "hashbrown 0.14.5", + "log", "moka", "octseq", "rand 0.8.5", @@ -3158,6 +3302,23 @@ dependencies = [ "tracing 0.1.41", ] +[[package]] +name = "domain-macros" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e197fdfd2cdb5fdeb7f8ddcf3aed5d5d04ecde2890d448b14ffb716f7376b70" +dependencies = [ + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", +] + +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + [[package]] name = "duct" version = "0.13.6" @@ -3178,9 +3339,9 @@ checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" [[package]] name = "dyn-clone" -version = "1.0.17" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d6ef0072f8a535281e4876be788938b528e9a1d43900b82c2569af7da799125" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" [[package]] name = "ecdsa" @@ -3225,6 +3386,9 @@ name = "either" version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" +dependencies = [ + "serde", +] [[package]] name = "elliptic-curve" @@ -3247,6 +3411,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "email_address" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e079f19b08ca6239f47f8ba8509c11cf3ea30095831f7fed61441475edd8c449" +dependencies = [ + "serde", +] + [[package]] name = "ena" version = "0.14.2" @@ -3301,8 +3474,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "21cdad81446a7f7dc43f6a77409efeb9733d2fa65553efef6018ef257c959b73" dependencies = [ "heck 0.4.1", - "proc-macro2 1.0.93", - "quote 1.0.38", + "proc-macro2 1.0.101", + "quote 1.0.40", "syn 1.0.109", ] @@ -3313,9 +3486,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ffccbb6966c05b32ef8fbac435df276c4ae4d3dc55a8cd0eb9745e6c12f546a" dependencies = [ "heck 0.4.1", - "proc-macro2 1.0.93", - "quote 1.0.38", - "syn 2.0.96", + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", ] [[package]] @@ -3325,29 +3498,29 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa18ce2bc66555b3218614519ac839ddb759a7d6720732f979ef8d13be147ecd" dependencies = [ "once_cell", - "proc-macro2 1.0.93", - "quote 1.0.38", - "syn 2.0.96", + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", ] [[package]] name = "enumflags2" -version = "0.7.10" +version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d232db7f5956f3f14313dc2f87985c58bd2c695ce124c8cdd984e08e15ac133d" +checksum = "1027f7680c853e056ebcec683615fb6fbbc07dbaa13b4d5d9442b146ded4ecef" dependencies = [ "enumflags2_derive", ] [[package]] name = "enumflags2_derive" -version = "0.7.10" +version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de0d48a183585823424a4ce1aa132d174a6a81bd540895822eb4c8373a8e49e8" +checksum = "67c78a4d8fdf9953a5c9d458f9efe940fd97a0cab0941c075a813ac594733827" dependencies = [ - "proc-macro2 1.0.93", - "quote 1.0.38", - "syn 2.0.96", + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", ] [[package]] @@ -3406,12 +3579,12 @@ dependencies = [ [[package]] name = "errno" -version = "0.3.8" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" +checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -3429,6 +3602,17 @@ version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "281e452d3bad4005426416cdba5ccfd4f5c1280e10099e21db27f7c1c28347fc" +[[package]] +name = "etcetera" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" +dependencies = [ + "cfg-if", + "home", + "windows-sys 0.48.0", +] + [[package]] name = "event-listener" version = "2.5.3" @@ -3467,6 +3651,28 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "evmap" +version = "10.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e3ea06a83f97d3dc2eb06e51e7a729b418f0717a5558a5c870e3d5156dc558d" +dependencies = [ + "hashbag", + "slab", + "smallvec", +] + +[[package]] +name = "evmap-derive" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "332b1937705b7ed2fce76837024e9ae6f41cd2ad18a32c052de081f89982561b" +dependencies = [ + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 1.0.109", +] + [[package]] name = "executor-trait" version = "2.1.0" @@ -3488,7 +3694,7 @@ version = "0.1.0" dependencies = [ "chrono", "fakedata_generator", - "rand 0.8.5", + "rand 0.9.2", ] [[package]] @@ -3511,13 +3717,25 @@ checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" [[package]] name = "fancy-regex" -version = "0.14.0" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6215aee357f8c7c989ebb4b8466ca4d7dc93b3957039f2fc3ea2ade8ea5f279" +dependencies = [ + "bit-set", + "derivative", + "regex-automata 0.4.8", + "regex-syntax", +] + +[[package]] +name = "fancy-regex" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e24cb5a94bcae1e5408b0effca5cd7172ea3c5755049c5f3af4cd283a165298" +checksum = "bf04c5ec15464ace8355a7b440a33aece288993475556d461154d7a62ad9947c" dependencies = [ - "bit-set 0.8.0", + "bit-set", "regex-automata 0.4.8", - "regex-syntax 0.8.5", + "regex-syntax", ] [[package]] @@ -3531,9 +3749,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.1.1" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "ff" @@ -3555,16 +3773,36 @@ checksum = "a481586acf778f1b1455424c343f71124b048ffa5f4fc3f8f6ae9dc432dcb3c7" name = "file-source" version = "0.1.0" dependencies = [ - "bstr 1.11.3", - "bytes 1.9.0", + "bytes 1.10.1", "chrono", - "crc", "criterion", - "dashmap 6.1.0", + "file-source-common", "flate2", "futures 0.3.31", "glob", - "indexmap 2.7.0", + "indexmap 2.11.0", + "libc", + "quickcheck", + "similar-asserts", + "tempfile", + "tokio", + "tracing 0.1.41", + "vector-common", + "winapi", +] + +[[package]] +name = "file-source-common" +version = "0.1.0" +dependencies = [ + "bstr 1.12.0", + "bytes 1.10.1", + "chrono", + "crc", + "criterion", + "dashmap", + "flate2", + "glob", "libc", "quickcheck", "scan_fmt", @@ -3572,25 +3810,12 @@ dependencies = [ "serde_json", "similar-asserts", "tempfile", - "tokio", "tracing 0.1.41", "vector-common", "vector-config", "winapi", ] -[[package]] -name = "filetime" -version = "0.2.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4029edd3e734da6fe05b6cd7bd2960760a616bd2ddd0d59a0124746d6272af0" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall 0.3.5", - "windows-sys 0.48.0", -] - [[package]] name = "finl_unicode" version = "1.2.0" @@ -3603,20 +3828,15 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" -[[package]] -name = "flagset" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d52a7e408202050813e6f1d9addadcaafef3dca7530c7ddfb005d4081cce6779" - [[package]] name = "flate2" -version = "1.0.35" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c936bfdafb507ebbf50b8074c54fa31c5be9a1e7e5f467dd659697041407d07c" +checksum = "4a3d7db9596fecd151c5f638c0ee5d5bd487b6e0ea232e5dc96d5250f6f94b1d" dependencies = [ "crc32fast", - "miniz_oxide 0.8.0", + "libz-rs-sys", + "miniz_oxide 0.8.8", ] [[package]] @@ -3634,6 +3854,17 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "fluent-uri" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1918b65d96df47d3591bed19c5cca17e3fa5d0707318e4b5ef2eae01764df7e5" +dependencies = [ + "borrow-or-share", + "ref-cast", + "serde", +] + [[package]] name = "flume" version = "0.10.14" @@ -3690,6 +3921,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fraction" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b486ab61634f05b11b591c38c71fb25139cb55e22be4fb6ecf649cc3736c074a" +dependencies = [ + "lazy_static", + "num", +] + [[package]] name = "fsevent-sys" version = "4.1.0" @@ -3763,6 +4004,17 @@ dependencies = [ "futures-util", ] +[[package]] +name = "futures-intrusive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot 0.12.4", +] + [[package]] name = "futures-io" version = "0.3.31" @@ -3784,15 +4036,25 @@ dependencies = [ "waker-fn", ] +[[package]] +name = "futures-lite" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5edaec856126859abb19ed65f39e90fea3a9574b9707f13539acf4abf7eb532" +dependencies = [ + "futures-core", + "pin-project-lite", +] + [[package]] name = "futures-macro" version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ - "proc-macro2 1.0.93", - "quote 1.0.38", - "syn 2.0.96", + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", ] [[package]] @@ -3868,17 +4130,58 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "getrandom" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi 0.13.3+wasi-0.2.2", + "wasm-bindgen", + "windows-targets 0.52.6", +] + [[package]] name = "gimli" version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" +[[package]] +name = "git2" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2deb07a133b1520dc1a5690e9bd08950108873d7ed5de38dcc74d3b5ebffa110" +dependencies = [ + "bitflags 2.9.0", + "libc", + "libgit2-sys", + "log", + "openssl-probe", + "openssl-sys", + "url", +] + [[package]] name = "glob" -version = "0.3.2" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + +[[package]] +name = "gloo-timers" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" +checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] [[package]] name = "gloo-utils" @@ -3895,14 +4198,14 @@ dependencies = [ [[package]] name = "goauth" -version = "0.14.0" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d351469a584f3b3565e2e740d4da60839bddc4320dadd7d61da8bdd77ffb373b" +checksum = "7b1f1228623a5a37d4834f984573a01086708b109bbf0f7c2ee8d70b0c90d7a5" dependencies = [ "arc-swap", "futures 0.3.31", "log", - "reqwest 0.11.26", + "reqwest 0.12.9", "serde", "serde_derive", "serde_json", @@ -3914,22 +4217,24 @@ dependencies = [ [[package]] name = "governor" -version = "0.7.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0746aa765db78b521451ef74221663b57ba595bf83f75d0ce23cc09447c8139f" +checksum = "444405bbb1a762387aa22dd569429533b54a1d8759d35d3b64cb39b0293eaa19" dependencies = [ "cfg-if", - "dashmap 6.1.0", + "dashmap", "futures-sink", "futures-timer", "futures-util", - "no-std-compat", + "getrandom 0.3.1", + "hashbrown 0.15.2", "nonzero_ext", - "parking_lot", + "parking_lot 0.12.4", "portable-atomic", - "rand 0.8.5", + "rand 0.9.2", "smallvec", "spinning_top", + "web-time", ] [[package]] @@ -3972,8 +4277,8 @@ dependencies = [ "graphql-parser", "heck 0.4.1", "lazy_static", - "proc-macro2 1.0.93", - "quote 1.0.38", + "proc-macro2 1.0.101", + "quote 1.0.40", "serde", "serde_json", "syn 1.0.109", @@ -3986,14 +4291,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "83febfa838f898cfa73dfaa7a8eb69ff3409021ac06ee94cfb3d622f6eeb1a97" dependencies = [ "graphql_client_codegen", - "proc-macro2 1.0.93", + "proc-macro2 1.0.101", "syn 1.0.109", ] [[package]] name = "greptime-proto" version = "0.1.0" -source = "git+https://github.com/GreptimeTeam/greptime-proto.git?tag=v0.7.0#4bc0d17577dbea47396a064c1ccf229a4c9539fa" +source = "git+https://github.com/GreptimeTeam/greptime-proto.git?tag=v0.9.0#396206c2801b5a3ec51bfe8984c66b686da910e6" dependencies = [ "prost 0.12.6", "serde", @@ -4007,30 +4312,30 @@ dependencies = [ [[package]] name = "greptimedb-ingester" version = "0.1.0" -source = "git+https://github.com/GreptimeTeam/greptimedb-ingester-rust?rev=2e6b0c5eb6a5e7549c3100e4d356b07d15cce66d#2e6b0c5eb6a5e7549c3100e4d356b07d15cce66d" +source = "git+https://github.com/GreptimeTeam/greptimedb-ingester-rust?rev=f7243393808640f5123b0d5b7b798da591a4df6e#f7243393808640f5123b0d5b7b798da591a4df6e" dependencies = [ - "dashmap 5.5.3", + "dashmap", "derive_builder", "enum_dispatch", "futures 0.3.31", "futures-util", "greptime-proto", - "parking_lot", + "parking_lot 0.12.4", "prost 0.12.6", - "rand 0.8.5", - "snafu 0.7.5", + "rand 0.9.2", + "snafu 0.8.9", "tokio", "tokio-stream", "tonic 0.11.0", "tonic-build 0.9.2", - "tower", + "tower 0.4.13", ] [[package]] name = "grok" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "273797968160270573071022613fc4aa28b91fe68f3eef6c96a1b2a1947ddfbd" +checksum = "6c52724b609896f661a3f4641dd3a44dc602958ef615857c12d00756b4e9355b" dependencies = [ "glob", "onig", @@ -4053,13 +4358,13 @@ version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" dependencies = [ - "bytes 1.9.0", + "bytes 1.10.1", "fnv", "futures-core", "futures-sink", "futures-util", "http 0.2.9", - "indexmap 2.7.0", + "indexmap 2.11.0", "slab", "tokio", "tokio-util", @@ -4068,17 +4373,17 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.7" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccae279728d634d083c00f6099cb58f01cc99c145b84b8be2f6c74618d79922e" +checksum = "f3c0b69cfcb4e1b9f1bf2f53f95f766e4661169728ec61cd3fe5a0166f2d1386" dependencies = [ "atomic-waker", - "bytes 1.9.0", + "bytes 1.10.1", "fnv", "futures-core", "futures-sink", - "http 1.1.0", - "indexmap 2.7.0", + "http 1.3.1", + "indexmap 2.11.0", "slab", "tokio", "tokio-util", @@ -4097,9 +4402,15 @@ dependencies = [ [[package]] name = "hash_hasher" -version = "2.0.3" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b4b9ebce26001bad2e6366295f64e381c1e9c479109202149b9e15e154973e9" + +[[package]] +name = "hashbag" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74721d007512d0cb3338cd20f0654ac913920061a4c4d0d8708edb3f2a698c0c" +checksum = "98f494b2060b2a8f5e63379e1e487258e014cee1b1725a735816c0107a2e9d93" [[package]] name = "hashbrown" @@ -4140,6 +4451,15 @@ dependencies = [ "foldhash", ] +[[package]] +name = "hashlink" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +dependencies = [ + "hashbrown 0.15.2", +] + [[package]] name = "hdrhistogram" version = "7.5.4" @@ -4150,7 +4470,7 @@ dependencies = [ "byteorder", "crossbeam-channel", "flate2", - "nom", + "nom 7.1.3", "num-traits", ] @@ -4161,7 +4481,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06683b93020a07e3dbcf5f8c0f6d40080d725bea7936fc01ad345c01b97dc270" dependencies = [ "base64 0.21.7", - "bytes 1.9.0", + "bytes 1.10.1", "headers-core", "http 0.2.9", "httpdate", @@ -4193,7 +4513,7 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "heim" version = "0.1.0-rc.1" -source = "git+https://github.com/vectordotdev/heim.git?branch=update-nix#4925b53083587ccd695c4149836cc42e0e5e76fb" +source = "git+https://github.com/vectordotdev/heim.git?branch=update-nix#f3537d9b32e69a2a8ab19a0d42a1e6f5577a5a45" dependencies = [ "heim-common", "heim-cpu", @@ -4207,10 +4527,10 @@ dependencies = [ [[package]] name = "heim-common" version = "0.1.0-rc.1" -source = "git+https://github.com/vectordotdev/heim.git?branch=update-nix#4925b53083587ccd695c4149836cc42e0e5e76fb" +source = "git+https://github.com/vectordotdev/heim.git?branch=update-nix#f3537d9b32e69a2a8ab19a0d42a1e6f5577a5a45" dependencies = [ "cfg-if", - "core-foundation", + "core-foundation 0.9.3", "futures-core", "futures-util", "lazy_static", @@ -4225,7 +4545,7 @@ dependencies = [ [[package]] name = "heim-cpu" version = "0.1.0-rc.1" -source = "git+https://github.com/vectordotdev/heim.git?branch=update-nix#4925b53083587ccd695c4149836cc42e0e5e76fb" +source = "git+https://github.com/vectordotdev/heim.git?branch=update-nix#f3537d9b32e69a2a8ab19a0d42a1e6f5577a5a45" dependencies = [ "cfg-if", "futures 0.3.31", @@ -4243,11 +4563,11 @@ dependencies = [ [[package]] name = "heim-disk" version = "0.1.0-rc.1" -source = "git+https://github.com/vectordotdev/heim.git?branch=update-nix#4925b53083587ccd695c4149836cc42e0e5e76fb" +source = "git+https://github.com/vectordotdev/heim.git?branch=update-nix#f3537d9b32e69a2a8ab19a0d42a1e6f5577a5a45" dependencies = [ "bitflags 1.3.2", "cfg-if", - "core-foundation", + "core-foundation 0.9.3", "heim-common", "heim-runtime", "libc", @@ -4259,7 +4579,7 @@ dependencies = [ [[package]] name = "heim-host" version = "0.1.0-rc.1" -source = "git+https://github.com/vectordotdev/heim.git?branch=update-nix#4925b53083587ccd695c4149836cc42e0e5e76fb" +source = "git+https://github.com/vectordotdev/heim.git?branch=update-nix#f3537d9b32e69a2a8ab19a0d42a1e6f5577a5a45" dependencies = [ "cfg-if", "heim-common", @@ -4276,7 +4596,7 @@ dependencies = [ [[package]] name = "heim-memory" version = "0.1.0-rc.1" -source = "git+https://github.com/vectordotdev/heim.git?branch=update-nix#4925b53083587ccd695c4149836cc42e0e5e76fb" +source = "git+https://github.com/vectordotdev/heim.git?branch=update-nix#f3537d9b32e69a2a8ab19a0d42a1e6f5577a5a45" dependencies = [ "cfg-if", "heim-common", @@ -4290,7 +4610,7 @@ dependencies = [ [[package]] name = "heim-net" version = "0.1.0-rc.1" -source = "git+https://github.com/vectordotdev/heim.git?branch=update-nix#4925b53083587ccd695c4149836cc42e0e5e76fb" +source = "git+https://github.com/vectordotdev/heim.git?branch=update-nix#f3537d9b32e69a2a8ab19a0d42a1e6f5577a5a45" dependencies = [ "bitflags 1.3.2", "cfg-if", @@ -4306,7 +4626,7 @@ dependencies = [ [[package]] name = "heim-runtime" version = "0.1.0-rc.1" -source = "git+https://github.com/vectordotdev/heim.git?branch=update-nix#4925b53083587ccd695c4149836cc42e0e5e76fb" +source = "git+https://github.com/vectordotdev/heim.git?branch=update-nix#f3537d9b32e69a2a8ab19a0d42a1e6f5577a5a45" dependencies = [ "futures 0.3.31", "futures-timer", @@ -4320,6 +4640,12 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +[[package]] +name = "hermit-abi" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc" + [[package]] name = "hex" version = "0.4.3" @@ -4328,11 +4654,12 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "hickory-proto" -version = "0.24.2" +version = "0.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "447afdcdb8afb9d0a852af6dc65d9b285ce720ed7a59e42a8bf2e931c67bc1b5" +checksum = "f8a6fe56c0038198998a6f217ca4e7ef3a5e51f46163bd6dd60b5c71ca6c6502" dependencies = [ "async-trait", + "bitflags 2.9.0", "cfg-if", "data-encoding", "enum-as-inner 0.6.0", @@ -4342,8 +4669,11 @@ dependencies = [ "idna 1.0.3", "ipnet", "once_cell", - "rand 0.8.5", - "thiserror 1.0.68", + "rand 0.9.2", + "ring", + "rustls-pki-types", + "thiserror 2.0.3", + "time", "tinyvec", "tracing 0.1.41", "url", @@ -4404,18 +4734,18 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" dependencies = [ - "bytes 1.9.0", + "bytes 1.10.1", "fnv", "itoa", ] [[package]] name = "http" -version = "1.1.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" dependencies = [ - "bytes 1.9.0", + "bytes 1.10.1", "fnv", "itoa", ] @@ -4426,7 +4756,7 @@ version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" dependencies = [ - "bytes 1.9.0", + "bytes 1.10.1", "http 0.2.9", "pin-project-lite", ] @@ -4437,8 +4767,8 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" dependencies = [ - "bytes 1.9.0", - "http 1.1.0", + "bytes 1.10.1", + "http 1.3.1", ] [[package]] @@ -4447,9 +4777,9 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" dependencies = [ - "bytes 1.9.0", + "bytes 1.10.1", "futures-util", - "http 1.1.0", + "http 1.3.1", "http-body 1.0.0", "pin-project-lite", ] @@ -4477,9 +4807,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e9b187a72d63adbfba487f48095306ac823049cb504ee195541e91c7775f5ad" dependencies = [ "anyhow", - "async-channel", + "async-channel 1.9.0", "base64 0.13.1", - "futures-lite", + "futures-lite 1.13.0", "infer", "pin-project-lite", "rand 0.7.3", @@ -4492,9 +4822,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.8.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" [[package]] name = "httpdate" @@ -4504,9 +4834,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "humantime" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +checksum = "9b112acc8b3adf4b107a8ec20977da0273a8c386765a3ec0229bd500a1443f9f" [[package]] name = "hyper" @@ -4514,7 +4844,7 @@ version = "0.14.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" dependencies = [ - "bytes 1.9.0", + "bytes 1.10.1", "futures-channel", "futures-core", "futures-util", @@ -4525,7 +4855,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.5.8", + "socket2 0.5.10", "tokio", "tower-service", "tracing 0.1.41", @@ -4534,20 +4864,22 @@ dependencies = [ [[package]] name = "hyper" -version = "1.4.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05" +checksum = "eb3aa54a13a0dfe7fbe3a59e0c76093041720fdc77b110cc0fc260fafb4dc51e" dependencies = [ - "bytes 1.9.0", + "atomic-waker", + "bytes 1.10.1", "futures-channel", - "futures-util", - "h2 0.4.7", - "http 1.1.0", + "futures-core", + "h2 0.4.12", + "http 1.3.1", "http-body 1.0.0", "httparse", "httpdate", "itoa", "pin-project-lite", + "pin-utils", "smallvec", "tokio", "want", @@ -4560,7 +4892,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73b7d8abf35697b81a825e386fc151e0d503e8cb5fcb93cc8669c376dfd6f278" dependencies = [ "hex", - "hyper 1.4.1", + "hyper 1.7.0", "hyper-util", "pin-project-lite", "tokio", @@ -4580,7 +4912,7 @@ dependencies = [ "once_cell", "openssl", "openssl-sys", - "parking_lot", + "parking_lot 0.12.4", "tokio", "tokio-openssl", "tower-layer", @@ -4592,14 +4924,14 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "527d4d619ca2c2aafa31ec139a3d1d60bf557bf7578a1f20f743637eccd9ca19" dependencies = [ - "http 1.1.0", - "hyper 1.4.1", + "http 1.3.1", + "hyper 1.7.0", "hyper-util", "linked_hash_set", "once_cell", "openssl", "openssl-sys", - "parking_lot", + "parking_lot 0.12.4", "pin-project", "tower-layer", "tower-service", @@ -4611,7 +4943,7 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca815a891b24fdfb243fa3239c86154392b0953ee584aa1a2a1f66d20cbe75cc" dependencies = [ - "bytes 1.9.0", + "bytes 1.10.1", "futures 0.3.31", "headers", "http 0.2.9", @@ -4632,7 +4964,7 @@ dependencies = [ "http 0.2.9", "hyper 0.14.28", "log", - "rustls 0.21.11", + "rustls 0.21.12", "rustls-native-certs 0.6.3", "tokio", "tokio-rustls 0.24.1", @@ -4640,21 +4972,21 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.26.0" +version = "0.27.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0bea761b46ae2b24eb4aef630d8d1c398157b6fc29e6350ecf090a0b70c952c" +checksum = "2d191583f3da1305256f22463b9bb0471acad48a4e534a5218b9963e9c1f59b2" dependencies = [ "futures-util", - "http 1.1.0", - "hyper 1.4.1", + "http 1.3.1", + "hyper 1.7.0", "hyper-util", - "log", - "rustls 0.22.4", - "rustls-native-certs 0.7.0", + "rustls 0.23.23", + "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", - "tokio-rustls 0.25.0", + "tokio-rustls 0.26.2", "tower-service", + "webpki-roots 0.26.1", ] [[package]] @@ -4675,7 +5007,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3203a961e5c83b6f5498933e78b6b263e208c197b63e9c6c53cc82ffd3f63793" dependencies = [ - "hyper 1.4.1", + "hyper 1.7.0", "hyper-util", "pin-project-lite", "tokio", @@ -4688,41 +5020,57 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ - "bytes 1.9.0", + "bytes 1.10.1", "hyper 0.14.28", "native-tls", "tokio", "tokio-native-tls", ] +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes 1.10.1", + "http-body-util", + "hyper 1.7.0", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + [[package]] name = "hyper-util" version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41296eb09f183ac68eec06e03cdbea2e759633d4067b2f6552fc2e009bcad08b" dependencies = [ - "bytes 1.9.0", + "bytes 1.10.1", "futures-channel", "futures-util", - "http 1.1.0", + "http 1.3.1", "http-body 1.0.0", - "hyper 1.4.1", + "hyper 1.7.0", "pin-project-lite", - "socket2 0.5.8", + "socket2 0.5.10", "tokio", "tower-service", "tracing 0.1.41", ] [[package]] -name = "hyperlocal-next" -version = "0.9.0" +name = "hyperlocal" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acf569d43fa9848e510358c07b80f4adf34084ddc28c6a4a651ee8474c070dcc" +checksum = "986c5ce3b994526b3cd75578e62554abd09f0899d6206de48b3e96ab34ccc8c7" dependencies = [ "hex", "http-body-util", - "hyper 1.4.1", + "hyper 1.7.0", "hyper-util", "pin-project-lite", "tokio", @@ -4865,9 +5213,9 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ - "proc-macro2 1.0.93", - "quote 1.0.38", - "syn 2.0.96", + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", ] [[package]] @@ -4921,9 +5269,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.7.0" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62f822373a4fe84d4bb149bf54e584a7f4abec90e072ed49cda0edea5b95471f" +checksum = "f2481980430f9f78649238835720ddccc57e52df14ffce1c6f37391d61b563e9" dependencies = [ "equivalent", "hashbrown 0.15.2", @@ -4932,23 +5280,23 @@ dependencies = [ [[package]] name = "indicatif" -version = "0.17.9" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbf675b85ed934d3c67b5c5469701eec7db22689d0a2139d856e0925fa28b281" +checksum = "70a646d946d06bedbbc4cac4c218acf4bbf2d87757a784857025f4d447e4e1cd" dependencies = [ - "console", - "number_prefix", + "console 0.16.0", "portable-atomic", "unicode-segmentation", "unicode-width 0.2.0", + "unit-prefix", "web-time", ] [[package]] name = "indoc" -version = "2.0.5" +version = "2.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" +checksum = "f4c7245a08504955605670dbf141fceab975f15ca21570696aebe9d2e71576bd" [[package]] name = "infer" @@ -4962,20 +5310,20 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22fa7ee6be451ea0b1912b962c91c8380835e97cf1584a77e18264e908448dcb" dependencies = [ - "bytes 1.9.0", + "bytes 1.10.1", "log", - "nom", + "nom 7.1.3", "smallvec", "snafu 0.7.5", ] [[package]] name = "inotify" -version = "0.10.2" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdd168d97690d0b8c412d6b6c10360277f4d7ee495c5d0d5d5fe0854923255cc" +checksum = "f37dccff2791ab604f9babef0ba14fbe0be30bd368dc541e2b08d07c8aa908f3" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.9.0", "inotify-sys", "libc", ] @@ -5005,8 +5353,8 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b23a0c8dfe501baac4adf6ebbfa6eddf8f0c07f56b058cc1288017e32397846c" dependencies = [ - "quote 1.0.38", - "syn 2.0.96", + "quote 1.0.40", + "syn 2.0.106", ] [[package]] @@ -5016,13 +5364,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" dependencies = [ "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", ] [[package]] name = "inventory" -version = "0.3.17" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b31349d02fe60f80bbbab1a9402364cad7460626d6030494b08ac4a2075bf81" +checksum = "bc61209c082fbeb19919bee74b176221b27223e27b65d781eb91af24eb1fb46e" dependencies = [ "rustversion", ] @@ -5033,11 +5384,22 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" dependencies = [ - "hermit-abi", + "hermit-abi 0.3.9", "libc", "windows-sys 0.48.0", ] +[[package]] +name = "io-uring" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d93587f37623a1a17d94ef2bc9ada592f5465fe7732084ab7beefabe5c77c0c4" +dependencies = [ + "bitflags 2.9.0", + "cfg-if", + "libc", +] + [[package]] name = "iovec" version = "0.1.4" @@ -5053,29 +5415,26 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f" dependencies = [ - "socket2 0.5.8", + "socket2 0.5.10", "widestring 1.0.2", "windows-sys 0.48.0", - "winreg 0.50.0", + "winreg", ] [[package]] name = "ipnet" -version = "2.10.1" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" +checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" dependencies = [ "serde", ] [[package]] name = "ipnetwork" -version = "0.20.0" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf466541e9d546596ee94f9f69590f89473455f88372423e0008fc1a7daf100e" -dependencies = [ - "serde", -] +checksum = "cf370abdafd54d13e54a620e8c3e1145f28e46cc9d704bc6d94414559df41763" [[package]] name = "is-terminal" @@ -5083,7 +5442,7 @@ version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" dependencies = [ - "hermit-abi", + "hermit-abi 0.3.9", "rustix 0.38.40", "windows-sys 0.48.0", ] @@ -5169,19 +5528,20 @@ checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" [[package]] name = "jobserver" -version = "0.1.27" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c37f63953c4c63420ed5fd3d6d398c719489b9f872b9fa683262f8edd363c7d" +checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" dependencies = [ "libc", ] [[package]] name = "js-sys" -version = "0.3.65" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54c0c35952f67de54bb584e9fd912b3023117cbafc0a77d8f3dee1fb5f572fe8" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" dependencies = [ + "once_cell", "wasm-bindgen", ] @@ -5218,9 +5578,35 @@ version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c6e529149475ca0b2820835d3dce8fcc41c6b943ca608d32f35b449255e4627" dependencies = [ - "fluent-uri", + "fluent-uri 0.1.4", + "serde", + "serde_json", +] + +[[package]] +name = "jsonschema" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24690c68dfcdde5980d676b0f1820981841016b1f29eecb4c42ad48ab4118681" +dependencies = [ + "ahash 0.8.11", + "base64 0.22.1", + "bytecount", + "email_address", + "fancy-regex 0.16.1", + "fraction", + "idna 1.0.3", + "itoa", + "num-cmp", + "num-traits", + "once_cell", + "percent-encoding", + "referencing", + "regex", + "regex-syntax", "serde", "serde_json", + "uuid-simd", ] [[package]] @@ -5232,7 +5618,7 @@ dependencies = [ "indoc", "k8s-openapi 0.16.0", "k8s-test-framework", - "rand 0.8.5", + "rand 0.9.2", "regex", "reqwest 0.11.26", "serde_json", @@ -5247,7 +5633,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d9455388f4977de4d0934efa9f7d36296295537d774574113a20f6082de03da" dependencies = [ "base64 0.13.1", - "bytes 1.9.0", + "bytes 1.10.1", "chrono", "serde", "serde-value", @@ -5289,9 +5675,9 @@ dependencies = [ [[package]] name = "kqueue" -version = "1.0.8" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7447f1ca1b7b563588a205fe93dea8df60fd981423a768bc1c0ded35ed147d0c" +checksum = "eac30106d7dce88daf4a3fcb4879ea939476d5074a9b7ddd0fb97fa4bed5596a" dependencies = [ "kqueue-sys", "libc", @@ -5335,15 +5721,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d81336eb3a5b10a40c97a5a97ad66622e92bad942ce05ee789edd730aa4f8603" dependencies = [ "base64 0.22.1", - "bytes 1.9.0", + "bytes 1.10.1", "chrono", "either", "futures 0.3.31", "home", - "http 1.1.0", + "http 1.3.1", "http-body 1.0.0", "http-body-util", - "hyper 1.4.1", + "hyper 1.7.0", "hyper-openssl 0.10.2", "hyper-timeout 0.5.1", "hyper-util", @@ -5355,11 +5741,11 @@ dependencies = [ "secrecy", "serde", "serde_json", - "serde_yaml 0.9.34+deprecated", + "serde_yaml", "thiserror 1.0.68", "tokio", "tokio-util", - "tower", + "tower 0.4.13", "tower-http 0.5.2", "tracing 0.1.41", ] @@ -5372,7 +5758,7 @@ checksum = "cce373a74d787d439063cdefab0f3672860bd7bac01a38e39019177e764a0fe6" dependencies = [ "chrono", "form_urlencoded", - "http 1.1.0", + "http 1.3.1", "json-patch", "k8s-openapi 0.22.0", "serde", @@ -5398,7 +5784,7 @@ dependencies = [ "jsonptr", "k8s-openapi 0.22.0", "kube-client", - "parking_lot", + "parking_lot 0.12.4", "pin-project", "serde", "serde_json", @@ -5410,32 +5796,25 @@ dependencies = [ [[package]] name = "lalrpop" -version = "0.20.0" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da4081d44f4611b66c6dd725e6de3169f9f63905421e8626fcb86b6a898998b8" +checksum = "06093b57658c723a21da679530e061a8c25340fa5a6f98e313b542268c7e2a1f" dependencies = [ "ascii-canvas", - "bit-set 0.5.3", - "diff", + "bit-set", "ena", - "is-terminal", - "itertools 0.10.5", - "lalrpop-util 0.20.0", + "itertools 0.13.0", + "lalrpop-util", "petgraph", "regex", - "regex-syntax 0.7.5", + "regex-syntax", + "sha3", "string_cache", - "term", - "tiny-keccak", + "term 1.0.1", "unicode-xid 0.2.4", + "walkdir", ] -[[package]] -name = "lalrpop-util" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f35c735096c0293d313e8f2a641627472b83d01b937177fe76e5e2708d31e0d" - [[package]] name = "lalrpop-util" version = "0.22.0" @@ -5448,9 +5827,9 @@ dependencies = [ [[package]] name = "lapin" -version = "2.5.0" +version = "2.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "209b09a06f4bd4952a0fd0594f90d53cf4496b062f59acc838a2823e1bb7d95c" +checksum = "4273975142078ed200dedd77f09c8903dec110d0b02a0c8ad45796b39b691ea9" dependencies = [ "amq-protocol", "async-global-executor-trait", @@ -5460,7 +5839,7 @@ dependencies = [ "flume 0.11.0", "futures-core", "futures-io", - "parking_lot", + "parking_lot 0.12.4", "pinky-swear", "reactor-trait", "serde", @@ -5479,9 +5858,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.169" +version = "0.2.175" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" +checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543" [[package]] name = "libflate" @@ -5507,17 +5886,74 @@ dependencies = [ "rle-decode-fast", ] +[[package]] +name = "libgit2-sys" +version = "0.18.2+1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c42fe03df2bd3c53a3a9c7317ad91d80c81cd1fb0caec8d7cc4cd2bfa10c222" +dependencies = [ + "cc", + "libc", + "libssh2-sys", + "libz-sys", + "openssl-sys", + "pkg-config", +] + [[package]] name = "libm" version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" +[[package]] +name = "libredox" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +dependencies = [ + "bitflags 2.9.0", + "libc", +] + +[[package]] +name = "libsqlite3-sys" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" +dependencies = [ + "pkg-config", + "vcpkg", +] + +[[package]] +name = "libssh2-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "220e4f05ad4a218192533b300327f5150e809b54c4ec83b5a1d91833601811b9" +dependencies = [ + "cc", + "libc", + "libz-sys", + "openssl-sys", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "libz-rs-sys" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "172a788537a2221661b480fee8dc5f96c580eb34fa88764d3205dc356c7e4221" +dependencies = [ + "zlib-rs", +] + [[package]] name = "libz-sys" -version = "1.1.12" +version = "1.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d97137b25e321a73eef1418d1d5d2eda4d77e12813f8e6dead84bc52c5870a7b" +checksum = "8b70e7a7df205e92a1a4cd9aaae7898dac0aa555503cc0a649494d0d60e7651d" dependencies = [ "cc", "libc", @@ -5552,11 +5988,17 @@ version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +[[package]] +name = "linux-raw-sys" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db9c683daf087dc577b7506e9695b3d556a9f3849903fa28186283afd6809e9" + [[package]] name = "listenfd" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0500463acd96259d219abb05dc57e5a076ef04b2db9a2112846929b5f174c96" +checksum = "b87bc54a4629b4294d0b3ef041b64c40c611097a677d9dc07b2c67739fe39dba" dependencies = [ "libc", "uuid", @@ -5569,11 +6011,17 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704" +[[package]] +name = "litrs" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ce301924b7887e9d637144fdade93f9dfff9b60981d4ac161db09720d39aa5" + [[package]] name = "lock_api" -version = "0.4.11" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +checksum = "96936507f153605bddfcda068dd804796c84324ed2510809e5b2a624c81da765" dependencies = [ "autocfg", "scopeguard", @@ -5587,9 +6035,9 @@ checksum = "3a69c0481fc2424cb55795de7da41add33372ea75a94f9b6588ab6a2826dfebc" [[package]] name = "log" -version = "0.4.22" +version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" [[package]] name = "loki-logproto" @@ -5610,6 +6058,12 @@ dependencies = [ "hashbrown 0.15.2", ] +[[package]] +name = "lru" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86ea4e65087ff52f3862caff188d489f1fab49a0cb09e01b2e3f1a617b10aaed" + [[package]] name = "lru-cache" version = "0.1.2" @@ -5640,24 +6094,32 @@ dependencies = [ [[package]] name = "lz4" -version = "1.24.0" +version = "1.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e9e2dd86df36ce760a60f6ff6ad526f7ba1f14ba0356f8254fb6905e6494df1" +checksum = "a20b523e860d03443e98350ceaac5e71c6ba89aea7d960769ec3ce37f4de5af4" dependencies = [ - "libc", "lz4-sys", ] [[package]] name = "lz4-sys" -version = "1.9.4" +version = "1.11.1+lz4-1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57d27b317e207b10f69f5e75494119e391a96f48861ae870d1da6edac98ca900" +checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6" dependencies = [ "cc", "libc", ] +[[package]] +name = "lz4_flex" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08ab2867e3eeeca90e844d1940eab391c9dc5228783db2ed999acbc0a9ed375a" +dependencies = [ + "twox-hash", +] + [[package]] name = "macaddr" version = "1.0.1" @@ -5690,11 +6152,11 @@ checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" [[package]] name = "matchers" -version = "0.1.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" dependencies = [ - "regex-automata 0.1.10", + "regex-automata 0.4.8", ] [[package]] @@ -5721,14 +6183,16 @@ dependencies = [ [[package]] name = "maxminddb" -version = "0.24.0" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6087e5d8ea14861bb7c7f573afbc7be3798d3ef0fae87ec4fd9a4de9a127c3c" +checksum = "2a197e44322788858682406c74b0b59bf8d9b4954fe1f224d9a25147f1880bba" dependencies = [ "ipnetwork", "log", "memchr", "serde", + "simdutf8", + "thiserror 2.0.3", ] [[package]] @@ -5743,15 +6207,15 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.4" +version = "2.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" [[package]] name = "memmap2" -version = "0.9.5" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd3f7eed9d3848f8b98834af67102b720745c4ec028fcd0aa0239277e7de374f" +checksum = "843a98750cd611cc2965a8213b53b43e715f13c37a9e096c6408e69990961db7" dependencies = [ "libc", ] @@ -5785,9 +6249,9 @@ dependencies = [ [[package]] name = "metrics" -version = "0.24.1" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a7deb012b3b2767169ff203fadb4c6b0b82b947512e5eb9e0b78c2e186ad9e3" +checksum = "25dea7ac8057892855ec285c440160265225438c3c45072613c25a4b26e98ef5" dependencies = [ "ahash 0.8.11", "portable-atomic", @@ -5799,7 +6263,7 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1ada651cd6bdffe01e5f35067df53491f1fe853d2b154008ca2bd30b3d3fcf6" dependencies = [ - "indexmap 2.7.0", + "indexmap 2.11.0", "itoa", "lockfree-object-pool", "metrics", @@ -5820,7 +6284,7 @@ dependencies = [ "crossbeam-epoch", "crossbeam-utils", "hashbrown 0.15.2", - "indexmap 2.7.0", + "indexmap 2.11.0", "metrics", "ordered-float 4.6.0", "quanta", @@ -5861,9 +6325,9 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.8.0" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +checksum = "3be647b768db090acb35d5ec5db2b0e1f1de11133ca123b9eacf5137868f892a" dependencies = [ "adler2", ] @@ -5874,7 +6338,7 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4569e456d394deccd22ce1c1913e6ea0e54519f577285001215d33557431afe4" dependencies = [ - "hermit-abi", + "hermit-abi 0.3.9", "libc", "log", "wasi 0.11.0+wasi-snapshot-preview1", @@ -5883,24 +6347,25 @@ dependencies = [ [[package]] name = "mlua" -version = "0.10.2" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ea43c3ffac2d0798bd7128815212dd78c98316b299b7a902dabef13dc7b6b8d" +checksum = "c1f5f8fbebc7db5f671671134b9321c4b9aa9adeafccfd9a8c020ae45c6a35d0" dependencies = [ - "bstr 1.11.3", + "bstr 1.12.0", "either", "mlua-sys", "mlua_derive", "num-traits", - "parking_lot", + "parking_lot 0.12.4", "rustc-hash", + "rustversion", ] [[package]] name = "mlua-sys" -version = "0.6.6" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63a11d485edf0f3f04a508615d36c7d50d299cf61a7ee6d3e2530651e0a31771" +checksum = "380c1f7e2099cafcf40e51d3a9f20a346977587aa4d012eae1f043149a728a93" dependencies = [ "cc", "cfg-if", @@ -5918,17 +6383,17 @@ dependencies = [ "itertools 0.13.0", "once_cell", "proc-macro-error2", - "proc-macro2 1.0.93", - "quote 1.0.38", + "proc-macro2 1.0.101", + "quote 1.0.40", "regex", - "syn 2.0.96", + "syn 2.0.106", ] [[package]] name = "mock_instant" -version = "0.5.2" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15dce1a197fd6e12d773d7e4e7e1681fa5de6eb689ecaae824dc3cd1f643e7dc" +checksum = "dce6dd36094cac388f119d2e9dc82dc730ef91c32a6222170d630e5414b956e6" [[package]] name = "moka" @@ -5944,7 +6409,7 @@ dependencies = [ "event-listener 5.3.1", "futures-util", "once_cell", - "parking_lot", + "parking_lot 0.12.4", "quanta", "rustc_version 0.4.1", "smallvec", @@ -5979,7 +6444,7 @@ dependencies = [ "percent-encoding", "rand 0.8.5", "rustc_version_runtime", - "rustls 0.21.11", + "rustls 0.21.12", "rustls-pemfile 1.0.4", "serde", "serde_bytes", @@ -6007,10 +6472,10 @@ version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a15d522be0a9c3e46fd2632e272d178f56387bdb5c9fbb3a36c649062e9b5219" dependencies = [ - "bytes 1.9.0", + "bytes 1.10.1", "encoding_rs", "futures-util", - "http 1.1.0", + "http 1.3.1", "httparse", "log", "memchr", @@ -6027,18 +6492,17 @@ checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" [[package]] name = "native-tls" -version = "0.2.11" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" dependencies = [ - "lazy_static", "libc", "log", "openssl", "openssl-probe", "openssl-sys", "schannel", - "security-framework", + "security-framework 2.10.0", "security-framework-sys", "tempfile", ] @@ -6064,7 +6528,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17ebbe97acce52d06aebed4cd4a87c0941f4b2519b59b82b4feb5bd0ce003dfd" dependencies = [ - "indexmap 2.7.0", + "indexmap 2.11.0", "itertools 0.13.0", "ndarray", "noisy_float", @@ -6123,7 +6587,7 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16c903aa70590cb93691bf97a767c8d1d6122d2cc9070433deb3bbf36ce8bd23" dependencies = [ - "bytes 1.9.0", + "bytes 1.10.1", "futures 0.3.31", "libc", "log", @@ -6136,6 +6600,15 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4a24736216ec316047a1fc4252e27dabb04218aa4a3f37c6e7ddbf1f9782b54" +[[package]] +name = "newtype-uuid" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee3224f0e8be7c2a1ebc77ef9c3eecb90f55c6594399ee825de964526b3c9056" +dependencies = [ + "uuid", +] + [[package]] name = "nibble_vec" version = "0.1.0" @@ -6161,7 +6634,7 @@ dependencies = [ [[package]] name = "nix" version = "0.26.2" -source = "git+https://github.com/vectordotdev/nix.git?branch=memfd/gnu/musl#6c53a918d2d5bf4307fd60a19d9e10913ae71eeb" +source = "git+https://github.com/vectordotdev/nix.git?branch=memfd%2Fgnu%2Fmusl#6c53a918d2d5bf4307fd60a19d9e10913ae71eeb" dependencies = [ "bitflags 1.3.2", "cfg-if", @@ -6172,21 +6645,21 @@ dependencies = [ [[package]] name = "nix" -version = "0.29.0" +version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" +checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.9.0", "cfg-if", - "cfg_aliases 0.2.1", + "cfg_aliases", "libc", ] [[package]] name = "nkeys" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f49e787f4c61cbd0f9320b31cc26e58719f6aa5068e34697dd3aea361412fe3" +checksum = "879011babc47a1c7fdf5a935ae3cfe94f34645ca0cac1c7f6424b36fc743d1bf" dependencies = [ "data-encoding", "ed25519", @@ -6199,19 +6672,19 @@ dependencies = [ [[package]] name = "no-proxy" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d688d11967f7f52e273fb8f8f07ecb094254fed630e22a0cab60cc98047a5db" +checksum = "9f79c902b31ceac6856e262af5dbaffef75390cf4647c9fef7b55da69a4b912e" dependencies = [ - "cidr-utils", + "cidr", "serde", ] [[package]] -name = "no-std-compat" -version = "0.4.1" +name = "nohash" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b93853da6d84c2e3c7d730d6473e8817692dd89be387eb01b94d7f108ecb5b8c" +checksum = "a0f889fb66f7acdf83442c35775764b51fed3c606ab9cee51500dbde2cf528ca" [[package]] name = "noisy_float" @@ -6232,6 +6705,24 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + +[[package]] +name = "nom-language" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2de2bc5b451bfedaef92c90b8939a8fff5770bdcc1fafd6239d086aab8fa6b29" +dependencies = [ + "nom 8.0.0", +] + [[package]] name = "nonzero_ext" version = "0.3.0" @@ -6240,12 +6731,11 @@ checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21" [[package]] name = "notify" -version = "7.0.0" +version = "8.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c533b4c39709f9ba5005d8002048266593c1cfaf3c5f0739d5b8ab0c6c504009" +checksum = "4d3d07927151ff8575b7087f245456e549fea62edf0ec4e565a5ee50c8402bc3" dependencies = [ - "bitflags 2.6.0", - "filetime", + "bitflags 2.9.0", "fsevent-sys", "inotify", "kqueue", @@ -6254,17 +6744,14 @@ dependencies = [ "mio", "notify-types", "walkdir", - "windows-sys 0.52.0", + "windows-sys 0.60.2", ] [[package]] name = "notify-types" -version = "1.0.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7393c226621f817964ffb3dc5704f9509e107a8b024b489cc2c1b217378785df" -dependencies = [ - "instant", -] +checksum = "5e0826a989adedc2a244799e823aece04662b66609d96af8dff7ac6df9a8925d" [[package]] name = "ntapi" @@ -6285,12 +6772,11 @@ dependencies = [ [[package]] name = "nu-ansi-term" -version = "0.46.0" +version = "0.50.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +checksum = "d4a28e057d01f97e61255210fcff094d74ed0466038633e95017f5beb68e4399" dependencies = [ - "overload", - "winapi", + "windows-sys 0.52.0", ] [[package]] @@ -6302,13 +6788,26 @@ dependencies = [ "rand 0.8.5", ] +[[package]] +name = "num" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b05180d69e3da0e530ba2a1dae5110317e49e3b7f3d41be227dc5f92e49ee7af" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational 0.4.2", + "num-traits", +] + [[package]] name = "num-bigint" -version = "0.4.4" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" dependencies = [ - "autocfg", "num-integer", "num-traits", ] @@ -6330,6 +6829,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "num-cmp" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63335b2e2c34fae2fb0aa2cecfd9f0832a1e24b3b32ecec612c3426d46dc8aaa" + [[package]] name = "num-complex" version = "0.4.4" @@ -6358,11 +6863,10 @@ dependencies = [ [[package]] name = "num-integer" -version = "0.1.45" +version = "0.1.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" dependencies = [ - "autocfg", "num-traits", ] @@ -6388,6 +6892,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -6404,7 +6919,7 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi", + "hermit-abi 0.3.9", "libc", ] @@ -6433,9 +6948,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96667db765a921f7b295ffee8b60472b686a51d4f21c2ee4ffdb94c7013b65a6" dependencies = [ "proc-macro-crate 1.3.1", - "proc-macro2 1.0.93", - "quote 1.0.38", - "syn 2.0.96", + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", ] [[package]] @@ -6445,9 +6960,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af1844ef2428cc3e1cb900be36181049ef3d3193c63e43026cfe202983b27a56" dependencies = [ "proc-macro-crate 3.2.0", - "proc-macro2 1.0.93", - "quote 1.0.38", - "syn 2.0.96", + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", ] [[package]] @@ -6494,6 +7009,25 @@ dependencies = [ "malloc_buf", ] +[[package]] +name = "objc2-core-foundation" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c10c2894a6fed806ade6027bcd50662746363a9589d3ec9d9bef30a4e4bc166" +dependencies = [ + "bitflags 2.9.0", +] + +[[package]] +name = "objc2-io-kit" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71c1c64d6120e51cd86033f67176b1cb66780c2efe34dec55176f77befd93c0a" +dependencies = [ + "libc", + "objc2-core-foundation", +] + [[package]] name = "object" version = "0.32.1" @@ -6509,7 +7043,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "126c3ca37c9c44cec575247f43a3e4374d8927684f129d2beeb0d2cef262fe12" dependencies = [ - "bytes 1.9.0", + "bytes 1.10.1", "serde", "smallvec", ] @@ -6525,17 +7059,21 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.20.2" +version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +dependencies = [ + "critical-section", + "portable-atomic", +] [[package]] name = "onig" -version = "6.4.0" +version = "6.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c4b31c8722ad9171c6d77d3557db078cab2bd50afcc9d09c8b315c59df8ca4f" +checksum = "336b9c63443aceef14bea841b899035ae3abe89b7c486aaf4c5bd8aafedac3f0" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.9.0", "libc", "once_cell", "onig_sys", @@ -6543,9 +7081,9 @@ dependencies = [ [[package]] name = "onig_sys" -version = "69.8.1" +version = "69.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b829e3d7e9cc74c7e315ee8edb185bf4190da5acde74afd7fc59c35b1f086e7" +checksum = "c7f86c6eef3d6df15f23bcfb6af487cbd2fed4e5581d58d5bf1f5f8b7f6727dc" dependencies = [ "cc", "pkg-config", @@ -6565,26 +7103,24 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "opendal" -version = "0.45.1" +version = "0.54.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52c17c077f23fa2d2c25d9d22af98baa43b8bbe2ef0de80cf66339aa70401467" +checksum = "ffb9838d0575c6dbaf3fcec7255af8d5771996d4af900bbb6fa9a314dec00a1a" dependencies = [ "anyhow", - "async-trait", "backon", - "base64 0.21.7", - "bytes 1.9.0", + "base64 0.22.1", + "bytes 1.10.1", "chrono", - "flagset", "futures 0.3.31", "getrandom 0.2.15", - "http 0.2.9", + "http 1.3.1", + "http-body 1.0.0", "log", "md-5", - "once_cell", "percent-encoding", - "quick-xml", - "reqwest 0.11.26", + "quick-xml 0.37.4", + "reqwest 0.12.9", "serde", "serde_json", "tokio", @@ -6593,9 +7129,9 @@ dependencies = [ [[package]] name = "openidconnect" -version = "3.4.0" +version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62d6050f6a84b81f23c569f5607ad883293e57491036e318fafe6fc4895fadb1" +checksum = "f47e80a9cfae4462dd29c41e987edd228971d6565553fbc14b8a11e666d91590" dependencies = [ "base64 0.13.1", "chrono", @@ -6616,7 +7152,7 @@ dependencies = [ "serde_json", "serde_path_to_error", "serde_plain", - "serde_with 3.12.0", + "serde_with 3.14.0", "sha2", "subtle", "thiserror 1.0.68", @@ -6625,11 +7161,11 @@ dependencies = [ [[package]] name = "openssl" -version = "0.10.68" +version = "0.10.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6174bc48f102d208783c2c84bf931bb75927a617866870de8a4ea85597f871f5" +checksum = "8505734d46c8ab1e19a1dce3aef597ad87dcb4c37e7188231769bd6bd51cebf8" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.9.0", "cfg-if", "foreign-types", "libc", @@ -6644,31 +7180,31 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ - "proc-macro2 1.0.93", - "quote 1.0.38", - "syn 2.0.96", + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", ] [[package]] name = "openssl-probe" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" [[package]] name = "openssl-src" -version = "300.4.1+3.4.0" +version = "300.5.2+3.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faa4eac4138c62414b5622d1b31c5c304f34b406b013c079c2bbc652fdd6678c" +checksum = "d270b79e2926f5150189d475bc7e9d2c69f9c4697b185fa917d5a32b792d21b4" dependencies = [ "cc", ] [[package]] name = "openssl-sys" -version = "0.9.104" +version = "0.9.109" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45abf306cbf99debc8195b66b7346498d7b10c210de50418b5ccd7ceba08c741" +checksum = "90096e2e47630d78b7d1c20952dc621f957103f8bc2c8359ec81290d75238571" dependencies = [ "cc", "libc", @@ -6681,8 +7217,9 @@ dependencies = [ name = "opentelemetry-proto" version = "0.1.0" dependencies = [ - "bytes 1.9.0", + "bytes 1.10.1", "chrono", + "glob", "hex", "ordered-float 4.6.0", "prost 0.12.6", @@ -6734,17 +7271,11 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4030760ffd992bef45b0ae3f10ce1aba99e33464c90d14dd7c039884963ddc7a" -[[package]] -name = "overload" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" - [[package]] name = "owo-colors" -version = "4.1.0" +version = "4.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb37767f6569cd834a413442455e0f066d0d522de8630436e2a1761d9726ba56" +checksum = "48dd4f4a2c8405440fd0462561f0e5806bd0f77e86f51c761481bdd4018b545e" dependencies = [ "supports-color 2.1.0", "supports-color 3.0.1", @@ -6791,41 +7322,57 @@ checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" [[package]] name = "parking_lot" -version = "0.12.3" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core 0.8.6", +] + +[[package]] +name = "parking_lot" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" +checksum = "70d58bf43669b5795d1576d0641cfb6fbb2057bf629506267a92807158584a13" dependencies = [ "lock_api", - "parking_lot_core", + "parking_lot_core 0.9.11", ] [[package]] name = "parking_lot_core" -version = "0.9.9" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" +checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" dependencies = [ "cfg-if", + "instant", "libc", - "redox_syscall 0.4.1", + "redox_syscall 0.2.16", "smallvec", - "windows-targets 0.48.5", + "winapi", ] [[package]] -name = "parse-size" -version = "1.1.0" +name = "parking_lot_core" +version = "0.9.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "487f2ccd1e17ce8c1bfab3a65c89525af41cfad4c8659021a1e9a2aacd73b89b" +checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.5.12", + "smallvec", + "windows-targets 0.52.6", +] [[package]] -name = "parse-zoneinfo" -version = "0.3.0" +name = "parse-size" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c705f256449c60da65e11ff6626e0c16a0a0b96aaa348de61376b249bc340f41" -dependencies = [ - "regex", -] +checksum = "487f2ccd1e17ce8c1bfab3a65c89525af41cfad4c8659021a1e9a2aacd73b89b" [[package]] name = "passt" @@ -6856,11 +7403,11 @@ checksum = "9e9ed2178b0575fff8e1b83b58ba6f75e727aafac2e1b6c795169ad3b17eb518" [[package]] name = "pem" -version = "3.0.2" +version = "3.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3163d2912b7c3b52d651a055f2c7eec9ba5cd22d26ef75b8dd3a59980b185923" +checksum = "38af38e8470ac9dee3ce1bae1af9c1671fffc44ddfd8bd1d0a3445bf349a8ef3" dependencies = [ - "base64 0.21.7", + "base64 0.22.1", "serde", ] @@ -6875,9 +7422,9 @@ dependencies = [ [[package]] name = "percent-encoding" -version = "2.3.1" +version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pest" @@ -6908,9 +7455,9 @@ checksum = "1381c29a877c6d34b8c176e734f35d7f7f5b3adaefe940cb4d1bb7af94678e2e" dependencies = [ "pest", "pest_meta", - "proc-macro2 1.0.93", - "quote 1.0.38", - "syn 2.0.96", + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", ] [[package]] @@ -6931,7 +7478,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" dependencies = [ "fixedbitset", - "indexmap 2.7.0", + "indexmap 2.11.0", ] [[package]] @@ -6944,61 +7491,59 @@ dependencies = [ ] [[package]] -name = "phf_codegen" -version = "0.11.2" +name = "phf" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8d39688d359e6b34654d328e262234662d16cc0f60ec8dcbe5e718709342a5a" +checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" dependencies = [ - "phf_generator", - "phf_shared 0.11.2", + "phf_shared 0.12.1", ] [[package]] -name = "phf_generator" -version = "0.11.2" +name = "phf_shared" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" +checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096" dependencies = [ - "phf_shared 0.11.2", - "rand 0.8.5", + "siphasher 0.3.11", ] [[package]] name = "phf_shared" -version = "0.10.0" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096" +checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" dependencies = [ - "siphasher", + "siphasher 0.3.11", ] [[package]] name = "phf_shared" -version = "0.11.2" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" +checksum = "06005508882fb681fd97892ecff4b7fd0fee13ef1aa569f8695dae7ab9099981" dependencies = [ - "siphasher", + "siphasher 1.0.1", ] [[package]] name = "pin-project" -version = "1.1.8" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e2ec53ad785f4d35dac0adea7f7dc6f1bb277ad84a680c7afefeae05d1f5916" +checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.8" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d56a66c0c55993aa927429d0f8a0abfd74f084e4d9c192cffed01e418d83eefb" +checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ - "proc-macro2 1.0.93", - "quote 1.0.38", - "syn 2.0.96", + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", ] [[package]] @@ -7021,7 +7566,7 @@ checksum = "d894b67aa7a4bf295db5e85349078c604edaa6fa5c8721e8eca3c7729a27f2ac" dependencies = [ "doc-comment", "flume 0.10.14", - "parking_lot", + "parking_lot 0.12.4", "tracing 0.1.41", ] @@ -7032,7 +7577,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "668d31b1c4eba19242f2088b2bf3316b82ca31082a8335764db4e083db7485d4" dependencies = [ "atomic-waker", - "fastrand 2.1.1", + "fastrand 2.3.0", "futures-io", ] @@ -7115,16 +7660,17 @@ dependencies = [ [[package]] name = "polling" -version = "3.3.0" +version = "3.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e53b6af1f60f36f8c2ac2aad5459d75a5a9b4be1e8cdd40264f315d78193e531" +checksum = "a604568c3202727d1507653cb121dbd627a58684eb09a820fd746bee38b4442f" dependencies = [ "cfg-if", "concurrent-queue", + "hermit-abi 0.4.0", "pin-project-lite", "rustix 0.38.40", "tracing 0.1.41", - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] @@ -7140,9 +7686,9 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.6.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" +checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e" [[package]] name = "portable-atomic-util" @@ -7157,16 +7703,15 @@ dependencies = [ name = "portpicker" version = "1.0.0" dependencies = [ - "rand 0.8.5", + "rand 0.9.2", ] [[package]] name = "postgres-openssl" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1de0ea6504e07ca78355a6fb88ad0f36cafe9e696cbc6717f16a207f3a60be72" +checksum = "fb14e4bbc2c0b3d165bf30b79c7a9c10412dff9d98491ffdd64ed810ab891d21" dependencies = [ - "futures 0.3.31", "openssl", "tokio", "tokio-openssl", @@ -7175,29 +7720,29 @@ dependencies = [ [[package]] name = "postgres-protocol" -version = "0.6.7" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acda0ebdebc28befa84bee35e651e4c5f09073d668c7aed4cf7e23c3cda84b23" +checksum = "76ff0abab4a9b844b93ef7b81f1efc0a366062aaef2cd702c76256b5dc075c54" dependencies = [ "base64 0.22.1", "byteorder", - "bytes 1.9.0", + "bytes 1.10.1", "fallible-iterator", "hmac", "md-5", "memchr", - "rand 0.8.5", + "rand 0.9.2", "sha2", "stringprep", ] [[package]] name = "postgres-types" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f66ea23a2d0e5734297357705193335e0a957696f34bed2f2faefacb2fec336f" +checksum = "613283563cd90e1dfc3518d548caee47e0e725455ed619881f5cf21f36de4b48" dependencies = [ - "bytes 1.9.0", + "bytes 1.10.1", "chrono", "fallible-iterator", "postgres-protocol", @@ -7265,7 +7810,7 @@ version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86" dependencies = [ - "proc-macro2 1.0.93", + "proc-macro2 1.0.101", "syn 1.0.109", ] @@ -7275,8 +7820,8 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" dependencies = [ - "proc-macro2 1.0.93", - "syn 2.0.96", + "proc-macro2 1.0.101", + "syn 2.0.106", ] [[package]] @@ -7288,7 +7833,7 @@ dependencies = [ "encode_unicode 1.0.0", "is-terminal", "lazy_static", - "term", + "term 0.7.0", "unicode-width 0.1.13", ] @@ -7311,45 +7856,13 @@ dependencies = [ "toml_edit 0.19.15", ] -[[package]] -name = "proc-macro-crate" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e8366a6159044a37876a2b9817124296703c586a5c92e2c53751fa06d8d43e8" -dependencies = [ - "toml_edit 0.20.7", -] - [[package]] name = "proc-macro-crate" version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecf48c7ca261d60b74ab1a7b20da18bede46776b2e55535cb958eb595c5fa7b" dependencies = [ - "toml_edit 0.22.20", -] - -[[package]] -name = "proc-macro-error" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" -dependencies = [ - "proc-macro-error-attr", - "proc-macro2 1.0.93", - "quote 1.0.38", - "version_check", -] - -[[package]] -name = "proc-macro-error-attr" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" -dependencies = [ - "proc-macro2 1.0.93", - "quote 1.0.38", - "version_check", + "toml_edit 0.22.27", ] [[package]] @@ -7358,8 +7871,8 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" dependencies = [ - "proc-macro2 1.0.93", - "quote 1.0.38", + "proc-macro2 1.0.101", + "quote 1.0.40", ] [[package]] @@ -7369,9 +7882,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" dependencies = [ "proc-macro-error-attr2", - "proc-macro2 1.0.93", - "quote 1.0.38", - "syn 2.0.96", + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", ] [[package]] @@ -7397,9 +7910,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.93" +version = "1.0.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99" +checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de" dependencies = [ "unicode-ident", ] @@ -7408,30 +7921,30 @@ dependencies = [ name = "prometheus-parser" version = "0.1.0" dependencies = [ - "indexmap 2.7.0", - "nom", + "indexmap 2.11.0", + "nom 8.0.0", "prost 0.12.6", "prost-build 0.12.6", "prost-types 0.12.6", - "snafu 0.7.5", + "snafu 0.8.9", "vector-common", ] [[package]] name = "proptest" -version = "1.5.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4c2511913b88df1637da85cc8d96ec8e43a3f8bb8ccb71ee1ac240d6f3df58d" +checksum = "6fcdab19deb5195a31cf7726a210015ff1496ba1464fd42cb4f537b8b01b471f" dependencies = [ - "bit-set 0.5.3", - "bit-vec 0.6.3", - "bitflags 2.6.0", + "bit-set", + "bit-vec", + "bitflags 2.9.0", "lazy_static", "num-traits", - "rand 0.8.5", - "rand_chacha 0.3.1", + "rand 0.9.2", + "rand_chacha 0.9.0", "rand_xorshift", - "regex-syntax 0.8.5", + "regex-syntax", "rusty-fork", "tempfile", "unarray", @@ -7439,13 +7952,13 @@ dependencies = [ [[package]] name = "proptest-derive" -version = "0.5.1" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ee1c9ac207483d5e7db4940700de86a9aae46ef90c48b57f99fe7edb8345e49" +checksum = "095a99f75c69734802359b682be8daaf8980296731f6470434ea2c652af1dd30" dependencies = [ - "proc-macro2 1.0.93", - "quote 1.0.38", - "syn 2.0.96", + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", ] [[package]] @@ -7454,7 +7967,7 @@ version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" dependencies = [ - "bytes 1.9.0", + "bytes 1.10.1", "prost-derive 0.11.9", ] @@ -7464,18 +7977,18 @@ version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29" dependencies = [ - "bytes 1.9.0", + "bytes 1.10.1", "prost-derive 0.12.6", ] [[package]] name = "prost" -version = "0.13.3" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b0487d90e047de87f984913713b85c601c05609aad5b0df4b4573fbf69aa13f" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" dependencies = [ - "bytes 1.9.0", - "prost-derive 0.13.3", + "bytes 1.10.1", + "prost-derive 0.13.5", ] [[package]] @@ -7484,7 +7997,7 @@ version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" dependencies = [ - "bytes 1.9.0", + "bytes 1.10.1", "heck 0.4.1", "itertools 0.10.5", "lazy_static", @@ -7506,7 +8019,7 @@ version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4" dependencies = [ - "bytes 1.9.0", + "bytes 1.10.1", "heck 0.5.0", "itertools 0.12.1", "log", @@ -7517,7 +8030,27 @@ dependencies = [ "prost 0.12.6", "prost-types 0.12.6", "regex", - "syn 2.0.96", + "syn 2.0.106", + "tempfile", +] + +[[package]] +name = "prost-build" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" +dependencies = [ + "heck 0.5.0", + "itertools 0.14.0", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease 0.2.15", + "prost 0.13.5", + "prost-types 0.13.5", + "regex", + "syn 2.0.106", "tempfile", ] @@ -7529,8 +8062,8 @@ checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" dependencies = [ "anyhow", "itertools 0.10.5", - "proc-macro2 1.0.93", - "quote 1.0.38", + "proc-macro2 1.0.101", + "quote 1.0.40", "syn 1.0.109", ] @@ -7542,34 +8075,34 @@ checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" dependencies = [ "anyhow", "itertools 0.12.1", - "proc-macro2 1.0.93", - "quote 1.0.38", - "syn 2.0.96", + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", ] [[package]] name = "prost-derive" -version = "0.13.3" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9552f850d5f0964a4e4d0bf306459ac29323ddfbae05e35a7c0d35cb0803cc5" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.13.0", - "proc-macro2 1.0.93", - "quote 1.0.38", - "syn 2.0.96", + "itertools 0.14.0", + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", ] [[package]] name = "prost-reflect" -version = "0.14.3" +version = "0.14.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20ae544fca2892fd4b7e9ff26cba1090cedf1d4d95c2aded1af15d2f93f270b8" +checksum = "7b5edd582b62f5cde844716e66d92565d7faf7ab1445c8cebce6e00fba83ddb2" dependencies = [ "base64 0.22.1", "once_cell", - "prost 0.13.3", - "prost-types 0.13.3", + "prost 0.13.5", + "prost-types 0.13.5", "serde", "serde-value", ] @@ -7594,11 +8127,11 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.13.3" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4759aa0d3a6232fb8dbdb97b61de2c20047c68aca932c7ed76da9d788508d670" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" dependencies = [ - "prost 0.13.3", + "prost 0.13.5", ] [[package]] @@ -7631,8 +8164,8 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac" dependencies = [ - "proc-macro2 1.0.93", - "quote 1.0.38", + "proc-macro2 1.0.101", + "quote 1.0.40", "syn 1.0.109", ] @@ -7648,30 +8181,28 @@ dependencies = [ [[package]] name = "pulsar" -version = "6.3.0" +version = "6.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7f3541ff84e39da334979ac4bf171e0f277f4f782603aeae65bf5795dc7275a" +checksum = "6cee616af00383c461f9ceb0067d15dee68e7d313ae47dbd7f8543236aed7ee9" dependencies = [ + "async-channel 2.3.1", "async-trait", - "bit-vec 0.6.3", - "bytes 1.9.0", + "bytes 1.10.1", "chrono", "crc", "data-url", "flate2", "futures 0.3.31", - "futures-io", - "futures-timer", "log", "lz4", "native-tls", - "nom", + "nom 7.1.3", "oauth2", "openidconnect", "pem", - "prost 0.11.9", - "prost-build 0.11.9", - "prost-derive 0.11.9", + "prost 0.13.5", + "prost-build 0.13.5", + "prost-derive 0.13.5", "rand 0.8.5", "regex", "serde", @@ -7682,7 +8213,7 @@ dependencies = [ "tokio-util", "url", "uuid", - "zstd 0.12.4", + "zstd 0.13.2", ] [[package]] @@ -7693,9 +8224,9 @@ checksum = "658fa1faf7a4cc5f057c9ee5ef560f717ad9d8dc66d975267f709624d6e1ab88" [[package]] name = "quanta" -version = "0.12.5" +version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bd1fe6824cea6538803de3ff1bc0cf3949024db3d43c9643024bfb33a807c0e" +checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7" dependencies = [ "crossbeam-utils", "libc", @@ -7712,6 +8243,21 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" +[[package]] +name = "quick-junit" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ed1a693391a16317257103ad06a88c6529ac640846021da7c435a06fffdacd7" +dependencies = [ + "chrono", + "indexmap 2.11.0", + "newtype-uuid", + "quick-xml 0.37.4", + "strip-ansi-escapes", + "thiserror 2.0.3", + "uuid", +] + [[package]] name = "quick-xml" version = "0.31.0" @@ -7722,6 +8268,16 @@ dependencies = [ "serde", ] +[[package]] +name = "quick-xml" +version = "0.37.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4ce8c88de324ff838700f36fb6ab86c96df0e3c4ab6ef3a9b2044465cce1369" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quickcheck" version = "1.0.3" @@ -7735,13 +8291,65 @@ dependencies = [ [[package]] name = "quickcheck_macros" -version = "1.0.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b22a693222d716a9587786f37ac3f6b4faedb5b80c23914e7303ff5a1d8016e9" +checksum = "f71ee38b42f8459a88d3362be6f9b841ad2d5421844f61eb1c59c11bff3ac14a" dependencies = [ - "proc-macro2 1.0.93", - "quote 1.0.38", - "syn 1.0.109", + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", +] + +[[package]] +name = "quinn" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62e96808277ec6f97351a2380e6c25114bc9e67037775464979f3037c92d05ef" +dependencies = [ + "bytes 1.10.1", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls 0.23.23", + "socket2 0.5.10", + "thiserror 2.0.3", + "tokio", + "tracing 0.1.41", +] + +[[package]] +name = "quinn-proto" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2fe5ef3495d7d2e377ff17b1a8ce2ee2ec2a18cde8b6ad6619d65d0701c135d" +dependencies = [ + "bytes 1.10.1", + "getrandom 0.2.15", + "rand 0.8.5", + "ring", + "rustc-hash", + "rustls 0.23.23", + "rustls-pki-types", + "slab", + "thiserror 2.0.3", + "tinyvec", + "tracing 0.1.41", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee4e529991f949c5e25755532370b8af5d114acae52326361d68d47af64aa842" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2 0.5.10", + "tracing 0.1.41", + "windows-sys 0.59.0", ] [[package]] @@ -7755,11 +8363,11 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.38" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" dependencies = [ - "proc-macro2 1.0.93", + "proc-macro2 1.0.101", ] [[package]] @@ -7808,6 +8416,16 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "rand" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.0", +] + [[package]] name = "rand_chacha" version = "0.2.2" @@ -7828,6 +8446,16 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.0", +] + [[package]] name = "rand_core" version = "0.5.1" @@ -7846,14 +8474,24 @@ dependencies = [ "getrandom 0.2.15", ] +[[package]] +name = "rand_core" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b08f3c9802962f7e1b25113931d94f43ed9725bebc59db9d0c3e9a23b67e15ff" +dependencies = [ + "getrandom 0.3.1", + "zerocopy 0.8.16", +] + [[package]] name = "rand_distr" -version = "0.4.3" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" +checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" dependencies = [ "num-traits", - "rand 0.8.5", + "rand 0.9.2", ] [[package]] @@ -7867,11 +8505,11 @@ dependencies = [ [[package]] name = "rand_xorshift" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d25bf25ec5ae4a3f1b92f929810509a2f53d7dca2f50b794ff57e3face536c8f" +checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" dependencies = [ - "rand_core 0.6.4", + "rand_core 0.9.0", ] [[package]] @@ -7880,14 +8518,14 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eabd94c2f37801c20583fc49dd5cd6b0ba68c716787c2dd6ed18571e1e63117b" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.9.0", "cassowary", "compact_str", - "crossterm", + "crossterm 0.28.1", "indoc", "instability", "itertools 0.13.0", - "lru", + "lru 0.12.5", "paste", "strum 0.26.3", "unicode-segmentation", @@ -7901,7 +8539,7 @@ version = "11.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d86a7c4638d42c44551f4791a20e687dbb4c3de1f33c43dd71e355cd429def1" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.9.0", ] [[package]] @@ -7984,24 +8622,27 @@ dependencies = [ [[package]] name = "redis" -version = "0.24.0" +version = "0.32.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c580d9cbbe1d1b479e8d67cf9daf6a62c957e6846048408b80b43ac3f6af84cd" +checksum = "7cd3650deebc68526b304898b192fa4102a4ef0b9ada24da096559cb60e0eef8" dependencies = [ "arc-swap", - "async-trait", - "bytes 1.9.0", + "backon", + "bytes 1.10.1", + "cfg-if", "combine 4.6.6", - "futures 0.3.31", + "futures-channel", "futures-util", "itoa", "native-tls", + "num-bigint", "percent-encoding", "pin-project-lite", + "rand 0.9.2", "ryu", + "socket2 0.6.0", "tokio", "tokio-native-tls", - "tokio-retry", "tokio-util", "url", ] @@ -8017,20 +8658,20 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.3.5" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" dependencies = [ "bitflags 1.3.2", ] [[package]] name = "redox_syscall" -version = "0.4.1" +version = "0.5.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +checksum = "928fca9cf2aa042393a8325b9ead81d2f0df4cb12e1e24cef072922ccd99c5af" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.9.0", ] [[package]] @@ -8044,16 +8685,61 @@ dependencies = [ "thiserror 1.0.68", ] +[[package]] +name = "redox_users" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd6f9d3d47bdd2ad6945c5015a226ec6155d0bcdfd8f7cd29f86b71f8de99d2b" +dependencies = [ + "getrandom 0.2.15", + "libredox", + "thiserror 2.0.3", +] + +[[package]] +name = "ref-cast" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a0ae411dbe946a674d89546582cea4ba2bb8defac896622d6496f14c23ba5cf" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1165225c21bff1f3bbce98f5a1f889949bc902d3575308cc7b0de30b4f6d27c7" +dependencies = [ + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", +] + +[[package]] +name = "referencing" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a3d769362109497b240e66462606bc28af68116436c8669bac17069533b908e" +dependencies = [ + "ahash 0.8.11", + "fluent-uri 0.3.2", + "once_cell", + "parking_lot 0.12.4", + "percent-encoding", + "serde_json", +] + [[package]] name = "regex" -version = "1.11.1" +version = "1.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +checksum = "23d7fd106d8c02486a8d64e778353d1cffe08ce79ac2e82f540c86d0facf6912" dependencies = [ "aho-corasick", "memchr", "regex-automata 0.4.8", - "regex-syntax 0.8.5", + "regex-syntax", ] [[package]] @@ -8061,9 +8747,6 @@ name = "regex-automata" version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" -dependencies = [ - "regex-syntax 0.6.29", -] [[package]] name = "regex-automata" @@ -8073,26 +8756,28 @@ checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.5", + "regex-syntax", ] [[package]] -name = "regex-lite" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30b661b2f27137bdbc16f00eda72866a92bb28af1753ffbd56744fb6e2e9cd8e" - -[[package]] -name = "regex-syntax" -version = "0.6.29" +name = "regex-filtered" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" +checksum = "4c11639076bf147be211b90e47790db89f4c22b6c8a9ca6e960833869da67166" +dependencies = [ + "aho-corasick", + "indexmap 2.11.0", + "itertools 0.13.0", + "nohash", + "regex", + "regex-syntax", +] [[package]] -name = "regex-syntax" -version = "0.7.5" +name = "regex-lite" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" +checksum = "30b661b2f27137bdbc16f00eda72866a92bb28af1753ffbd56744fb6e2e9cd8e" [[package]] name = "regex-syntax" @@ -8122,7 +8807,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78bf93c4af7a8bb7d879d51cebe797356ff10ae8516ace542b5182d9dcac10b2" dependencies = [ "base64 0.21.7", - "bytes 1.9.0", + "bytes 1.10.1", "encoding_rs", "futures-core", "futures-util", @@ -8131,7 +8816,7 @@ dependencies = [ "http-body 0.4.5", "hyper 0.14.28", "hyper-rustls 0.24.2", - "hyper-tls", + "hyper-tls 0.5.0", "ipnet", "js-sys", "log", @@ -8140,7 +8825,7 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls 0.21.11", + "rustls 0.21.12", "rustls-pemfile 1.0.4", "serde", "serde_json", @@ -8158,42 +8843,51 @@ dependencies = [ "wasm-streams", "web-sys", "webpki-roots 0.25.2", - "winreg 0.50.0", + "winreg", ] [[package]] name = "reqwest" -version = "0.12.4" +version = "0.12.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "566cafdd92868e0939d3fb961bd0dc25fcfaaed179291093b3d43e6b3150ea10" +checksum = "a77c62af46e79de0a562e1a9849205ffcb7fc1238876e9bd743357570e04046f" dependencies = [ "base64 0.22.1", - "bytes 1.9.0", + "bytes 1.10.1", + "cookie", + "cookie_store", + "futures-channel", "futures-core", "futures-util", - "http 1.1.0", + "h2 0.4.12", + "http 1.3.1", "http-body 1.0.0", "http-body-util", - "hyper 1.4.1", - "hyper-rustls 0.26.0", + "hyper 1.7.0", + "hyper-rustls 0.27.5", + "hyper-tls 0.6.0", "hyper-util", "ipnet", "js-sys", "log", "mime", "mime_guess", + "native-tls", "once_cell", "percent-encoding", "pin-project-lite", - "rustls 0.22.4", + "quinn", + "rustls 0.23.23", + "rustls-native-certs 0.8.1", "rustls-pemfile 2.1.0", "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", - "sync_wrapper 0.1.2", + "sync_wrapper 1.0.1", "tokio", - "tokio-rustls 0.25.0", + "tokio-native-tls", + "tokio-rustls 0.26.2", "tokio-util", "tower-service", "url", @@ -8202,7 +8896,43 @@ dependencies = [ "wasm-streams", "web-sys", "webpki-roots 0.26.1", - "winreg 0.52.0", + "windows-registry", +] + +[[package]] +name = "reqwest-middleware" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57f17d28a6e6acfe1733fe24bcd30774d13bffa4b8a22535b4c8c98423088d4e" +dependencies = [ + "anyhow", + "async-trait", + "http 1.3.1", + "reqwest 0.12.9", + "serde", + "thiserror 1.0.68", + "tower-service", +] + +[[package]] +name = "reqwest-retry" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29c73e4195a6bfbcb174b790d9b3407ab90646976c55de58a6515da25d851178" +dependencies = [ + "anyhow", + "async-trait", + "futures 0.3.31", + "getrandom 0.2.15", + "http 1.3.1", + "hyper 1.7.0", + "parking_lot 0.11.2", + "reqwest 0.12.9", + "reqwest-middleware", + "retry-policies", + "thiserror 1.0.68", + "tokio", + "wasm-timer", ] [[package]] @@ -8215,6 +8945,15 @@ dependencies = [ "quick-error", ] +[[package]] +name = "retry-policies" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5875471e6cab2871bc150ecb8c727db5113c9338cc3354dc5ee3425b6aa40a1c" +dependencies = [ + "rand 0.8.5", +] + [[package]] name = "rfc6979" version = "0.4.0" @@ -8227,16 +8966,16 @@ dependencies = [ [[package]] name = "ring" -version = "0.17.5" +version = "0.17.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb0205304757e5d899b9c2e448b867ffd03ae7f988002e47cd24954391394d0b" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", + "cfg-if", "getrandom 0.2.15", "libc", - "spin 0.9.8", "untrusted", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -8247,7 +8986,7 @@ checksum = "9008cd6385b9e161d8229e1f6549dd23c3d022f132a2ea37ac3a10ac4935779b" dependencies = [ "bitvec", "bytecheck", - "bytes 1.9.0", + "bytes 1.10.1", "hashbrown 0.12.3", "ptr_meta", "rend", @@ -8263,8 +9002,8 @@ version = "0.7.45" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "503d1d27590a2b0a3a4ca4c94755aa2875657196ecbf401a42eff41d7de532c0" dependencies = [ - "proc-macro2 1.0.93", - "quote 1.0.38", + "proc-macro2 1.0.101", + "quote 1.0.40", "syn 1.0.109", ] @@ -8310,9 +9049,9 @@ dependencies = [ [[package]] name = "roaring" -version = "0.10.10" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a652edd001c53df0b3f96a36a8dc93fce6866988efc16808235653c6bcac8bf2" +checksum = "f08d6a905edb32d74a5d5737a0c9d7e950c312f3c46cb0ca0a2ca09ea11878a0" dependencies = [ "bytemuck", "byteorder", @@ -8346,31 +9085,30 @@ dependencies = [ [[package]] name = "rstest" -version = "0.24.0" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03e905296805ab93e13c1ec3a03f4b6c4f35e9498a3d5fa96dc626d22c03cd89" +checksum = "f5a3193c063baaa2a95a33f03035c8a72b83d97a54916055ba22d35ed3839d49" dependencies = [ "futures-timer", "futures-util", "rstest_macros", - "rustc_version 0.4.1", ] [[package]] name = "rstest_macros" -version = "0.24.0" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef0053bbffce09062bee4bcc499b0fbe7a57b879f1efe088d6d8d4c7adcdef9b" +checksum = "9c845311f0ff7951c5506121a9ad75aec44d083c31583b2ea5a30bcb0b0abba0" dependencies = [ "cfg-if", "glob", "proc-macro-crate 3.2.0", - "proc-macro2 1.0.93", - "quote 1.0.38", + "proc-macro2 1.0.101", + "quote 1.0.40", "regex", "relative-path", "rustc_version 0.4.1", - "syn 2.0.96", + "syn 2.0.106", "unicode-ident", ] @@ -8380,7 +9118,7 @@ version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1568e15fab2d546f940ed3a21f48bbbd1c494c90c99c4481339364a497f94a9" dependencies = [ - "bytes 1.9.0", + "bytes 1.10.1", "flume 0.11.0", "futures-util", "log", @@ -8399,13 +9137,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06676aec5ccb8fc1da723cc8c0f9a46549f21ebb8753d3915c6c41db1e7f1dc4" dependencies = [ "arrayvec", - "borsh", - "bytes 1.9.0", "num-traits", - "rand 0.8.5", - "rkyv", - "serde", - "serde_json", ] [[package]] @@ -8435,7 +9167,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" dependencies = [ - "semver 1.0.24", + "semver 1.0.26", ] [[package]] @@ -8468,18 +9200,31 @@ version = "0.38.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "99e4ea3e1cdc4b559b8e5650f9c8e5998e3e5c1343b4eaf034565f32318d63c0" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.9.0", "errno", "libc", "linux-raw-sys 0.4.14", "windows-sys 0.52.0", ] +[[package]] +name = "rustix" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dade4812df5c384711475be5fcd8c162555352945401aed22a35bffeab61f657" +dependencies = [ + "bitflags 2.9.0", + "errno", + "libc", + "linux-raw-sys 0.9.2", + "windows-sys 0.59.0", +] + [[package]] name = "rustls" -version = "0.21.11" +version = "0.21.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fecbfb7b1444f477b345853b1fce097a2c6fb637b2bfb87e6bc5db0f043fae4" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" dependencies = [ "log", "ring", @@ -8503,9 +9248,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.21" +version = "0.23.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f287924602bf649d949c63dc8ac8b235fa5387d394020705b80c4eb597ce5b8" +checksum = "47796c98c480fce5406ef69d1c76378375492c3b0a0de587be0c1d9feb12f395" dependencies = [ "once_cell", "ring", @@ -8524,7 +9269,7 @@ dependencies = [ "openssl-probe", "rustls-pemfile 1.0.4", "schannel", - "security-framework", + "security-framework 2.10.0", ] [[package]] @@ -8537,20 +9282,19 @@ dependencies = [ "rustls-pemfile 2.1.0", "rustls-pki-types", "schannel", - "security-framework", + "security-framework 2.10.0", ] [[package]] name = "rustls-native-certs" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcaf18a4f2be7326cd874a5fa579fae794320a0f388d365dca7e480e55f83f8a" +checksum = "7fcff2dd52b58a8d98a70243663a0d234c4e2b79235637849d15913394a247d3" dependencies = [ "openssl-probe", - "rustls-pemfile 2.1.0", "rustls-pki-types", "schannel", - "security-framework", + "security-framework 3.3.0", ] [[package]] @@ -8577,6 +9321,9 @@ name = "rustls-pki-types" version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2bf47e6ff922db3825eb750c4e2ff784c6ff8fb9e13046ef6a1d1c5401b0b37" +dependencies = [ + "web-time", +] [[package]] name = "rustls-webpki" @@ -8619,17 +9366,17 @@ dependencies = [ [[package]] name = "rustyline" -version = "15.0.0" +version = "16.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ee1e066dc922e513bda599c6ccb5f3bb2b0ea5870a579448f2622993f0a9a2f" +checksum = "62fd9ca5ebc709e8535e8ef7c658eb51457987e48c98ead2be482172accc408d" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.9.0", "cfg-if", "clipboard-win", "libc", "log", "memchr", - "nix 0.29.0", + "nix 0.30.1", "unicode-segmentation", "unicode-width 0.2.0", "utf8parse", @@ -8638,9 +9385,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.18" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" [[package]] name = "salsa20" @@ -8691,6 +9438,30 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "schemars" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd191f9397d57d581cddd31014772520aa448f65ef991055d7f61582c65165f" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + +[[package]] +name = "schemars" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1375ba8ef45a6f15d83fa8748f1079428295d403d6ea991d09ab100155fbc06d" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + [[package]] name = "scoped-tls" version = "1.0.1" @@ -8750,7 +9521,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "770452e37cad93e0a50d5abc3990d2bc351c36d0328f86cefec2f2fb206eaef6" dependencies = [ "bitflags 1.3.2", - "core-foundation", + "core-foundation 0.9.3", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80fb1d92c5028aa318b4b8bd7302a5bfcf48be96a37fc6fc790f806b0004ee0c" +dependencies = [ + "bitflags 2.9.0", + "core-foundation 0.10.1", "core-foundation-sys", "libc", "security-framework-sys", @@ -8758,9 +9542,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.10.0" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f3cc463c0ef97e11c3461a9d3787412d30e8e7eb907c79180c4a57bf7c04ef" +checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" dependencies = [ "core-foundation-sys", "libc", @@ -8777,9 +9561,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.24" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cb6eb87a131f756572d7fb904f6e7b68633f09cca868c5df1c4b8d1a694bbba" +checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" dependencies = [ "serde", ] @@ -8792,20 +9576,20 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.217" +version = "1.0.219" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" dependencies = [ "serde_derive", ] [[package]] name = "serde-toml-merge" -version = "0.3.8" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93b4e415d6bff989e5e48649ca9b8b4d4997cb069a0c90a84bfd38c7df5e3968" +checksum = "0fc44799282f511a5d403d72a4ff028dc2c87f7fe6830abe3c33bb2fa6dfccec" dependencies = [ - "toml", + "toml 0.9.5", ] [[package]] @@ -8819,34 +9603,23 @@ dependencies = [ ] [[package]] -name = "serde-wasm-bindgen" -version = "0.6.5" +name = "serde_bytes" +version = "0.11.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8302e169f0eddcc139c70f139d19d6467353af16f9fce27e8c30158036a1e16b" +checksum = "8437fd221bde2d4ca316d61b90e337e9e702b3820b87d63caa9ba6c02bd06d96" dependencies = [ - "js-sys", "serde", - "wasm-bindgen", ] [[package]] -name = "serde_bytes" -version = "0.11.15" +name = "serde_derive" +version = "1.0.219" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "387cc504cb06bb40a96c8e04e951fe01854cf6bc921053c954e4a606d9675c6a" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" dependencies = [ - "serde", -] - -[[package]] -name = "serde_derive" -version = "1.0.217" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" -dependencies = [ - "proc-macro2 1.0.93", - "quote 1.0.38", - "syn 2.0.96", + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", ] [[package]] @@ -8855,18 +9628,18 @@ version = "0.29.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ - "proc-macro2 1.0.93", - "quote 1.0.38", - "syn 2.0.96", + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", ] [[package]] name = "serde_json" -version = "1.0.135" +version = "1.0.143" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b0d7ba2887406110130a978386c4e1befb98c674b4fba677954e4db976630d9" +checksum = "d401abef1d108fbd9cbaebc3e46611f4b1021f714a0597a71f41ee463f5f4a5a" dependencies = [ - "indexmap 2.7.0", + "indexmap 2.11.0", "itoa", "memchr", "ryu", @@ -8918,16 +9691,25 @@ version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3081f5ffbb02284dda55132aa26daecedd7372a42417bbbab6f14ab7d6bb9145" dependencies = [ - "proc-macro2 1.0.93", - "quote 1.0.38", - "syn 2.0.96", + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", +] + +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", ] [[package]] name = "serde_spanned" -version = "0.6.7" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb5b1b31579f3811bf615c144393417496f152e12ac8b7663bf664f4a815306d" +checksum = "40734c41988f7306bb04f0ecf60ec0f3f1caa34290e4e8ea471dcd3346483b83" dependencies = [ "serde", ] @@ -8956,19 +9738,21 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.12.0" +version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6b6f7f2fcb69f747921f79f3926bd1e203fce4fef62c268dd3abfb6d86029aa" +checksum = "f2c45cd61fefa9db6f254525d46e392b852e0e61d9a1fd36e5bd183450a556d5" dependencies = [ "base64 0.22.1", "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.7.0", + "indexmap 2.11.0", + "schemars 0.9.0", + "schemars 1.0.3", "serde", "serde_derive", "serde_json", - "serde_with_macros 3.12.0", + "serde_with_macros 3.14.0", "time", ] @@ -8979,33 +9763,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e182d6ec6f05393cc0e5ed1bf81ad6db3a8feedf8ee515ecdd369809bcce8082" dependencies = [ "darling 0.13.4", - "proc-macro2 1.0.93", - "quote 1.0.38", + "proc-macro2 1.0.101", + "quote 1.0.40", "syn 1.0.109", ] [[package]] name = "serde_with_macros" -version = "3.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d00caa5193a3c8362ac2b73be6b9e768aa5a4b2f721d8f4b339600c3cb51f8e" -dependencies = [ - "darling 0.20.8", - "proc-macro2 1.0.93", - "quote 1.0.38", - "syn 2.0.96", -] - -[[package]] -name = "serde_yaml" -version = "0.8.26" +version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578a7433b776b56a35785ed5ce9a7e777ac0598aac5a6dd1b4b18a307c7fc71b" +checksum = "de90945e6565ce0d9a25098082ed4ee4002e047cb59892c318d66821e14bb30f" dependencies = [ - "indexmap 1.9.3", - "ryu", - "serde", - "yaml-rust", + "darling 0.20.11", + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", ] [[package]] @@ -9014,7 +9786,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.7.0", + "indexmap 2.11.0", "itoa", "ryu", "serde", @@ -9045,9 +9817,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.8" +version = "0.10.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", "cpufeatures", @@ -9083,6 +9855,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "signal-hook" version = "0.3.17" @@ -9137,9 +9915,9 @@ dependencies = [ [[package]] name = "simdutf8" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" [[package]] name = "similar" @@ -9153,11 +9931,11 @@ dependencies = [ [[package]] name = "similar-asserts" -version = "1.6.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfe85670573cd6f0fa97940f26e7e6601213c3b0555246c24234131f88c5709e" +checksum = "b5b441962c817e33508847a22bd82f03a30cff43642dc2fae8b050566121eb9a" dependencies = [ - "console", + "console 0.15.7", "similar", ] @@ -9173,6 +9951,12 @@ version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" +[[package]] +name = "siphasher" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" + [[package]] name = "sketches-ddsketch" version = "0.3.0" @@ -9190,9 +9974,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.13.2" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" dependencies = [ "serde", ] @@ -9203,7 +9987,7 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13f2b548cd8447f8de0fdf1c592929f70f4fc7039a05e47404b0d096ec6987a1" dependencies = [ - "async-channel", + "async-channel 1.9.0", "async-executor", "async-fs", "async-io 1.13.0", @@ -9211,7 +9995,7 @@ dependencies = [ "async-net", "async-process", "blocking", - "futures-lite", + "futures-lite 1.13.0", ] [[package]] @@ -9237,18 +10021,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4de37ad025c587a29e8f3f5605c00f70b98715ef90b9061a815b9e59e9042d6" dependencies = [ "doc-comment", - "futures-core", - "pin-project", "snafu-derive 0.7.5", ] [[package]] name = "snafu" -version = "0.8.0" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d342c51730e54029130d7dc9fd735d28c4cd360f1368c01981d4f03ff207f096" +checksum = "6e84b3f4eacbf3a1ce05eac6763b4d629d60cbc94d632e4092c54ade71f1e1a2" dependencies = [ - "snafu-derive 0.8.0", + "futures-core", + "pin-project", + "snafu-derive 0.8.9", ] [[package]] @@ -9258,21 +10042,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "990079665f075b699031e9c08fd3ab99be5029b96f3b78dc0709e8f77e4efebf" dependencies = [ "heck 0.4.1", - "proc-macro2 1.0.93", - "quote 1.0.38", + "proc-macro2 1.0.101", + "quote 1.0.40", "syn 1.0.109", ] [[package]] name = "snafu-derive" -version = "0.8.0" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "080c44971436b1af15d6f61ddd8b543995cf63ab8e677d46b00cc06f4ef267a0" +checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" dependencies = [ - "heck 0.4.1", - "proc-macro2 1.0.93", - "quote 1.0.38", - "syn 2.0.96", + "heck 0.5.0", + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", ] [[package]] @@ -9293,14 +10077,24 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.8" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" dependencies = [ "libc", "windows-sys 0.52.0", ] +[[package]] +name = "socket2" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "233504af464074f9d066d7b5416c5f9b894a5862a6506e306f7b816cdd6f1807" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + [[package]] name = "spin" version = "0.5.2" @@ -9335,6 +10129,198 @@ dependencies = [ "der", ] +[[package]] +name = "sqlx" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fefb893899429669dcdd979aff487bd78f4064e5e7907e4269081e0ef7d97dc" +dependencies = [ + "sqlx-core", + "sqlx-macros", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", +] + +[[package]] +name = "sqlx-core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6" +dependencies = [ + "base64 0.22.1", + "bytes 1.10.1", + "chrono", + "crc", + "crossbeam-queue", + "either", + "event-listener 5.3.1", + "futures-core", + "futures-intrusive", + "futures-io", + "futures-util", + "hashbrown 0.15.2", + "hashlink", + "indexmap 2.11.0", + "log", + "memchr", + "once_cell", + "percent-encoding", + "serde", + "serde_json", + "sha2", + "smallvec", + "thiserror 2.0.3", + "tokio", + "tokio-stream", + "tracing 0.1.41", + "url", +] + +[[package]] +name = "sqlx-macros" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2d452988ccaacfbf5e0bdbc348fb91d7c8af5bee192173ac3636b5fb6e6715d" +dependencies = [ + "proc-macro2 1.0.101", + "quote 1.0.40", + "sqlx-core", + "sqlx-macros-core", + "syn 2.0.106", +] + +[[package]] +name = "sqlx-macros-core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19a9c1841124ac5a61741f96e1d9e2ec77424bf323962dd894bdb93f37d5219b" +dependencies = [ + "dotenvy", + "either", + "heck 0.5.0", + "hex", + "once_cell", + "proc-macro2 1.0.101", + "quote 1.0.40", + "serde", + "serde_json", + "sha2", + "sqlx-core", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", + "syn 2.0.106", + "tokio", + "url", +] + +[[package]] +name = "sqlx-mysql" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526" +dependencies = [ + "atoi", + "base64 0.22.1", + "bitflags 2.9.0", + "byteorder", + "bytes 1.10.1", + "chrono", + "crc", + "digest", + "dotenvy", + "either", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "generic-array", + "hex", + "hkdf", + "hmac", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "percent-encoding", + "rand 0.8.5", + "rsa", + "serde", + "sha1", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror 2.0.3", + "tracing 0.1.41", + "whoami", +] + +[[package]] +name = "sqlx-postgres" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" +dependencies = [ + "atoi", + "base64 0.22.1", + "bitflags 2.9.0", + "byteorder", + "chrono", + "crc", + "dotenvy", + "etcetera", + "futures-channel", + "futures-core", + "futures-util", + "hex", + "hkdf", + "hmac", + "home", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "rand 0.8.5", + "serde", + "serde_json", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror 2.0.3", + "tracing 0.1.41", + "whoami", +] + +[[package]] +name = "sqlx-sqlite" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2d12fe70b2c1b4401038055f90f151b78208de1f9f89a7dbfd41587a10c3eea" +dependencies = [ + "atoi", + "chrono", + "flume 0.11.0", + "futures-channel", + "futures-core", + "futures-executor", + "futures-intrusive", + "futures-util", + "libsqlite3-sys", + "log", + "percent-encoding", + "serde", + "serde_urlencoded", + "sqlx-core", + "thiserror 2.0.3", + "tracing 0.1.41", + "url", +] + [[package]] name = "stable_deref_trait" version = "1.2.0" @@ -9372,7 +10358,7 @@ checksum = "f91138e76242f575eb1d3b38b4f1362f10d3a43f47d182a5b359af488a02293b" dependencies = [ "new_debug_unreachable", "once_cell", - "parking_lot", + "parking_lot 0.12.4", "phf_shared 0.10.0", "precomputed-hash", ] @@ -9390,9 +10376,9 @@ dependencies = [ [[package]] name = "strip-ansi-escapes" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55ff8ef943b384c414f54aefa961dd2bd853add74ec75e7ac74cf91dba62bcfa" +checksum = "2a8f8038e7e7969abb3f1b7c2a811225e9296da208539e0f79c5251d6cac0025" dependencies = [ "vte", ] @@ -9405,9 +10391,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "strsim" -version = "0.11.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ee073c9e4cd00e28217186dbe12796d692868f432bf2e97ee73bed0c56dfa01" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "strum" @@ -9431,10 +10417,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23dc1fa9ac9c169a78ba62f0b841814b7abae11bdd047b9c58f893439e309ea0" dependencies = [ "heck 0.4.1", - "proc-macro2 1.0.93", - "quote 1.0.38", + "proc-macro2 1.0.101", + "quote 1.0.40", "rustversion", - "syn 2.0.96", + "syn 2.0.106", ] [[package]] @@ -9444,10 +10430,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" dependencies = [ "heck 0.5.0", - "proc-macro2 1.0.93", - "quote 1.0.38", + "proc-macro2 1.0.101", + "quote 1.0.40", "rustversion", - "syn 2.0.96", + "syn 2.0.106", ] [[package]] @@ -9492,34 +10478,22 @@ version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ - "proc-macro2 1.0.93", - "quote 1.0.38", + "proc-macro2 1.0.101", + "quote 1.0.40", "unicode-ident", ] [[package]] name = "syn" -version = "2.0.96" +version = "2.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5d0adab1ae378d7f53bdebc67a39f1f151407ef230f0ce2883572f5d8985c80" +checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6" dependencies = [ - "proc-macro2 1.0.93", - "quote 1.0.38", + "proc-macro2 1.0.101", + "quote 1.0.40", "unicode-ident", ] -[[package]] -name = "syn_derive" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1329189c02ff984e9736652b1631330da25eaa6bc639089ed4915d25446cbe7b" -dependencies = [ - "proc-macro-error", - "proc-macro2 1.0.93", - "quote 1.0.38", - "syn 2.0.96", -] - [[package]] name = "sync_wrapper" version = "0.1.2" @@ -9531,6 +10505,9 @@ name = "sync_wrapper" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" +dependencies = [ + "futures-core", +] [[package]] name = "synstructure" @@ -9538,23 +10515,23 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ - "proc-macro2 1.0.93", - "quote 1.0.38", - "syn 2.0.96", + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", ] [[package]] name = "sysinfo" -version = "0.32.1" +version = "0.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c33cd241af0f2e9e3b5c32163b873b29956890b5342e6745b917ce9d490f4af" +checksum = "07cec4dc2d2e357ca1e610cfb07de2fa7a10fc3e9fe89f72545f3d244ea87753" dependencies = [ - "core-foundation-sys", "libc", "memchr", "ntapi 0.4.1", - "rayon", - "windows 0.57.0", + "objc2-core-foundation", + "objc2-io-kit", + "windows 0.60.0", ] [[package]] @@ -9572,12 +10549,22 @@ dependencies = [ [[package]] name = "syslog_loose" -version = "0.21.0" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "161028c00842709450114c39db3b29f44c898055ed8833bb9b535aba7facf30e" +checksum = "d6ec4df26907adce53e94eac201a9ba38744baea3bc97f34ffd591d5646231a6" dependencies = [ "chrono", - "nom", + "nom 8.0.0", +] + +[[package]] +name = "syslog_loose" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ddbead3f86a91bd8c9b559ce0c5b0cbf03e774ffb7f8511eaf78375cc4a7837" +dependencies = [ + "chrono", + "nom 8.0.0", ] [[package]] @@ -9587,7 +10574,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" dependencies = [ "bitflags 1.3.2", - "core-foundation", + "core-foundation 0.9.3", "system-configuration-sys", ] @@ -9632,22 +10619,21 @@ dependencies = [ [[package]] name = "temp-dir" -version = "0.1.14" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc1ee6eef34f12f765cb94725905c6312b6610ab2b0940889cfe58dae7bc3c72" +checksum = "83176759e9416cf81ee66cb6508dbfe9c96f20b8b56265a39917551c23c70964" [[package]] name = "tempfile" -version = "3.15.0" +version = "3.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8a559c81686f576e8cd0290cd2a24a2a9ad80c98b3478856500fcbd7acd704" +checksum = "15b61f8f20e3a6f7e0649d825294eaf317edce30f82cf6026e7e4cb9222a7d1e" dependencies = [ - "cfg-if", - "fastrand 2.1.1", - "getrandom 0.2.15", + "fastrand 2.3.0", + "getrandom 0.3.1", "once_cell", - "rustix 0.38.40", - "windows-sys 0.59.0", + "rustix 1.0.1", + "windows-sys 0.60.2", ] [[package]] @@ -9661,6 +10647,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "term" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3bb6001afcea98122260987f8b7b5da969ecad46dbf0b5453702f776b491a41" +dependencies = [ + "home", + "windows-sys 0.52.0", +] + [[package]] name = "termcolor" version = "1.3.0" @@ -9722,9 +10718,9 @@ version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7c61ec9a6f64d2793d8a45faba21efbe3ced62a886d44c36a009b2b519b4c7e" dependencies = [ - "proc-macro2 1.0.93", - "quote 1.0.38", - "syn 2.0.96", + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", ] [[package]] @@ -9733,19 +10729,18 @@ version = "2.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f077553d607adc1caf65430528a576c757a71ed73944b66ebb58ef2bbd243568" dependencies = [ - "proc-macro2 1.0.93", - "quote 1.0.38", - "syn 2.0.96", + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", ] [[package]] name = "thread_local" -version = "1.1.7" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" dependencies = [ "cfg-if", - "once_cell", ] [[package]] @@ -9802,15 +10797,6 @@ dependencies = [ "time-core", ] -[[package]] -name = "tiny-keccak" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" -dependencies = [ - "crunchy", -] - [[package]] name = "tinystr" version = "0.7.6" @@ -9848,21 +10834,23 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.43.0" +version = "1.47.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d61fa4ffa3de412bfea335c6ecff681de2b609ba3c77ef3e00e521813a9ed9e" +checksum = "89e49afdadebb872d3145a5638b59eb0691ea23e46ca484037cfab3b76b95038" dependencies = [ "backtrace", - "bytes 1.9.0", + "bytes 1.10.1", + "io-uring", "libc", "mio", - "parking_lot", + "parking_lot 0.12.4", "pin-project-lite", "signal-hook-registry", - "socket2 0.5.8", + "slab", + "socket2 0.6.0", "tokio-macros", "tracing 0.1.41", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -9892,9 +10880,9 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ - "proc-macro2 1.0.93", - "quote 1.0.38", - "syn 2.0.96", + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", ] [[package]] @@ -9920,25 +10908,25 @@ dependencies = [ [[package]] name = "tokio-postgres" -version = "0.7.12" +version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b5d3742945bc7d7f210693b0c58ae542c6fd47b17adbbda0885f3dcb34a6bdb" +checksum = "6c95d533c83082bb6490e0189acaa0bbeef9084e60471b696ca6988cd0541fb0" dependencies = [ "async-trait", "byteorder", - "bytes 1.9.0", + "bytes 1.10.1", "fallible-iterator", "futures-channel", "futures-util", "log", - "parking_lot", + "parking_lot 0.12.4", "percent-encoding", - "phf", + "phf 0.11.2", "pin-project-lite", "postgres-protocol", "postgres-types", - "rand 0.8.5", - "socket2 0.5.8", + "rand 0.9.2", + "socket2 0.5.10", "tokio", "tokio-util", "whoami", @@ -9961,7 +10949,7 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls 0.21.11", + "rustls 0.21.12", "tokio", ] @@ -9978,11 +10966,11 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.26.1" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f6d0975eaace0cf0fcadee4e4aaa5da15b5c079146f2cffb67c113be122bf37" +checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" dependencies = [ - "rustls 0.23.21", + "rustls 0.23.23", "tokio", ] @@ -10005,7 +10993,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2468baabc3311435b55dd935f702f42cd1b8abb7e754fb7dfb16bd36aa88f9f7" dependencies = [ "async-stream", - "bytes 1.9.0", + "bytes 1.10.1", "futures-core", "tokio", "tokio-stream", @@ -10019,7 +11007,7 @@ checksum = "212d5dcb2a1ce06d81107c3d0ffa3121fe974b73f068c8282cb1c32328113b6c" dependencies = [ "futures-util", "log", - "rustls 0.21.11", + "rustls 0.21.12", "tokio", "tungstenite 0.20.1", ] @@ -10041,7 +11029,7 @@ name = "tokio-util" version = "0.7.13" source = "git+https://github.com/vectordotdev/tokio?branch=tokio-util-0.7.13-framed-read-continue-on-error#b4bdfda8fe8aa24eba36de0d60063b14f30c7fe7" dependencies = [ - "bytes 1.9.0", + "bytes 1.10.1", "futures-core", "futures-io", "futures-sink", @@ -10057,37 +11045,61 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f591660438b3038dd04d16c938271c79e7e06260ad2ea2885a4861bfb238605d" dependencies = [ "base64 0.22.1", - "bytes 1.9.0", + "bytes 1.10.1", "futures-core", "futures-sink", - "http 1.1.0", + "http 1.3.1", "httparse", "rand 0.8.5", "ring", - "rustls-native-certs 0.8.0", "rustls-pki-types", "tokio", - "tokio-rustls 0.26.1", + "tokio-rustls 0.26.2", "tokio-util", + "webpki-roots 0.26.1", ] [[package]] name = "toml" -version = "0.8.19" +version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" dependencies = [ "serde", - "serde_spanned", - "toml_datetime", - "toml_edit 0.22.20", + "serde_spanned 0.6.9", + "toml_datetime 0.6.11", + "toml_edit 0.22.27", +] + +[[package]] +name = "toml" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75129e1dc5000bfbaa9fee9d1b21f974f9fbad9daec557a521ee6e080825f6e8" +dependencies = [ + "indexmap 2.11.0", + "serde", + "serde_spanned 1.0.0", + "toml_datetime 0.7.0", + "toml_parser", + "toml_writer", + "winnow 0.7.10", ] [[package]] name = "toml_datetime" -version = "0.6.8" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_datetime" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" +checksum = "bade1c3e902f58d73d3f294cd7f20391c1cb2fbcb643b73566bc773971df91e3" dependencies = [ "serde", ] @@ -10098,35 +11110,46 @@ version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap 2.7.0", - "toml_datetime", + "indexmap 2.11.0", + "toml_datetime 0.6.11", "winnow 0.5.18", ] [[package]] name = "toml_edit" -version = "0.20.7" +version = "0.22.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70f427fce4d84c72b5b732388bf4a9f4531b53f74e2887e3ecb2481f68f66d81" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ - "indexmap 2.7.0", - "toml_datetime", - "winnow 0.5.18", + "indexmap 2.11.0", + "serde", + "serde_spanned 0.6.9", + "toml_datetime 0.6.11", + "toml_write", + "winnow 0.7.10", ] [[package]] -name = "toml_edit" -version = "0.22.20" +name = "toml_parser" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "583c44c02ad26b0c3f3066fe629275e50627026c51ac2e595cca4c230ce1ce1d" +checksum = "b551886f449aa90d4fe2bdaa9f4a2577ad2dde302c61ecf262d80b116db95c10" dependencies = [ - "indexmap 2.7.0", - "serde", - "serde_spanned", - "toml_datetime", - "winnow 0.6.18", + "winnow 0.7.10", ] +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + +[[package]] +name = "toml_writer" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcc842091f2def52017664b53082ecbbeb5c7731092bad69d2c63050401dfd64" + [[package]] name = "tonic" version = "0.11.0" @@ -10137,7 +11160,7 @@ dependencies = [ "async-trait", "axum 0.6.20", "base64 0.21.7", - "bytes 1.9.0", + "bytes 1.10.1", "flate2", "h2 0.3.26", "http 0.2.9", @@ -10153,7 +11176,7 @@ dependencies = [ "tokio", "tokio-rustls 0.25.0", "tokio-stream", - "tower", + "tower 0.4.13", "tower-layer", "tower-service", "tracing 0.1.41", @@ -10170,21 +11193,21 @@ dependencies = [ "async-trait", "axum 0.7.5", "base64 0.22.1", - "bytes 1.9.0", - "h2 0.4.7", - "http 1.1.0", + "bytes 1.10.1", + "h2 0.4.12", + "http 1.3.1", "http-body 1.0.0", "http-body-util", - "hyper 1.4.1", + "hyper 1.7.0", "hyper-timeout 0.5.1", "hyper-util", "percent-encoding", "pin-project", - "prost 0.13.3", - "socket2 0.5.8", + "prost 0.13.5", + "socket2 0.5.10", "tokio", "tokio-stream", - "tower", + "tower 0.4.13", "tower-layer", "tower-service", "tracing 0.1.41", @@ -10197,9 +11220,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6fdaae4c2c638bb70fe42803a26fbd6fc6ac8c72f5c59f67ecc2a2dcabf4b07" dependencies = [ "prettyplease 0.1.25", - "proc-macro2 1.0.93", + "proc-macro2 1.0.101", "prost-build 0.11.9", - "quote 1.0.38", + "quote 1.0.40", "syn 1.0.109", ] @@ -10210,10 +11233,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be4ef6dd70a610078cb4e338a0f79d06bc759ff1b22d2120c2ff02ae264ba9c2" dependencies = [ "prettyplease 0.2.15", - "proc-macro2 1.0.93", + "proc-macro2 1.0.101", "prost-build 0.12.6", - "quote 1.0.38", - "syn 2.0.96", + "quote 1.0.40", + "syn 2.0.106", ] [[package]] @@ -10236,6 +11259,25 @@ dependencies = [ "tracing 0.1.41", ] +[[package]] +name = "tower" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +dependencies = [ + "futures-core", + "futures-util", + "indexmap 2.11.0", + "pin-project-lite", + "slab", + "sync_wrapper 1.0.1", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing 0.1.41", +] + [[package]] name = "tower-http" version = "0.4.4" @@ -10243,8 +11285,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61c5bb1d698276a2443e5ecfabc1008bf15a36c12e6a7176e7bf089ea9131140" dependencies = [ "async-compression", - "bitflags 2.6.0", - "bytes 1.9.0", + "bitflags 2.9.0", + "bytes 1.10.1", "futures-core", "futures-util", "http 0.2.9", @@ -10265,9 +11307,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5" dependencies = [ "base64 0.21.7", - "bitflags 2.6.0", - "bytes 1.9.0", - "http 1.1.0", + "bitflags 2.9.0", + "bytes 1.10.1", + "http 1.3.1", "http-body 1.0.0", "http-body-util", "mime", @@ -10279,15 +11321,15 @@ dependencies = [ [[package]] name = "tower-layer" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" [[package]] name = "tower-service" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tower-test" @@ -10331,9 +11373,9 @@ version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ - "proc-macro2 1.0.93", - "quote 1.0.38", - "syn 2.0.96", + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", ] [[package]] @@ -10391,7 +11433,7 @@ name = "tracing-limit" version = "0.1.0" dependencies = [ "criterion", - "dashmap 6.1.0", + "dashmap", "mock_instant", "tracing 0.1.41", "tracing-core 0.1.33", @@ -10421,14 +11463,14 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.19" +version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" +checksum = "2054a14f5307d601f88daf0553e1cbf472acc4f2c51afab632431cdcd72124d5" dependencies = [ "matchers", "nu-ansi-term", "once_cell", - "regex", + "regex-automata 0.4.8", "serde", "serde_json", "sharded-slab", @@ -10457,8 +11499,8 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04659ddb06c87d233c566112c1c9c5b9e98256d9af50ec3bc9c8327f873a7568" dependencies = [ - "quote 1.0.38", - "syn 2.0.96", + "quote 1.0.40", + "syn 2.0.106", ] [[package]] @@ -10515,7 +11557,7 @@ dependencies = [ "lazy_static", "log", "lru-cache", - "parking_lot", + "parking_lot 0.12.4", "resolv-conf", "smallvec", "thiserror 1.0.68", @@ -10531,11 +11573,10 @@ checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" [[package]] name = "tryhard" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c9f0a709784e86923586cff0d872dba54cd2d2e116b3bc57587d15737cfce9d" +checksum = "9fe58ebd5edd976e0fe0f8a14d2a04b7c81ef153ea9a54eebc42e67c2c23b4e5" dependencies = [ - "futures 0.3.31", "pin-project-lite", "tokio", ] @@ -10547,7 +11588,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e3dac10fd62eaf6617d3a904ae222845979aec67c615d1c842b4002c7666fb9" dependencies = [ "byteorder", - "bytes 1.9.0", + "bytes 1.10.1", "data-encoding", "http 0.2.9", "httparse", @@ -10566,9 +11607,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ef1a641ea34f399a848dea702823bbecfb4c486f911735368f1f137cb8257e1" dependencies = [ "byteorder", - "bytes 1.9.0", + "bytes 1.10.1", "data-encoding", - "http 1.1.0", + "http 1.3.1", "httparse", "log", "rand 0.8.5", @@ -10580,9 +11621,9 @@ dependencies = [ [[package]] name = "twox-hash" -version = "2.1.0" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7b17f197b3050ba473acf9181f7b1d3b66d1cf7356c6cc57886662276e65908" +checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" [[package]] name = "typed-builder" @@ -10590,8 +11631,8 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89851716b67b937e393b3daa8423e67ddfc4bbbf1654bcf05488e95e0828db0c" dependencies = [ - "proc-macro2 1.0.93", - "quote 1.0.38", + "proc-macro2 1.0.101", + "quote 1.0.40", "syn 1.0.109", ] @@ -10610,9 +11651,9 @@ version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f03ca4cb38206e2bef0700092660bb74d696f808514dae47fa1467cbfe26e96e" dependencies = [ - "proc-macro2 1.0.93", - "quote 1.0.38", - "syn 2.0.96", + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", ] [[package]] @@ -10623,9 +11664,9 @@ checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "typetag" -version = "0.2.19" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "044fc3365ddd307c297fe0fe7b2e70588cdab4d0f62dc52055ca0d11b174cf0e" +checksum = "73f22b40dd7bfe8c14230cf9702081366421890435b2d625fa92b4acc4c3de6f" dependencies = [ "erased-serde", "inventory", @@ -10636,13 +11677,13 @@ dependencies = [ [[package]] name = "typetag-impl" -version = "0.2.19" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9d30226ac9cbd2d1ff775f74e8febdab985dab14fb14aa2582c29a92d5555dc" +checksum = "35f5380909ffc31b4de4f4bdf96b877175a016aa2ca98cee39fcfd8c4d53d952" dependencies = [ - "proc-macro2 1.0.93", - "quote 1.0.38", - "syn 2.0.96", + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", ] [[package]] @@ -10655,17 +11696,14 @@ dependencies = [ ] [[package]] -name = "uaparser" -version = "0.6.1" +name = "ua-parser" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf694e7b0434d4fad6c879e984e8fdc3a62f5533c3d421762244f9e9d03f6927" +checksum = "d7176a413a0b7e94926d11a2054c6db5ac7fa42bf4ebe7e9571152e3f024ddfd" dependencies = [ - "derive_more", - "lazy_static", "regex", + "regex-filtered", "serde", - "serde_derive", - "serde_yaml 0.8.26", ] [[package]] @@ -10712,9 +11750,9 @@ dependencies = [ [[package]] name = "unicode-segmentation" -version = "1.10.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" [[package]] name = "unicode-truncate" @@ -10750,6 +11788,12 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" +[[package]] +name = "unit-prefix" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "323402cff2dd658f39ca17c789b502021b3f18707c91cdf22e3838e1b4023817" + [[package]] name = "universal-hash" version = "0.5.1" @@ -10787,7 +11831,7 @@ version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1ee6bfd0a27bf614353809a035cf6880b74239ec6c5e39a7b2860ca16809137" dependencies = [ - "num-rational", + "num-rational 0.3.2", "num-traits", "typenum", ] @@ -10842,16 +11886,28 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "uuid" -version = "1.11.1" +version = "1.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b913a3b5fe84142e269d63cc62b64319ccaf89b748fc31fe025177f767a756c4" +checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2" dependencies = [ - "getrandom 0.2.15", - "rand 0.8.5", + "getrandom 0.3.1", + "js-sys", + "rand 0.9.2", "serde", "wasm-bindgen", ] +[[package]] +name = "uuid-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b082222b4f6619906941c17eb2297fff4c2fb96cb60164170522942a200bd8" +dependencies = [ + "outref", + "uuid", + "vsimd", +] + [[package]] name = "valuable" version = "0.1.0" @@ -10869,6 +11925,7 @@ name = "vdev" version = "0.1.0" dependencies = [ "anyhow", + "cfg-if", "chrono", "clap", "clap-verbosity-flag", @@ -10876,27 +11933,30 @@ dependencies = [ "confy", "directories", "dunce", + "git2", "glob", "hex", - "indexmap 2.7.0", + "indexmap 2.11.0", "indicatif", + "indoc", "itertools 0.14.0", "log", "owo-colors", "paste", "regex", "reqwest 0.11.26", + "semver 1.0.26", "serde", "serde_json", - "serde_yaml 0.9.34+deprecated", + "serde_yaml", "sha2", "tempfile", - "toml", + "toml 0.9.5", ] [[package]] name = "vector" -version = "0.45.0" +version = "0.50.0" dependencies = [ "apache-avro", "approx", @@ -10911,11 +11971,13 @@ dependencies = [ "async-trait", "aws-config", "aws-credential-types", + "aws-runtime", "aws-sdk-cloudwatch", "aws-sdk-cloudwatchlogs", "aws-sdk-elasticsearch", "aws-sdk-firehose", "aws-sdk-kinesis", + "aws-sdk-kms", "aws-sdk-s3", "aws-sdk-secretsmanager", "aws-sdk-sns", @@ -10937,17 +11999,19 @@ dependencies = [ "bloomy", "bollard", "byteorder", - "bytes 1.9.0", + "bytes 1.10.1", "bytesize", + "cfg-if", "chrono", "chrono-tz", "clap", "colored", "console-subscriber", "criterion", - "crossterm", + "crossterm 0.29.0", "csv", "databend-client", + "deadpool", "derivative", "dirs-next", "dnsmsg-parser", @@ -10955,6 +12019,8 @@ dependencies = [ "dyn-clone", "encoding_rs", "enum_dispatch", + "evmap", + "evmap-derive", "exitcode", "fakedata", "flate2", @@ -10964,7 +12030,7 @@ dependencies = [ "goauth", "governor", "greptimedb-ingester", - "h2 0.4.7", + "h2 0.4.12", "hash_hasher", "hashbrown 0.14.5", "headers", @@ -10973,13 +12039,14 @@ dependencies = [ "hickory-proto", "hostname 0.4.0", "http 0.2.9", - "http 1.1.0", + "http 1.3.1", "http-body 0.4.5", "http-serde", + "humantime", "hyper 0.14.28", "hyper-openssl 0.9.2", "hyper-proxy", - "indexmap 2.7.0", + "indexmap 2.11.0", "indoc", "inventory", "ipnet", @@ -10990,7 +12057,7 @@ dependencies = [ "libc", "listenfd", "loki-logproto", - "lru", + "lru 0.16.0", "maxminddb", "md-5", "metrics", @@ -11003,7 +12070,7 @@ dependencies = [ "netlink-sys", "nix 0.26.2", "nkeys", - "nom", + "nom 8.0.0", "notify", "num-format", "number_prefix", @@ -11024,8 +12091,9 @@ dependencies = [ "prost-reflect", "prost-types 0.12.6", "pulsar", + "quick-junit", "quickcheck", - "rand 0.8.5", + "rand 0.9.2", "rand_distr", "ratatui", "rdkafka", @@ -11038,25 +12106,27 @@ dependencies = [ "rstest", "rumqttc", "seahash", - "semver 1.0.24", + "semver 1.0.26", "serde", "serde-toml-merge", "serde_bytes", "serde_json", - "serde_with 3.12.0", - "serde_yaml 0.9.34+deprecated", + "serde_with 3.14.0", + "serde_yaml", "similar-asserts", "smallvec", "smpl_jwt", - "snafu 0.7.5", + "snafu 0.8.9", "snap", - "socket2 0.5.8", + "socket2 0.5.10", + "sqlx", "stream-cancel", "strip-ansi-escapes", "sysinfo", "syslog", "tempfile", "test-generator", + "thread_local", "tikv-jemallocator", "tokio", "tokio-openssl", @@ -11065,10 +12135,10 @@ dependencies = [ "tokio-test", "tokio-tungstenite 0.20.1", "tokio-util", - "toml", + "toml 0.9.5", "tonic 0.11.0", "tonic-build 0.11.0", - "tower", + "tower 0.5.2", "tower-http 0.4.4", "tower-test", "tracing 0.1.41", @@ -11120,12 +12190,13 @@ dependencies = [ "async-stream", "async-trait", "bytecheck", - "bytes 1.9.0", + "bytes 1.10.1", "clap", "crc32fast", "criterion", "crossbeam-queue", "crossbeam-utils", + "dashmap", "derivative", "fslock", "futures 0.3.31", @@ -11135,14 +12206,15 @@ dependencies = [ "metrics-tracing-context", "metrics-util", "num-traits", + "ordered-float 4.6.0", "paste", "proptest", "quickcheck", - "rand 0.8.5", + "rand 0.9.2", "rkyv", "serde", - "serde_yaml 0.9.34+deprecated", - "snafu 0.7.5", + "serde_yaml", + "snafu 0.8.9", "temp-dir", "tokio", "tokio-test", @@ -11159,12 +12231,12 @@ name = "vector-common" version = "0.1.0" dependencies = [ "async-stream", - "bytes 1.9.0", + "bytes 1.10.1", "chrono", "crossbeam-utils", "derivative", "futures 0.3.31", - "indexmap 2.7.0", + "indexmap 2.11.0", "metrics", "paste", "pin-project", @@ -11187,15 +12259,15 @@ dependencies = [ "chrono-tz", "encoding_rs", "http 0.2.9", - "indexmap 2.7.0", + "indexmap 2.11.0", "inventory", "no-proxy", "num-traits", "serde", "serde_json", - "serde_with 3.12.0", - "snafu 0.7.5", - "toml", + "serde_with 3.14.0", + "snafu 0.8.9", + "toml 0.9.5", "tracing 0.1.41", "url", "vector-config-common", @@ -11207,13 +12279,13 @@ dependencies = [ name = "vector-config-common" version = "0.1.0" dependencies = [ - "convert_case 0.6.0", - "darling 0.20.8", - "proc-macro2 1.0.93", - "quote 1.0.38", + "convert_case 0.8.0", + "darling 0.20.11", + "proc-macro2 1.0.101", + "quote 1.0.40", "serde", "serde_json", - "syn 2.0.96", + "syn 2.0.106", "tracing 0.1.41", ] @@ -11221,12 +12293,12 @@ dependencies = [ name = "vector-config-macros" version = "0.1.0" dependencies = [ - "darling 0.20.8", - "proc-macro2 1.0.93", - "quote 1.0.38", + "darling 0.20.11", + "proc-macro2 1.0.101", + "quote 1.0.40", "serde", "serde_derive_internals", - "syn 2.0.96", + "syn 2.0.106", "vector-config", "vector-config-common", ] @@ -11238,7 +12310,7 @@ dependencies = [ "async-trait", "base64 0.22.1", "bitmask-enum", - "bytes 1.9.0", + "bytes 1.10.1", "cfg-if", "chrono", "chrono-tz", @@ -11254,7 +12326,8 @@ dependencies = [ "headers", "http 0.2.9", "hyper-proxy", - "indexmap 2.7.0", + "indexmap 2.11.0", + "inventory", "ipnet", "metrics", "metrics-tracing-context", @@ -11266,7 +12339,7 @@ dependencies = [ "noisy_float", "openssl", "ordered-float 4.6.0", - "parking_lot", + "parking_lot 0.12.4", "pin-project", "proptest", "prost 0.12.6", @@ -11275,26 +12348,26 @@ dependencies = [ "quanta", "quickcheck", "quickcheck_macros", - "rand 0.8.5", + "rand 0.9.2", "rand_distr", "regex", "ryu", "schannel", - "security-framework", + "security-framework 3.3.0", "serde", "serde_json", - "serde_with 3.12.0", - "serde_yaml 0.9.34+deprecated", + "serde_with 3.14.0", + "serde_yaml", "similar-asserts", "smallvec", - "snafu 0.7.5", - "socket2 0.5.8", + "snafu 0.8.9", + "socket2 0.5.10", "tokio", "tokio-openssl", "tokio-stream", "tokio-test", "tokio-util", - "toml", + "toml 0.9.5", "tonic 0.11.0", "tracing 0.1.41", "tracing-subscriber", @@ -11316,6 +12389,7 @@ dependencies = [ "codecs", "enrichment", "file-source", + "file-source-common", "opentelemetry-proto", "prometheus-parser", "vector-api-client", @@ -11350,11 +12424,11 @@ dependencies = [ "futures-util", "pin-project", "proptest", - "rand 0.8.5", + "rand 0.9.2", "rand_distr", "tokio", "tokio-util", - "tower", + "tower 0.5.2", "tracing 0.1.41", "twox-hash", "vector-common", @@ -11366,14 +12440,10 @@ name = "vector-tap" version = "0.1.0" dependencies = [ "async-graphql", - "chrono", "colored", "futures 0.3.31", - "futures-util", "glob", - "portpicker", - "serde_json", - "serde_yaml 0.9.34+deprecated", + "serde_yaml", "tokio", "tokio-stream", "tokio-tungstenite 0.20.1", @@ -11425,12 +12495,13 @@ version = "0.1.0" dependencies = [ "cargo-lock", "enrichment", + "getrandom 0.2.15", "gloo-utils", "serde", - "serde-wasm-bindgen", "vector-vrl-functions", "vrl", "wasm-bindgen", + "web-sys", ] [[package]] @@ -11447,8 +12518,8 @@ checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" [[package]] name = "vrl" -version = "0.21.0" -source = "git+https://github.com/vectordotdev/vrl?branch=main#c0245e16a4ef23c3306f7ddabde0d8869e754423" +version = "0.26.0" +source = "git+https://github.com/vectordotdev/vrl.git?branch=main#a5706e81ff1dd798540831d4e2680b9ad90eb728" dependencies = [ "aes", "aes-siv", @@ -11456,8 +12527,8 @@ dependencies = [ "arbitrary", "base16", "base62", - "base64 0.22.1", - "bytes 1.9.0", + "base64-simd", + "bytes 1.10.1", "cbc", "cfb-mode", "cfg-if", @@ -11466,46 +12537,45 @@ dependencies = [ "chrono", "chrono-tz", "ciborium", - "cidr-utils", + "cidr", "clap", "codespan-reporting", "community-id", - "convert_case 0.6.0", + "convert_case 0.7.1", "crc", "crypto_secretbox", "csv", "ctr", - "data-encoding", "digest", "dns-lookup", "domain", "dyn-clone", "encoding_rs", "exitcode", - "fancy-regex", + "fancy-regex 0.15.0", "flate2", "grok", "hex", "hmac", "hostname 0.4.0", - "humantime", "iana-time-zone", "idna 1.0.3", - "indexmap 2.7.0", + "indexmap 2.11.0", "indoc", "influxdb-line-protocol", "itertools 0.14.0", + "jsonschema", "lalrpop", - "lalrpop-util 0.22.0", + "lalrpop-util", + "lz4_flex", "md-5", "mlua", - "nom", + "nom 8.0.0", + "nom-language", "ofb", - "once_cell", "onig", "ordered-float 4.6.0", "parse-size", - "paste", "peeking_take_while", "percent-encoding", "pest", @@ -11514,7 +12584,7 @@ dependencies = [ "prettytable-rs", "proptest", "proptest-derive", - "prost 0.13.3", + "prost 0.13.5", "prost-reflect", "psl", "psl-types", @@ -11523,29 +12593,36 @@ dependencies = [ "quoted_printable", "rand 0.8.5", "regex", + "reqwest 0.12.9", + "reqwest-middleware", + "reqwest-retry", "roxmltree", "rust_decimal", "rustyline", "seahash", "serde", "serde_json", + "serde_yaml", "sha-1", "sha2", "sha3", - "snafu 0.8.0", + "simdutf8", + "snafu 0.8.9", "snap", "strip-ansi-escapes", - "syslog_loose", + "syslog_loose 0.22.0", "termcolor", "thiserror 2.0.3", "tokio", "tracing 0.1.41", - "uaparser", + "ua-parser", + "unicode-segmentation", "url", "utf8-width", "uuid", "webbrowser", "woothee", + "xxhash-rust", "zstd 0.13.2", ] @@ -11557,22 +12634,11 @@ checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" [[package]] name = "vte" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5022b5fbf9407086c180e9557be968742d839e68346af7792b8592489732197" -dependencies = [ - "utf8parse", - "vte_generate_state_changes", -] - -[[package]] -name = "vte_generate_state_changes" -version = "0.1.1" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d257817081c7dffcdbab24b9e62d2def62e2ff7d00b1c20062551e6cccc145ff" +checksum = "231fdcd7ef3037e8330d8e17e61011a2c244126acc0a982f4040ac3f9f0bc077" dependencies = [ - "proc-macro2 1.0.93", - "quote 1.0.38", + "memchr", ] [[package]] @@ -11615,7 +12681,7 @@ version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4378d202ff965b011c64817db11d5829506d3404edeadb61f190d111da3f231c" dependencies = [ - "bytes 1.9.0", + "bytes 1.10.1", "futures-channel", "futures-util", "headers", @@ -11649,6 +12715,15 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasi" +version = "0.13.3+wasi-0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26816d2e1a4a36a2940b96c5296ce403917633dff8f3440e9b236ed6f6bacad2" +dependencies = [ + "wit-bindgen-rt", +] + [[package]] name = "wasite" version = "0.1.0" @@ -11675,9 +12750,9 @@ checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" dependencies = [ "bumpalo", "log", - "proc-macro2 1.0.93", - "quote 1.0.38", - "syn 2.0.96", + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", "wasm-bindgen-shared", ] @@ -11699,7 +12774,7 @@ version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" dependencies = [ - "quote 1.0.38", + "quote 1.0.40", "wasm-bindgen-macro-support", ] @@ -11709,9 +12784,9 @@ version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ - "proc-macro2 1.0.93", - "quote 1.0.38", - "syn 2.0.96", + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -11738,11 +12813,26 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wasm-timer" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be0ecb0db480561e9a7642b5d3e4187c128914e58aa84330b9493e3eb68c5e7f" +dependencies = [ + "futures 0.3.31", + "js-sys", + "parking_lot 0.11.2", + "pin-utils", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "web-sys" -version = "0.3.65" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5db499c5f66323272151db0e666cd34f78617522fb0c1604d31a27c50c206a85" +checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" dependencies = [ "js-sys", "wasm-bindgen", @@ -11764,7 +12854,7 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60b6f804e41d0852e16d2eaee61c7e4f7d3e8ffdb7b8ed85886aeb0791fe9fcd" dependencies = [ - "core-foundation", + "core-foundation 0.9.3", "home", "jni", "log", @@ -11881,12 +12971,24 @@ dependencies = [ [[package]] name = "windows" -version = "0.57.0" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12342cb4d8e3b046f3d80effd474a7a02447231330ef77d71daa6fbc40681143" +checksum = "ddf874e74c7a99773e62b1c671427abf01a425e77c3d3fb9fb1e4883ea934529" dependencies = [ - "windows-core 0.57.0", - "windows-targets 0.52.6", + "windows-collections", + "windows-core 0.60.1", + "windows-future", + "windows-link", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5467f79cc1ba3f52ebb2ed41dbb459b8e7db636cc3429458d9a852e15bc24dec" +dependencies = [ + "windows-core 0.60.1", ] [[package]] @@ -11900,56 +13002,122 @@ dependencies = [ [[package]] name = "windows-core" -version = "0.57.0" +version = "0.60.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2ed2439a290666cd67ecce2b0ffaad89c2a56b976b736e6ece670297897832d" +checksum = "ca21a92a9cae9bf4ccae5cf8368dce0837100ddf6e6d57936749e85f152f6247" dependencies = [ "windows-implement", "windows-interface", - "windows-result", - "windows-targets 0.52.6", + "windows-link", + "windows-result 0.3.1", + "windows-strings 0.3.1", +] + +[[package]] +name = "windows-future" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a787db4595e7eb80239b74ce8babfb1363d8e343ab072f2ffe901400c03349f0" +dependencies = [ + "windows-core 0.60.1", + "windows-link", ] [[package]] name = "windows-implement" -version = "0.57.0" +version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7" +checksum = "83577b051e2f49a058c308f17f273b570a6a758386fc291b5f6a934dd84e48c1" dependencies = [ - "proc-macro2 1.0.93", - "quote 1.0.38", - "syn 2.0.96", + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", ] [[package]] name = "windows-interface" -version = "0.57.0" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" +dependencies = [ + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", +] + +[[package]] +name = "windows-link" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dccfd733ce2b1753b03b6d3c65edf020262ea35e20ccdf3e288043e6dd620e3" + +[[package]] +name = "windows-numerics" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "005dea54e2f6499f2cee279b8f703b3cf3b5734a2d8d21867c8f44003182eeed" +dependencies = [ + "windows-core 0.60.1", + "windows-link", +] + +[[package]] +name = "windows-registry" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7" +checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0" dependencies = [ - "proc-macro2 1.0.93", - "quote 1.0.38", - "syn 2.0.96", + "windows-result 0.2.0", + "windows-strings 0.1.0", + "windows-targets 0.52.6", ] [[package]] name = "windows-result" -version = "0.1.2" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8" +checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-result" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06374efe858fab7e4f881500e6e86ec8bc28f9462c47e5a9941a0142ad86b189" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-service" -version = "0.7.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d24d6bcc7f734a4091ecf8d7a64c5f7d7066f45585c1861eba06449909609c8a" +checksum = "193cae8e647981c35bc947fdd57ba7928b1fa0d4a79305f6dd2dc55221ac35ac" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.9.0", "widestring 1.0.2", - "windows-sys 0.52.0", + "windows-sys 0.59.0", +] + +[[package]] +name = "windows-strings" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" +dependencies = [ + "windows-result 0.2.0", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-strings" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87fa48cc5d406560701792be122a10132491cff9d0aeb23583cc2dcafc847319" +dependencies = [ + "windows-link", ] [[package]] @@ -11988,6 +13156,15 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.2", +] + [[package]] name = "windows-targets" version = "0.42.2" @@ -12027,13 +13204,29 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm", + "windows_i686_gnullvm 0.52.6", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", "windows_x86_64_msvc 0.52.6", ] +[[package]] +name = "windows-targets" +version = "0.53.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c66f69fcc9ce11da9966ddb31a40968cad001c5bedeb5c2b82ede4253ab48aef" +dependencies = [ + "windows_aarch64_gnullvm 0.53.0", + "windows_aarch64_msvc 0.53.0", + "windows_i686_gnu 0.53.0", + "windows_i686_gnullvm 0.53.0", + "windows_i686_msvc 0.53.0", + "windows_x86_64_gnu 0.53.0", + "windows_x86_64_gnullvm 0.53.0", + "windows_x86_64_msvc 0.53.0", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.42.2" @@ -12052,6 +13245,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" + [[package]] name = "windows_aarch64_msvc" version = "0.42.2" @@ -12070,6 +13269,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" + [[package]] name = "windows_i686_gnu" version = "0.42.2" @@ -12088,12 +13293,24 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" + [[package]] name = "windows_i686_msvc" version = "0.42.2" @@ -12112,6 +13329,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_i686_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" + [[package]] name = "windows_x86_64_gnu" version = "0.42.2" @@ -12130,6 +13353,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" + [[package]] name = "windows_x86_64_gnullvm" version = "0.42.2" @@ -12148,6 +13377,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" + [[package]] name = "windows_x86_64_msvc" version = "0.42.2" @@ -12166,6 +13401,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" + [[package]] name = "winnow" version = "0.5.18" @@ -12177,9 +13418,9 @@ dependencies = [ [[package]] name = "winnow" -version = "0.6.18" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68a9bda4691f099d435ad181000724da8e5899daa10713c2d432552b9ccd3a6f" +checksum = "c06928c8748d81b05c9be96aad92e1b6ff01833332f281e8cfca3be4b35fc9ec" dependencies = [ "memchr", ] @@ -12194,30 +13435,19 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "winreg" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5" -dependencies = [ - "cfg-if", - "windows-sys 0.48.0", -] - [[package]] name = "wiremock" -version = "0.6.2" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fff469918e7ca034884c7fd8f93fe27bacb7fcb599fd879df6c7b429a29b646" +checksum = "08db1edfb05d9b3c1542e521aea074442088292f00b5f28e435c714a98f85031" dependencies = [ "assert-json-diff", - "async-trait", "base64 0.22.1", "deadpool", "futures 0.3.31", - "http 1.1.0", + "http 1.3.1", "http-body-util", - "hyper 1.4.1", + "hyper 1.7.0", "hyper-util", "log", "once_cell", @@ -12228,6 +13458,15 @@ dependencies = [ "url", ] +[[package]] +name = "wit-bindgen-rt" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c" +dependencies = [ + "bitflags 2.9.0", +] + [[package]] name = "woothee" version = "0.13.0" @@ -12266,13 +13505,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" [[package]] -name = "yaml-rust" -version = "0.4.5" +name = "xxhash-rust" +version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" -dependencies = [ - "linked-hash-map", -] +checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" [[package]] name = "yoke" @@ -12292,9 +13528,9 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" dependencies = [ - "proc-macro2 1.0.93", - "quote 1.0.38", - "syn 2.0.96", + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", "synstructure", ] @@ -12304,7 +13540,16 @@ version = "0.7.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c4061bedbb353041c12f413700357bec76df2c7e2ca8e4df8bac24c6bf68e3d" dependencies = [ - "zerocopy-derive", + "zerocopy-derive 0.7.31", +] + +[[package]] +name = "zerocopy" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b8c07a70861ce02bad1607b5753ecb2501f67847b9f9ada7c160fff0ec6300c" +dependencies = [ + "zerocopy-derive 0.8.16", ] [[package]] @@ -12313,9 +13558,20 @@ version = "0.7.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3c129550b3e6de3fd0ba67ba5c81818f9805e58b8d7fee80a3a59d2c9fc601a" dependencies = [ - "proc-macro2 1.0.93", - "quote 1.0.38", - "syn 2.0.96", + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5226bc9a9a9836e7428936cde76bb6b22feea1a8bfdbc0d241136e4d13417e25" +dependencies = [ + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", ] [[package]] @@ -12333,9 +13589,9 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" dependencies = [ - "proc-macro2 1.0.93", - "quote 1.0.38", - "syn 2.0.96", + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", "synstructure", ] @@ -12362,11 +13618,17 @@ version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ - "proc-macro2 1.0.93", - "quote 1.0.38", - "syn 2.0.96", + "proc-macro2 1.0.101", + "quote 1.0.40", + "syn 2.0.106", ] +[[package]] +name = "zlib-rs" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "626bd9fa9734751fc50d6060752170984d7053f5a39061f524cda68023d4db8a" + [[package]] name = "zstd" version = "0.12.4" diff --git a/Cargo.toml b/Cargo.toml index a23e1566be..dfa2ca2909 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,8 +1,8 @@ [package] name = "vector" -version = "0.45.0" +version = "0.50.0" authors = ["Vector Contributors "] -edition = "2021" +edition = "2024" description = "A lightweight and ultra-fast tool for building observability pipelines" homepage = "https://vector.dev" license = "MPL-2.0" @@ -12,7 +12,7 @@ default-run = "vector" autobenches = false # our benchmarks are not runnable on their own either way # Minimum supported rust version # See docs/DEVELOPING.md for policy -rust-version = "1.81" +rust-version = "1.88" [[bin]] name = "vector" @@ -41,7 +41,7 @@ path = "tests/integration/lib.rs" name = "e2e" path = "tests/e2e/mod.rs" -# CI-based builds use full release optimization. See scripts/environment/release-flags.sh. +# CI-based builds use full release optimization. See scripts/environment/release-flags.sh. # This results in roughly a 5% reduction in performance when compiling locally vs when # compiled via the CI pipeline. [profile.release] @@ -108,6 +108,7 @@ members = [ "lib/enrichment", "lib/fakedata", "lib/file-source", + "lib/file-source-common", "lib/k8s-e2e-tests", "lib/k8s-test-framework", "lib/loki-logproto", @@ -134,49 +135,79 @@ members = [ ] [workspace.dependencies] -anyhow = "1.0.95" -cfg-if = { version = "1.0.0", default-features = false } -chrono = { version = "0.4.39", default-features = false, features = ["clock", "serde"] } -chrono-tz = { version = "0.10.0", default-features = false, features = ["serde"] } -clap = { version = "4.5.26", default-features = false, features = ["derive", "error-context", "env", "help", "std", "string", "usage", "wrap_help"] } -flate2 = { version = "1.0.35", default-features = false, features = ["default"] } +anyhow = { version = "1.0.99", default-features = false, features = ["std"] } +async-stream = { version = "0.3.6", default-features = false } +async-trait = { version = "0.1.89", default-features = false } +bytes = { version = "1.10.1", default-features = false, features = ["serde"] } +base64 = { version = "0.22.1", default-features = false } +cfg-if = { version = "1.0.3", default-features = false } +chrono = { version = "0.4.41", default-features = false, features = ["clock", "serde"] } +chrono-tz = { version = "0.10.4", default-features = false, features = ["serde"] } +clap = { version = "4.5.47", default-features = false, features = ["derive", "error-context", "env", "help", "std", "string", "usage", "wrap_help"] } +colored = { version = "3.0.0", default-features = false } +crossbeam-utils = { version = "0.8.21", default-features = false } +darling = { version = "0.20.11", default-features = false, features = ["suggestions"] } +dashmap = { version = "6.1.0", default-features = false } +derivative = { version = "2.2.0", default-features = false } +flate2 = { version = "1.1.2", default-features = false, features = ["zlib-rs"] } futures = { version = "0.3.31", default-features = false, features = ["compat", "io-compat", "std"], package = "futures" } -glob = { version = "0.3.2", default-features = false } -hickory-proto = { version = "0.24.2", default-features = false, features = ["dnssec"] } -indexmap = { version = "2.7.0", default-features = false, features = ["serde", "std"] } -metrics = "0.24.1" +futures-util = { version = "0.3.29", default-features = false } +glob = { version = "0.3.3", default-features = false } +hickory-proto = { version = "0.25.2", default-features = false, features = ["dnssec-ring"] } +humantime = { version = "2.2.0", default-features = false } +indexmap = { version = "2.11.0", default-features = false, features = ["serde", "std"] } +indoc = { version = "2.0.6" } +inventory = { version = "0.3" } +itertools = { version = "0.14.0", default-features = false, features = ["use_alloc"] } +metrics = "0.24.2" metrics-tracing-context = { version = "0.17.0", default-features = false } metrics-util = { version = "0.18.0", default-features = false, features = ["registry"] } -pin-project = { version = "1.1.8", default-features = false } -proptest = { version = "1.5" } -proptest-derive = { version = "0.5.1" } +nom = { version = "8.0.0", default-features = false } +ordered-float = { version = "4.6.0", default-features = false } +paste = { version = "1.0.15" } +pin-project = { version = "1.1.10", default-features = false } +proptest = { version = "1.7" } +proptest-derive = { version = "0.6.0" } prost = { version = "0.12", default-features = false, features = ["std"] } prost-build = { version = "0.12", default-features = false } prost-reflect = { version = "0.14", features = ["serde"], default-features = false } prost-types = { version = "0.12", default-features = false } -rand = { version = "0.8.5", default-features = false, features = ["small_rng"] } -serde_json = { version = "1.0.135", default-features = false, features = ["raw_value", "std"] } -serde = { version = "1.0.217", default-features = false, features = ["alloc", "derive", "rc"] } -snafu = { version = "0.7.5", default-features = false, features = ["futures", "std"] } -tokio = { version = "1.43.0", default-features = false, features = ["full"] } -toml = { version = "0.8.19", default-features = false, features = ["display", "parse"] } +rand = { version = "0.9.2", default-features = false, features = ["small_rng", "thread_rng"] } +rand_distr = { version = "0.5.1", default-features = false } +regex = { version = "1.11.2", default-features = false, features = ["std", "perf"] } +reqwest = { version = "0.11.26", features = ["json"] } +semver = { version = "1.0.26", default-features = false, features = ["serde", "std"] } +serde = { version = "1.0.219", default-features = false, features = ["alloc", "derive", "rc"] } +serde_json = { version = "1.0.143", default-features = false, features = ["raw_value", "std"] } +serde_yaml = { version = "0.9.34", default-features = false } +snafu = { version = "0.8.9", default-features = false, features = ["futures", "std"] } +socket2 = { version = "0.5.10", default-features = false } +tempfile = "3.21.0" +tokio = { version = "1.45.1", default-features = false } +toml = { version = "0.9.5", default-features = false, features = ["serde", "display", "parse"] } tonic = { version = "0.11", default-features = false, features = ["transport", "codegen", "prost", "tls", "tls-roots", "gzip"] } tonic-build = { version = "0.11", default-features = false, features = ["transport", "prost"] } -uuid = { version = "1.11.1", features = ["v4", "v7", "serde"] } +tracing = { version = "0.1.34", default-features = false } +tracing-subscriber = { version = "0.3.20", default-features = false, features = ["fmt"] } +uuid = { version = "1.18.1", features = ["v4", "v7", "serde"] } vector-lib = { path = "lib/vector-lib", default-features = false, features = ["vrl"] } vector-config = { path = "lib/vector-config" } vector-config-common = { path = "lib/vector-config-common" } vector-config-macros = { path = "lib/vector-config-macros" } -vrl = { git = "https://github.com/vectordotdev/vrl", branch = "main", features = ["arbitrary", "cli", "test", "test_framework"] } +vrl = { git = "https://github.com/vectordotdev/vrl.git", branch = "main", features = ["arbitrary", "cli", "test", "test_framework"] } [dependencies] -pin-project.workspace = true +cfg-if.workspace = true clap.workspace = true -uuid.workspace = true -vrl.workspace = true +indoc.workspace = true +paste.workspace = true +pin-project.workspace = true proptest = { workspace = true, optional = true } proptest-derive = { workspace = true, optional = true } +semver.workspace = true snafu.workspace = true +uuid.workspace = true +vrl.workspace = true # Internal libs dnsmsg-parser = { path = "lib/dnsmsg-parser", optional = true } @@ -184,7 +215,7 @@ dnstap-parser = { path = "lib/dnstap-parser", optional = true } fakedata = { path = "lib/fakedata", optional = true } portpicker = { path = "lib/portpicker" } tracing-limit = { path = "lib/tracing-limit" } -vector-common = { path = "lib/vector-common", default-features = false} +vector-common = { path = "lib/vector-common", default-features = false } vector-lib.workspace = true vector-config.workspace = true vector-config-common.workspace = true @@ -193,20 +224,20 @@ vector-vrl-functions = { path = "lib/vector-vrl/functions" } loki-logproto = { path = "lib/loki-logproto", optional = true } # Tokio / Futures -async-stream = { version = "0.3.6", default-features = false } -async-trait = { version = "0.1.85", default-features = false } +async-stream.workspace = true +async-trait.workspace = true futures.workspace = true -tokio = { version = "1.43.0", default-features = false, features = ["full"] } +tokio = { workspace = true, features = ["full"] } tokio-openssl = { version = "0.6.5", default-features = false } tokio-stream = { version = "0.1.17", default-features = false, features = ["net", "sync", "time"] } tokio-util = { version = "0.7", default-features = false, features = ["io", "time"] } console-subscriber = { version = "0.4.1", default-features = false, optional = true } # Tracing -tracing = { version = "0.1.34", default-features = false } +tracing.workspace = true tracing-core = { version = "0.1.26", default-features = false } tracing-futures = { version = "0.2.5", default-features = false, features = ["futures-03"] } -tracing-subscriber = { version = "0.3.19", default-features = false, features = ["ansi", "env-filter", "fmt", "json", "registry", "tracing-log"] } +tracing-subscriber = { workspace = true, features = ["ansi", "env-filter", "fmt", "json", "registry", "tracing-log"] } tracing-tower = { git = "https://github.com/tokio-rs/tracing", default-features = false, rev = "e0642d949891546a3bb7e47080365ee7274f05cd" } # Metrics @@ -214,28 +245,35 @@ metrics.workspace = true metrics-tracing-context.workspace = true # AWS - Official SDK -aws-sdk-s3 = { version = "1.4.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } -aws-sdk-sqs = { version = "1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } -aws-sdk-sns = { version = "1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } -aws-sdk-cloudwatch = { version = "1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } -aws-sdk-cloudwatchlogs = { version = "1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } -aws-sdk-elasticsearch = { version = "1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } -aws-sdk-firehose = { version = "1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } -aws-sdk-kinesis = { version = "1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } -aws-sdk-secretsmanager = { version = "1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } +aws-runtime = { version = "1.5.9", optional = true } +aws-config = { version = "1.6.1", default-features = false, features = ["behavior-version-latest", "credentials-process", "sso", "rt-tokio"], optional = true } +aws-credential-types = { version = "1.2.4", default-features = false, features = ["hardcoded-credentials"], optional = true } +aws-sdk-cloudwatch = { version = "1.70.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } +aws-sdk-cloudwatchlogs = { version = "1.76.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } +aws-sdk-elasticsearch = { version = "1.67.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } +aws-sdk-firehose = { version = "1.71.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } +aws-sdk-kinesis = { version = "1.66.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } +aws-sdk-kms = { version = "1.75.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } +aws-sdk-s3 = { version = "1.15.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } +aws-sdk-secretsmanager = { version = "1.76.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } +aws-sdk-sns = { version = "1.73.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } +aws-sdk-sqs = { version = "1.64.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } +aws-types = { version = "1.3.8", default-features = false, optional = true } + # The sts crate is needed despite not being referred to anywhere in the code because we need to set the # `behavior-version-latest` feature. Without this we get a runtime panic when `auth.assume_role` authentication # is configured. -aws-sdk-sts = { version = "1.3.1", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } -aws-types = { version = "1.3.3", default-features = false, optional = true } -aws-sigv4 = { version = "1.2.6", default-features = false, features = ["sign-http"], optional = true } -aws-config = { version = "~1.0.1", default-features = false, features = ["behavior-version-latest", "credentials-process", "sso", "rt-tokio"], optional = true } -aws-credential-types = { version = "1.2.1", default-features = false, features = ["hardcoded-credentials"], optional = true } -aws-smithy-http = { version = "0.60", default-features = false, features = ["event-stream", "rt-tokio"], optional = true } -aws-smithy-types = { version = "1.2.11", default-features = false, features = ["rt-tokio"], optional = true } +aws-sdk-sts = { version = "1.73.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } + +# The `aws-sdk-sts` crate is needed despite not being referred to anywhere in the code because we need to set the +# `behavior-version-latest` feature. Without this we get a runtime panic when `auth.assume_role` authentication is configured. +aws-sigv4 = { version = "1.3.2", default-features = false, features = ["sign-http"], optional = true } + +aws-smithy-async = { version = "1.2.5", default-features = false, features = ["rt-tokio"], optional = true } +aws-smithy-http = { version = "0.62", default-features = false, features = ["event-stream", "rt-tokio"], optional = true } +aws-smithy-runtime = { version = "1.8.3", default-features = false, features = ["client", "connector-hyper-0-14-x", "rt-tokio"], optional = true } aws-smithy-runtime-api = { version = "1.7.3", default-features = false, optional = true } -aws-smithy-runtime = { version = "1.7.4", default-features = false, features = ["client", "connector-hyper-0-14-x", "rt-tokio"], optional = true } -aws-smithy-async = { version = "1.2.3", default-features = false, features = ["rt-tokio"], optional = true } +aws-smithy-types = { version = "1.2.11", default-features = false, features = ["rt-tokio"], optional = true } # Azure azure_core = { version = "0.17", default-features = false, features = ["enable_reqwest"], optional = true } @@ -244,18 +282,18 @@ azure_storage = { version = "0.17", default-features = false, optional = true } azure_storage_blobs = { version = "0.17", default-features = false, optional = true } # OpenDAL -opendal = { version = "0.45", default-features = false, features = ["native-tls", "services-webhdfs"], optional = true } +opendal = { version = "0.54", default-features = false, features = ["services-webhdfs"], optional = true } # Tower -tower = { version = "0.4.13", default-features = false, features = ["buffer", "limit", "retry", "timeout", "util", "balance", "discover"] } +tower = { version = "0.5.2", default-features = false, features = ["buffer", "limit", "retry", "timeout", "util", "balance", "discover"] } tower-http = { version = "0.4.4", default-features = false, features = ["compression-full", "decompression-gzip", "trace"] } # Serde serde.workspace = true -serde-toml-merge = { version = "0.3.8", default-features = false } -serde_bytes = { version = "0.11.15", default-features = false, features = ["std"], optional = true } +serde-toml-merge = { version = "0.3.11", default-features = false } +serde_bytes = { version = "0.11.17", default-features = false, features = ["std"], optional = true } serde_json.workspace = true -serde_with = { version = "3.12.0", default-features = false, features = ["macros", "std"] } -serde_yaml = { version = "0.9.34", default-features = false } +serde_with = { version = "3.14.0", default-features = false, features = ["macros", "std"] } +serde_yaml.workspace = true # Messagepack rmp-serde = { version = "1.3.0", default-features = false, optional = true } @@ -267,18 +305,19 @@ prost-reflect = { workspace = true, optional = true } prost-types = { workspace = true, optional = true } # GCP -goauth = { version = "0.14.0", optional = true } +goauth = { version = "0.16.0", optional = true } smpl_jwt = { version = "0.8.0", default-features = false, optional = true } # AMQP -lapin = { version = "2.5.0", default-features = false, features = ["native-tls"], optional = true } +lapin = { version = "2.5.3", default-features = false, features = ["native-tls"], optional = true } +deadpool = { version = "0.12.2", default-features = false, features = ["managed", "rt_tokio_1"], optional = true } # API -async-graphql = { version = "7.0.7", default-features = false, optional = true, features = ["chrono", "playground"] } -async-graphql-warp = { version = "7.0.7", default-features = false, optional = true } +async-graphql = { version = "7.0.17", default-features = false, optional = true, features = ["chrono", "playground"] } +async-graphql-warp = { version = "7.0.17", default-features = false, optional = true } # API client -crossterm = { version = "0.28.1", default-features = false, features = ["event-stream", "windows"], optional = true } +crossterm = { version = "0.29.0", default-features = false, features = ["event-stream", "windows"], optional = true } num-format = { version = "0.4.4", default-features = false, features = ["with-num-bigint"], optional = true } number_prefix = { version = "0.4.0", default-features = false, features = ["std"], optional = true } ratatui = { version = "0.29.0", optional = true, default-features = false, features = ["crossterm"] } @@ -288,35 +327,37 @@ ratatui = { version = "0.29.0", optional = true, default-features = false, featu hex = { version = "0.4.3", default-features = false, optional = true } # GreptimeDB -greptimedb-ingester = { git = "https://github.com/GreptimeTeam/greptimedb-ingester-rust", rev = "2e6b0c5eb6a5e7549c3100e4d356b07d15cce66d", optional = true } +greptimedb-ingester = { git = "https://github.com/GreptimeTeam/greptimedb-ingester-rust", rev = "f7243393808640f5123b0d5b7b798da591a4df6e", optional = true } # External libs arc-swap = { version = "1.7", default-features = false, optional = true } -async-compression = { version = "0.4.18", default-features = false, features = ["tokio", "gzip", "zstd"], optional = true } +async-compression = { version = "0.4.27", default-features = false, features = ["tokio", "gzip", "zstd"], optional = true } apache-avro = { version = "0.16.0", default-features = false, optional = true } axum = { version = "0.6.20", default-features = false } -base64 = { version = "0.22.1", default-features = false, optional = true } +base64 = { workspace = true, optional = true } bloomy = { version = "1.2.0", default-features = false, optional = true } -bollard = { version = "0.16.1", default-features = false, features = ["ssl", "chrono"], optional = true } -bytes = { version = "1.9.0", default-features = false, features = ["serde"] } -bytesize = { version = "1.3.0", default-features = false } +bollard = { version = "0.19.1", default-features = false, features = ["pipe", "ssl", "chrono"], optional = true } +bytes = { workspace = true, features = ["serde"] } +bytesize = { version = "2.0.1", default-features = false } chrono.workspace = true chrono-tz.workspace = true -colored = { version = "3.0.0", default-features = false } +colored.workspace = true csv = { version = "1.3", default-features = false } -databend-client = { version = "0.22.2", default-features = false, features = ["rustls"], optional = true } -derivative = { version = "2.2.0", default-features = false } +databend-client = { version = "0.28.0", default-features = false, features = ["rustls"], optional = true } +derivative.workspace = true dirs-next = { version = "2.0.0", default-features = false, optional = true } -dyn-clone = { version = "1.0.17", default-features = false } +dyn-clone = { version = "1.0.20", default-features = false } encoding_rs = { version = "0.8.35", default-features = false, features = ["serde"] } enum_dispatch = { version = "0.3.13", default-features = false } +evmap = { version = "10.0.2", default-features = false, optional = true } +evmap-derive = { version = "0.2.0", default-features = false, optional = true } exitcode = { version = "1.1.2", default-features = false } flate2.workspace = true -futures-util = { version = "0.3.29", default-features = false } +futures-util.workspace = true glob.workspace = true -governor = { version = "0.7.0", default-features = false, features = ["dashmap", "jitter", "std"], optional = true } -h2 = { version = "0.4.7", default-features = false, optional = true } -hash_hasher = { version = "2.0.0", default-features = false } +governor = { version = "0.10.0", default-features = false, features = ["dashmap", "jitter", "std"], optional = true } +h2 = { version = "0.4.11", default-features = false, optional = true } +hash_hasher = { version = "2.0.4", default-features = false } hashbrown = { version = "0.14.5", default-features = false, optional = true, features = ["ahash"] } headers = { version = "0.3.9", default-features = false } hostname = { version = "0.4.0", default-features = false } @@ -324,54 +365,55 @@ http = { version = "0.2.9", default-features = false } http-1 = { package = "http", version = "1.0", default-features = false, features = ["std"] } http-serde = "1.1.3" http-body = { version = "0.4.5", default-features = false } +humantime.workspace = true hyper = { version = "0.14.28", default-features = false, features = ["client", "runtime", "http1", "http2", "server", "stream"] } hyper-openssl = { version = "0.9.2", default-features = false } hyper-proxy = { version = "0.9.1", default-features = false, features = ["openssl-tls"] } indexmap.workspace = true -indoc = { version = "2.0.5", default-features = false } -inventory = { version = "0.3.17", default-features = false } +inventory = { version = "0.3.20", default-features = false } ipnet = { version = "2", default-features = false, optional = true, features = ["serde", "std"] } -itertools = { version = "0.14.0", default-features = false, optional = false, features = ["use_alloc"] } +itertools.workspace = true k8s-openapi = { version = "0.22.0", default-features = false, features = ["v1_26"], optional = true } kube = { version = "0.93.0", default-features = false, features = ["client", "openssl-tls", "runtime"], optional = true } -listenfd = { version = "1.0.1", default-features = false, optional = true } -lru = { version = "0.12.5", default-features = false, optional = true } -maxminddb = { version = "0.24.0", default-features = false, optional = true } +listenfd = { version = "1.0.2", default-features = false, optional = true } +lru = { version = "0.16.0", default-features = false } +maxminddb = { version = "0.26.0", default-features = false, optional = true, features = ["simdutf8"] } md-5 = { version = "0.10", default-features = false, optional = true } mongodb = { version = "2.8.2", default-features = false, features = ["tokio-runtime"], optional = true } -async-nats = { version = "0.38.0", default-features = false, features = ["ring"], optional = true } -nkeys = { version = "0.4.4", default-features = false, optional = true } -nom = { version = "7.1.3", default-features = false, optional = true } -notify = { version = "7.0.0", default-features = false, features = ["macos_fsevent"] } -openssl = { version = "0.10.68", default-features = false, features = ["vendored"] } -openssl-probe = { version = "0.1.5", default-features = false } -ordered-float = { version = "4.6.0", default-features = false } -paste = "1.0.15" +async-nats = { version = "0.42.0", default-features = false, optional = true, features = ["ring"] } +nkeys = { version = "0.4.5", default-features = false, optional = true } +nom = { workspace = true, optional = true } +notify = { version = "8.1.0", default-features = false, features = ["macos_fsevent"] } +openssl = { version = "0.10.73", default-features = false, features = ["vendored"] } +openssl-probe = { version = "0.1.6", default-features = false } +ordered-float.workspace = true percent-encoding = { version = "2.3.1", default-features = false } -postgres-openssl = { version = "0.5.0", default-features = false, features = ["runtime"], optional = true } -pulsar = { version = "6.3.0", default-features = false, features = ["tokio-runtime", "auth-oauth2", "flate2", "lz4", "snap", "zstd"], optional = true } +postgres-openssl = { version = "0.5.1", default-features = false, features = ["runtime"], optional = true } +pulsar = { version = "6.3.1", default-features = false, features = ["tokio-runtime", "auth-oauth2", "flate2", "lz4", "snap", "zstd"], optional = true } +quick-junit = { version = "0.5.1" } rand.workspace = true -rand_distr = { version = "0.4.3", default-features = false } +rand_distr.workspace = true rdkafka = { version = "0.37.0", default-features = false, features = ["curl-static", "tokio", "libz", "ssl", "zstd"], optional = true } -redis = { version = "0.24.0", default-features = false, features = ["connection-manager", "tokio-comp", "tokio-native-tls-comp"], optional = true } -regex = { version = "1.11.1", default-features = false, features = ["std", "perf"] } -roaring = { version = "0.10.10", default-features = false, features = ["std"], optional = true } +redis = { version = "0.32.4", default-features = false, features = ["connection-manager", "sentinel", "tokio-comp", "tokio-native-tls-comp"], optional = true } +regex.workspace = true +roaring = { version = "0.11.2", default-features = false, features = ["std"], optional = true } rumqttc = { version = "0.24.0", default-features = false, features = ["use-rustls"], optional = true } seahash = { version = "4.1.0", default-features = false } -semver = { version = "1.0.24", default-features = false, features = ["serde", "std"], optional = true } smallvec = { version = "1", default-features = false, features = ["union", "serde"] } snap = { version = "1.1.1", default-features = false } -socket2 = { version = "0.5.8", default-features = false } +socket2.workspace = true +sqlx = { version = "0.8.6", default-features = false, features = ["derive", "postgres", "chrono", "runtime-tokio"], optional = true } stream-cancel = { version = "0.8.2", default-features = false } -strip-ansi-escapes = { version = "0.2.0", default-features = false } +strip-ansi-escapes = { version = "0.2.1", default-features = false } syslog = { version = "6.1.1", default-features = false, optional = true } tikv-jemallocator = { version = "0.6.0", default-features = false, features = ["unprefixed_malloc_on_supported_platforms"], optional = true } -tokio-postgres = { version = "0.7.12", default-features = false, features = ["runtime", "with-chrono-0_4"], optional = true } +tokio-postgres = { version = "0.7.13", default-features = false, features = ["runtime", "with-chrono-0_4"], optional = true } tokio-tungstenite = { version = "0.20.1", default-features = false, features = ["connect"], optional = true } toml.workspace = true -tonic = { workspace = true, optional = true } hickory-proto = { workspace = true, optional = true } -typetag = { version = "0.2.19", default-features = false } +tonic = { workspace = true, optional = true } +thread_local = { version = "1.1.9", default-features = false, optional = true } +typetag = { version = "0.2.20", default-features = false } url = { version = "2.5.4", default-features = false, features = ["serde"] } warp = { version = "0.3.7", default-features = false } zstd = { version = "0.13.0", default-features = false } @@ -382,12 +424,12 @@ arr_macro = { version = "0.2.1" } heim = { git = "https://github.com/vectordotdev/heim.git", branch = "update-nix", default-features = false, features = ["disk"] } # make sure to update the external docs when the Lua version changes -mlua = { version = "0.10.2", default-features = false, features = ["lua54", "send", "vendored", "macros"], optional = true } -sysinfo = "0.32.1" +mlua = { version = "0.10.5", default-features = false, features = ["lua54", "send", "vendored", "macros"], optional = true } +sysinfo = "0.36.1" byteorder = "1.5.0" [target.'cfg(windows)'.dependencies] -windows-service = "0.7.0" +windows-service = "0.8.0" [target.'cfg(unix)'.dependencies] nix = { version = "0.26.2", default-features = false, features = ["socket", "signal"] } @@ -406,30 +448,30 @@ openssl-src = { version = "300", default-features = false, features = ["force-en [dev-dependencies] approx = "0.5.1" -assert_cmd = { version = "2.0.16", default-features = false } -aws-smithy-runtime = { version = "1.7.4", default-features = false, features = ["tls-rustls"] } +assert_cmd = { version = "2.0.17", default-features = false } +aws-smithy-runtime = { version = "1.8.3", default-features = false, features = ["tls-rustls"] } azure_core = { version = "0.17", default-features = false, features = ["enable_reqwest", "azurite_workaround"] } azure_identity = { version = "0.17", default-features = false, features = ["enable_reqwest"] } azure_storage_blobs = { version = "0.17", default-features = false, features = ["azurite_workaround"] } azure_storage = { version = "0.17", default-features = false } base64 = "0.22.1" -criterion = { version = "0.5.1", features = ["html_reports", "async_tokio"] } -itertools = { version = "0.14.0", default-features = false, features = ["use_alloc"] } -libc = "0.2.167" -similar-asserts = "1.6.0" +criterion = { version = "0.7.0", features = ["html_reports", "async_tokio"] } +itertools.workspace = true +libc = "0.2.174" +similar-asserts = "1.7.0" proptest.workspace = true quickcheck = "1.0.3" -reqwest = { version = "0.11", features = ["json"] } -rstest = { version = "0.24.0" } -tempfile = "3.15.0" +reqwest.workspace = true +rstest = { version = "0.26.1" } +tempfile.workspace = true test-generator = "0.3.1" -tokio = { version = "1.43.0", features = ["test-util"] } +tokio = { workspace = true, features = ["test-util"] } tokio-test = "0.4.4" tower-test = "0.4.0" vector-lib = { workspace = true, features = ["test"] } vrl.workspace = true -wiremock = "0.6.2" +wiremock = "0.6.4" zstd = { version = "0.13.0", default-features = false } [patch.crates-io] @@ -506,6 +548,7 @@ api-client = [ ] aws-core = [ + "aws-runtime", "aws-config", "dep:aws-credential-types", "dep:aws-sigv4", @@ -524,9 +567,10 @@ protobuf-build = ["dep:tonic-build", "dep:prost-build"] gcp = ["dep:base64", "dep:goauth", "dep:smpl_jwt"] # Enrichment Tables -enrichment-tables = ["enrichment-tables-geoip", "enrichment-tables-mmdb"] +enrichment-tables = ["enrichment-tables-geoip", "enrichment-tables-mmdb", "enrichment-tables-memory"] enrichment-tables-geoip = ["dep:maxminddb"] enrichment-tables-mmdb = ["dep:maxminddb"] +enrichment-tables-memory = ["dep:evmap", "dep:evmap-derive", "dep:thread_local"] # Codecs codecs-syslog = ["vector-lib/syslog"] @@ -558,7 +602,9 @@ sources-logs = [ "sources-kafka", "sources-kubernetes_logs", "sources-logstash", + "sources-mqtt", "sources-nats", + "sources-okta", "sources-opentelemetry", "sources-pulsar", "sources-file_descriptor", @@ -568,6 +614,7 @@ sources-logs = [ "sources-stdin", "sources-syslog", "sources-vector", + "sources-websocket", ] sources-metrics = [ "dep:prost", @@ -583,13 +630,14 @@ sources-metrics = [ "sources-static_metrics", "sources-statsd", "sources-vector", + "sources-websocket", ] sources-amqp = ["lapin"] sources-apache_metrics = ["sources-utils-http-client"] sources-aws_ecs_metrics = ["sources-utils-http-client"] sources-aws_kinesis_firehose = ["dep:base64"] -sources-aws_s3 = ["aws-core", "dep:aws-sdk-sqs", "dep:aws-sdk-s3", "dep:semver", "dep:async-compression", "sources-aws_sqs", "tokio-util/io"] +sources-aws_s3 = ["aws-core", "dep:aws-sdk-sqs", "dep:aws-sdk-s3", "dep:async-compression", "sources-aws_sqs", "tokio-util/io"] sources-aws_sqs = ["aws-core", "dep:aws-sdk-sqs"] sources-datadog_agent = ["sources-utils-http-error", "protobuf-build", "dep:prost"] sources-demo_logs = ["dep:fakedata"] @@ -599,7 +647,7 @@ sources-eventstoredb_metrics = [] sources-exec = [] sources-file = ["vector-lib/file-source"] sources-file_descriptor = ["tokio-util/io"] -sources-fluent = ["dep:base64", "sources-utils-net-tcp", "tokio-util/net", "dep:rmpv", "dep:rmp-serde", "dep:serde_bytes"] +sources-fluent = ["dep:base64", "sources-utils-net-tcp", "sources-utils-net-unix", "tokio-util/net", "dep:rmpv", "dep:rmp-serde", "dep:serde_bytes"] sources-gcp_pubsub = ["gcp", "dep:h2", "dep:prost", "dep:prost-types", "protobuf-build", "dep:tonic"] sources-heroku_logs = ["sources-utils-http", "sources-utils-http-query", "sources-http_server"] sources-host_metrics = ["heim/cpu", "heim/host", "heim/memory", "heim/net"] @@ -613,8 +661,10 @@ sources-kafka = ["dep:rdkafka"] sources-kubernetes_logs = ["vector-lib/file-source", "kubernetes", "transforms-reduce"] sources-logstash = ["sources-utils-net-tcp", "tokio-util/net"] sources-mongodb_metrics = ["dep:mongodb"] +sources-mqtt = ["dep:rumqttc"] sources-nats = ["dep:async-nats", "dep:nkeys"] sources-nginx_metrics = ["dep:nom"] +sources-okta = ["sources-utils-http-client"] sources-opentelemetry = ["dep:hex", "vector-lib/opentelemetry", "dep:prost", "dep:prost-types", "sources-http_server", "sources-utils-http", "sources-utils-http-headers", "sources-vector"] sources-postgresql_metrics = ["dep:postgres-openssl", "dep:tokio-postgres"] sources-prometheus = ["sources-prometheus-scrape", "sources-prometheus-remote-write", "sources-prometheus-pushgateway"] @@ -640,6 +690,7 @@ sources-utils-net = ["sources-utils-net-tcp", "sources-utils-net-udp", "sources- sources-utils-net-tcp = ["listenfd", "dep:ipnet"] sources-utils-net-udp = ["listenfd"] sources-utils-net-unix = [] +sources-websocket = ["dep:tokio-tungstenite"] sources-vector = ["dep:prost", "dep:tonic", "protobuf-build"] @@ -649,6 +700,7 @@ transforms-logs = [ "transforms-aws_ec2_metadata", "transforms-dedupe", "transforms-filter", + "transforms-window", "transforms-log_to_metric", "transforms-lua", "transforms-metric_to_log", @@ -662,6 +714,7 @@ transforms-logs = [ transforms-metrics = [ "transforms-aggregate", "transforms-filter", + "transforms-incremental_to_absolute", "transforms-log_to_metric", "transforms-lua", "transforms-metric_to_log", @@ -674,6 +727,8 @@ transforms-aggregate = [] transforms-aws_ec2_metadata = ["dep:arc-swap"] transforms-dedupe = ["transforms-impl-dedupe"] transforms-filter = [] +transforms-incremental_to_absolute = [] +transforms-window = [] transforms-log_to_metric = [] transforms-lua = ["dep:mlua", "vector-lib/lua"] transforms-metric_to_log = [] @@ -687,7 +742,7 @@ transforms-throttle = ["dep:governor"] # Implementations of transforms transforms-impl-sample = [] -transforms-impl-dedupe = ["dep:lru"] +transforms-impl-dedupe = [] transforms-impl-reduce = [] # Sinks @@ -721,6 +776,7 @@ sinks-logs = [ "sinks-humio", "sinks-influxdb", "sinks-kafka", + "sinks-keep", "sinks-loki", "sinks-mezmo", "sinks-mqtt", @@ -729,6 +785,7 @@ sinks-logs = [ "sinks-new_relic_logs", "sinks-opentelemetry", "sinks-papertrail", + "sinks-postgres", "sinks-pulsar", "sinks-redis", "sinks-sematext", @@ -737,6 +794,7 @@ sinks-logs = [ "sinks-vector", "sinks-webhdfs", "sinks-websocket", + "sinks-websocket-server", ] sinks-metrics = [ "sinks-appsignal", @@ -755,9 +813,9 @@ sinks-metrics = [ "sinks-splunk_hec" ] -sinks-amqp = ["lapin"] +sinks-amqp = ["deadpool", "lapin"] sinks-appsignal = [] -sinks-aws_cloudwatch_logs = ["aws-core", "dep:aws-sdk-cloudwatchlogs"] +sinks-aws_cloudwatch_logs = ["aws-core", "dep:aws-sdk-cloudwatchlogs", "dep:aws-sdk-kms"] sinks-aws_cloudwatch_metrics = ["aws-core", "dep:aws-sdk-cloudwatch"] sinks-aws_kinesis_firehose = ["aws-core", "dep:aws-sdk-firehose"] sinks-aws_kinesis_streams = ["aws-core", "dep:aws-sdk-kinesis"] @@ -787,6 +845,7 @@ sinks-http = [] sinks-humio = ["sinks-splunk_hec", "transforms-metric_to_log"] sinks-influxdb = [] sinks-kafka = ["dep:rdkafka"] +sinks-keep = [] sinks-mezmo = [] sinks-loki = ["loki-logproto"] sinks-mqtt = ["dep:rumqttc"] @@ -796,7 +855,8 @@ sinks-new_relic = [] sinks-opentelemetry = ["sinks-http"] sinks-papertrail = ["dep:syslog"] sinks-prometheus = ["dep:base64", "dep:prost", "vector-lib/prometheus"] -sinks-pulsar = ["dep:apache-avro", "dep:pulsar", "dep:lru"] +sinks-postgres = ["dep:sqlx"] +sinks-pulsar = ["dep:apache-avro", "dep:pulsar"] sinks-redis = ["dep:redis"] sinks-sematext = ["sinks-elasticsearch", "sinks-influxdb"] sinks-socket = ["sinks-utils-udp"] @@ -805,6 +865,7 @@ sinks-statsd = ["sinks-utils-udp", "tokio-util/net"] sinks-utils-udp = [] sinks-vector = ["sinks-utils-udp", "dep:tonic", "protobuf-build", "dep:prost"] sinks-websocket = ["dep:tokio-tungstenite"] +sinks-websocket-server = ["dep:tokio-tungstenite", "sources-utils-http-auth", "sources-utils-http-error", "sources-utils-http-prelude"] sinks-webhdfs = ["dep:opendal"] # Identifies that the build is a nightly build @@ -844,6 +905,7 @@ all-integration-tests = [ "nginx-integration-tests", "opentelemetry-integration-tests", "postgresql_metrics-integration-tests", + "postgres_sink-integration-tests", "prometheus-integration-tests", "pulsar-integration-tests", "redis-integration-tests", @@ -903,11 +965,12 @@ kafka-integration-tests = ["sinks-kafka", "sources-kafka"] logstash-integration-tests = ["docker", "sources-logstash"] loki-integration-tests = ["sinks-loki"] mongodb_metrics-integration-tests = ["sources-mongodb_metrics"] -mqtt-integration-tests = ["sinks-mqtt"] +mqtt-integration-tests = ["sinks-mqtt", "sources-mqtt"] nats-integration-tests = ["sinks-nats", "sources-nats"] nginx-integration-tests = ["sources-nginx_metrics"] opentelemetry-integration-tests = ["sources-opentelemetry", "dep:prost"] postgresql_metrics-integration-tests = ["sources-postgresql_metrics"] +postgres_sink-integration-tests = ["sinks-postgres"] prometheus-integration-tests = ["sinks-prometheus", "sources-prometheus", "sinks-influxdb"] pulsar-integration-tests = ["sinks-pulsar", "sources-pulsar"] redis-integration-tests = ["sinks-redis", "sources-redis"] @@ -921,13 +984,24 @@ test-utils = [] # End-to-End testing-related features all-e2e-tests = [ - "e2e-tests-datadog" + "e2e-tests-datadog", + "e2e-tests-opentelemetry" ] e2e-tests-datadog = [ "sources-datadog_agent", "sinks-datadog_logs", - "sinks-datadog_metrics" + "sinks-datadog_metrics", + "dep:async-compression" +] + +e2e-tests-opentelemetry = [ + "sources-opentelemetry", + "sinks-opentelemetry", + "sources-internal_metrics", + "transforms-remap", + "sinks-console", + "sinks-file" ] vector-api-tests = [ @@ -979,7 +1053,7 @@ remap-benches = ["transforms-remap"] transform-benches = ["transforms-filter", "transforms-dedupe", "transforms-reduce", "transforms-route"] codecs-benches = [] loki-benches = ["sinks-loki"] -enrichment-tables-benches = ["enrichment-tables-geoip", "enrichment-tables-mmdb"] +enrichment-tables-benches = ["enrichment-tables-geoip", "enrichment-tables-mmdb", "enrichment-tables-memory"] proptest = ["dep:proptest", "dep:proptest-derive", "vrl/proptest"] [[bench]] diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index fdf4e295ec..6f7e83daaa 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -46,15 +46,16 @@ async-signal,https://github.com/smol-rs/async-signal,Apache-2.0 OR MIT,John Nunl async-stream,https://github.com/tokio-rs/async-stream,MIT,Carl Lerche async-task,https://github.com/smol-rs/async-task,Apache-2.0 OR MIT,Stjepan Glavina async-trait,https://github.com/dtolnay/async-trait,MIT OR Apache-2.0,David Tolnay +atoi,https://github.com/pacman82/atoi-rs,MIT,Markus Klein atomic-waker,https://github.com/smol-rs/atomic-waker,Apache-2.0 OR MIT,"Stjepan Glavina , Contributors to futures-rs" aws-config,https://github.com/smithy-lang/smithy-rs,Apache-2.0,"AWS Rust SDK Team , Russell Cohen " aws-credential-types,https://github.com/smithy-lang/smithy-rs,Apache-2.0,AWS Rust SDK Team -aws-http,https://github.com/smithy-lang/smithy-rs,Apache-2.0,"AWS Rust SDK Team , Russell Cohen " aws-runtime,https://github.com/smithy-lang/smithy-rs,Apache-2.0,AWS Rust SDK Team aws-sdk-cloudwatch,https://github.com/awslabs/aws-sdk-rust,Apache-2.0,"AWS Rust SDK Team , Russell Cohen " aws-sdk-cloudwatchlogs,https://github.com/awslabs/aws-sdk-rust,Apache-2.0,"AWS Rust SDK Team , Russell Cohen " aws-sdk-firehose,https://github.com/awslabs/aws-sdk-rust,Apache-2.0,"AWS Rust SDK Team , Russell Cohen " aws-sdk-kinesis,https://github.com/awslabs/aws-sdk-rust,Apache-2.0,"AWS Rust SDK Team , Russell Cohen " +aws-sdk-kms,https://github.com/awslabs/aws-sdk-rust,Apache-2.0,"AWS Rust SDK Team , Russell Cohen " aws-sdk-s3,https://github.com/awslabs/aws-sdk-rust,Apache-2.0,"AWS Rust SDK Team , Russell Cohen " aws-sdk-secretsmanager,https://github.com/awslabs/aws-sdk-rust,Apache-2.0,"AWS Rust SDK Team , Russell Cohen " aws-sdk-sns,https://github.com/awslabs/aws-sdk-rust,Apache-2.0,"AWS Rust SDK Team , Russell Cohen " @@ -65,9 +66,12 @@ aws-sdk-sts,https://github.com/awslabs/aws-sdk-rust,Apache-2.0,"AWS Rust SDK Tea aws-sigv4,https://github.com/smithy-lang/smithy-rs,Apache-2.0,"AWS Rust SDK Team , David Barsky " aws-smithy-async,https://github.com/smithy-lang/smithy-rs,Apache-2.0,"AWS Rust SDK Team , John DiSanti " aws-smithy-checksums,https://github.com/smithy-lang/smithy-rs,Apache-2.0,"AWS Rust SDK Team , Zelda Hessler " +aws-smithy-compression,https://github.com/smithy-lang/smithy-rs,Apache-2.0,"AWS Rust SDK Team , Zelda Hessler " aws-smithy-eventstream,https://github.com/smithy-lang/smithy-rs,Apache-2.0,"AWS Rust SDK Team , John DiSanti " aws-smithy-http,https://github.com/smithy-lang/smithy-rs,Apache-2.0,"AWS Rust SDK Team , Russell Cohen " +aws-smithy-http-client,https://github.com/smithy-lang/smithy-rs,Apache-2.0,AWS Rust SDK Team aws-smithy-json,https://github.com/smithy-lang/smithy-rs,Apache-2.0,"AWS Rust SDK Team , John DiSanti " +aws-smithy-observability,https://github.com/awslabs/smithy-rs,Apache-2.0,AWS Rust SDK Team aws-smithy-query,https://github.com/smithy-lang/smithy-rs,Apache-2.0,"AWS Rust SDK Team , John DiSanti " aws-smithy-runtime,https://github.com/smithy-lang/smithy-rs,Apache-2.0,"AWS Rust SDK Team , Zelda Hessler " aws-smithy-runtime-api,https://github.com/smithy-lang/smithy-rs,Apache-2.0,"AWS Rust SDK Team , Zelda Hessler " @@ -81,19 +85,17 @@ azure_identity,https://github.com/azure/azure-sdk-for-rust,MIT,Microsoft Corp. azure_storage,https://github.com/azure/azure-sdk-for-rust,MIT,Microsoft Corp. azure_storage_blobs,https://github.com/azure/azure-sdk-for-rust,MIT,Microsoft Corp. backoff,https://github.com/ihrwein/backoff,MIT OR Apache-2.0,Tibor Benke -backon,https://github.com/Xuanwo/backon,Apache-2.0,Xuanwo +backon,https://github.com/Xuanwo/backon,Apache-2.0,The backon Authors backtrace,https://github.com/rust-lang/backtrace-rs,MIT OR Apache-2.0,The Rust Project Developers base16,https://github.com/thomcc/rust-base16,CC0-1.0,Thom Chiovoloni base16ct,https://github.com/RustCrypto/formats/tree/master/base16ct,Apache-2.0 OR MIT,RustCrypto Developers -base62,https://github.com/fbernier/base62,MIT,"François Bernier , Chai T. Rex , Kevin Darlington , Christopher Tarquini " +base62,https://github.com/fbernier/base62,MIT,"François Bernier , Chai T. Rex " base64,https://github.com/marshallpierce/rust-base64,MIT OR Apache-2.0,"Alice Maz , Marshall Pierce " base64,https://github.com/marshallpierce/rust-base64,MIT OR Apache-2.0,Marshall Pierce base64-simd,https://github.com/Nugine/simd,MIT,The base64-simd Authors base64ct,https://github.com/RustCrypto/formats/tree/master/base64ct,Apache-2.0 OR MIT,RustCrypto Developers bit-set,https://github.com/contain-rs/bit-set,Apache-2.0 OR MIT,Alexis Beingessner -bit-set,https://github.com/contain-rs/bit-set,MIT OR Apache-2.0,Alexis Beingessner bit-vec,https://github.com/contain-rs/bit-vec,Apache-2.0 OR MIT,Alexis Beingessner -bit-vec,https://github.com/contain-rs/bit-vec,MIT OR Apache-2.0,Alexis Beingessner bitflags,https://github.com/bitflags/bitflags,MIT OR Apache-2.0,The Rust Project Developers bitmask-enum,https://github.com/Lukas3674/rust-bitmask-enum,MIT OR Apache-2.0,Lukas3674 bitvec,https://github.com/bitvecto-rs/bitvec,MIT,The bitvec Authors @@ -102,26 +104,26 @@ block-padding,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto D blocking,https://github.com/smol-rs/blocking,Apache-2.0 OR MIT,Stjepan Glavina bloomy,https://docs.rs/bloomy/,MIT,"Aleksandr Bezobchuk , Alexis Sellier " bollard,https://github.com/fussybeaver/bollard,Apache-2.0,Bollard contributors -borsh,https://github.com/near/borsh-rs,MIT OR Apache-2.0,Near Inc -borsh-derive,https://github.com/nearprotocol/borsh,Apache-2.0,Near Inc +borrow-or-share,https://github.com/yescallop/borrow-or-share,MIT-0,Scallop Ye brotli,https://github.com/dropbox/rust-brotli,BSD-3-Clause AND MIT,"Daniel Reiter Horn , The Brotli Authors" brotli-decompressor,https://github.com/dropbox/rust-brotli-decompressor,BSD-3-Clause OR MIT,"Daniel Reiter Horn , The Brotli Authors" bson,https://github.com/mongodb/bson-rust,MIT,"Y. T. Chung , Kevin Yeh , Saghm Rossi , Patrick Freed , Isabel Atkinson , Abraham Egnor " bstr,https://github.com/BurntSushi/bstr,MIT OR Apache-2.0,Andrew Gallant bumpalo,https://github.com/fitzgen/bumpalo,MIT OR Apache-2.0,Nick Fitzgerald bytecheck,https://github.com/djkoloski/bytecheck,MIT,David Koloski +bytecount,https://github.com/llogiq/bytecount,Apache-2.0 OR MIT,"Andre Bogus , Joshua Landau " bytemuck,https://github.com/Lokathor/bytemuck,Zlib OR Apache-2.0 OR MIT,Lokathor byteorder,https://github.com/BurntSushi/byteorder,Unlicense OR MIT,Andrew Gallant bytes,https://github.com/carllerche/bytes,MIT,Carl Lerche bytes,https://github.com/tokio-rs/bytes,MIT,"Carl Lerche , Sean McArthur " bytes-utils,https://github.com/vorner/bytes-utils,Apache-2.0 OR MIT,Michal 'vorner' Vaner -bytesize,https://github.com/hyunsik/bytesize,Apache-2.0,Hyunsik Choi +bytesize,https://github.com/bytesize-rs/bytesize,Apache-2.0,"Hyunsik Choi , MrCroxx , Rob Ede " cassowary,https://github.com/dylanede/cassowary-rs,MIT OR Apache-2.0,Dylan Ede castaway,https://github.com/sagebind/castaway,MIT,Stephen M. Coakley cbc,https://github.com/RustCrypto/block-modes,MIT OR Apache-2.0,RustCrypto Developers cesu8,https://github.com/emk/cesu8-rs,Apache-2.0 OR MIT,Eric Kidd cfb-mode,https://github.com/RustCrypto/block-modes,MIT OR Apache-2.0,RustCrypto Developers -cfg-if,https://github.com/alexcrichton/cfg-if,MIT OR Apache-2.0,Alex Crichton +cfg-if,https://github.com/rust-lang/cfg-if,MIT OR Apache-2.0,Alex Crichton chacha20,https://github.com/RustCrypto/stream-ciphers,Apache-2.0 OR MIT,RustCrypto Developers chacha20poly1305,https://github.com/RustCrypto/AEADs/tree/master/chacha20poly1305,Apache-2.0 OR MIT,RustCrypto Developers charset,https://github.com/hsivonen/charset,MIT OR Apache-2.0,Henri Sivonen @@ -129,7 +131,6 @@ chrono,https://github.com/chronotope/chrono,MIT OR Apache-2.0,The chrono Authors chrono-tz,https://github.com/chronotope/chrono-tz,MIT OR Apache-2.0,The chrono-tz Authors ciborium,https://github.com/enarx/ciborium,Apache-2.0,Nathaniel McCallum cidr,https://github.com/stbuehler/rust-cidr,MIT,Stefan Bühler -cidr-utils,https://github.com/magiclen/cidr-utils,MIT,Magic Len cipher,https://github.com/RustCrypto/traits,MIT OR Apache-2.0,RustCrypto Developers clap,https://github.com/clap-rs/clap,MIT OR Apache-2.0,The clap Authors clap_builder,https://github.com/clap-rs/clap,MIT OR Apache-2.0,The clap_builder Authors @@ -147,17 +148,21 @@ concurrent-queue,https://github.com/smol-rs/concurrent-queue,Apache-2.0 OR MIT," const-oid,https://github.com/RustCrypto/formats/tree/master/const-oid,Apache-2.0 OR MIT,RustCrypto Developers const_fn,https://github.com/taiki-e/const_fn,Apache-2.0 OR MIT,The const_fn Authors convert_case,https://github.com/rutrum/convert-case,MIT,David Purdum -convert_case,https://github.com/rutrum/convert-case,MIT,Rutrum +convert_case,https://github.com/rutrum/convert-case,MIT,rutrum +cookie,https://github.com/SergioBenitez/cookie-rs,MIT OR Apache-2.0,"Sergio Benitez , Alex Crichton " cookie-factory,https://github.com/rust-bakery/cookie-factory,MIT,"Geoffroy Couprie , Pierre Chifflier " +cookie_store,https://github.com/pfernie/cookie_store,MIT OR Apache-2.0,Patrick Fernie core-foundation,https://github.com/servo/core-foundation-rs,MIT OR Apache-2.0,The Servo Project Developers +core-foundation,https://github.com/servo/core-foundation-rs,MIT OR Apache-2.0,The Servo Project Developers core2,https://github.com/bbqsrc/core2,Apache-2.0 OR MIT,Brendan Molloy cpufeatures,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers crc,https://github.com/mrhooray/crc-rs,MIT OR Apache-2.0,"Rui Hu , Akhil Velagapudi <4@4khil.com>" crc-catalog,https://github.com/akhilles/crc-catalog,MIT OR Apache-2.0,Akhil Velagapudi crc32c,https://github.com/zowens/crc32c,Apache-2.0 OR MIT,Zack Owens crc32fast,https://github.com/srijs/rust-crc32fast,MIT OR Apache-2.0,"Sam Rijs , Alex Crichton " +crc64fast-nvme,https://github.com/awesomized/crc64fast-nvme,MIT OR Apache-2.0,"The TiKV Project Developers, Don MacAskill" +critical-section,https://github.com/rust-embedded/critical-section,MIT OR Apache-2.0,The critical-section Authors crossbeam-channel,https://github.com/crossbeam-rs/crossbeam,MIT OR Apache-2.0,The crossbeam-channel Authors -crossbeam-deque,https://github.com/crossbeam-rs/crossbeam,MIT OR Apache-2.0,The crossbeam-deque Authors crossbeam-epoch,https://github.com/crossbeam-rs/crossbeam,MIT OR Apache-2.0,The crossbeam-epoch Authors crossbeam-queue,https://github.com/crossbeam-rs/crossbeam,MIT OR Apache-2.0,The crossbeam-queue Authors crossbeam-utils,https://github.com/crossbeam-rs/crossbeam,MIT OR Apache-2.0,The crossbeam-utils Authors @@ -179,6 +184,7 @@ data-encoding,https://github.com/ia0/data-encoding,MIT,Julien Cretin databend-client,https://github.com/databendlabs/bendsql,Apache-2.0,Databend Authors dbl,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers +deadpool,https://github.com/bikeshedder/deadpool,MIT OR Apache-2.0,Michael P. Jung der,https://github.com/RustCrypto/formats/tree/master/der,Apache-2.0 OR MIT,RustCrypto Developers deranged,https://github.com/jhpratt/deranged,MIT OR Apache-2.0,Jacob Pratt derivative,https://github.com/mcarton/rust-derivative,MIT OR Apache-2.0,mcarton @@ -193,13 +199,16 @@ dirs-sys-next,https://github.com/xdg-rs/dirs/tree/master/dirs-sys,MIT OR Apache- displaydoc,https://github.com/yaahc/displaydoc,MIT OR Apache-2.0,Jane Lusby dns-lookup,https://github.com/keeperofdakeys/dns-lookup,MIT OR Apache-2.0,Josh Driver doc-comment,https://github.com/GuillaumeGomez/doc-comment,MIT,Guillaume Gomez +document-features,https://github.com/slint-ui/document-features,MIT OR Apache-2.0,Slint Developers domain,https://github.com/nlnetlabs/domain,BSD-3-Clause,NLnet Labs +dotenvy,https://github.com/allan2/dotenvy,MIT,"Noemi Lapresta , Craig Hills , Mike Piccolo , Alice Maz , Sean Griffin , Adam Sharp , Arpad Borsos , Allan Zhang " dyn-clone,https://github.com/dtolnay/dyn-clone,MIT OR Apache-2.0,David Tolnay ecdsa,https://github.com/RustCrypto/signatures/tree/master/ecdsa,Apache-2.0 OR MIT,RustCrypto Developers ed25519,https://github.com/RustCrypto/signatures/tree/master/ed25519,Apache-2.0 OR MIT,RustCrypto Developers ed25519-dalek,https://github.com/dalek-cryptography/ed25519-dalek,BSD-3-Clause,"isis lovecruft , Tony Arcieri , Michael Rosenberg " either,https://github.com/bluss/either,MIT OR Apache-2.0,bluss elliptic-curve,https://github.com/RustCrypto/traits/tree/master/elliptic-curve,Apache-2.0 OR MIT,RustCrypto Developers +email_address,https://github.com/johnstonskj/rust-email_address,MIT,Simon Johnston encode_unicode,https://github.com/tormol/encode_unicode,Apache-2.0 OR MIT,Torbjørn Birch Moltu encoding_rs,https://github.com/hsivonen/encoding_rs,(Apache-2.0 OR MIT) AND BSD-3-Clause,Henri Sivonen endian-type,https://github.com/Lolirofle/endian-type,MIT,Lolirofle @@ -212,20 +221,21 @@ erased-serde,https://github.com/dtolnay/erased-serde,MIT OR Apache-2.0,David Tol errno,https://github.com/lambda-fairy/rust-errno,MIT OR Apache-2.0,Chris Wong error-chain,https://github.com/rust-lang-nursery/error-chain,MIT OR Apache-2.0,"Brian Anderson , Paul Colomiets , Colin Kiegel , Yamakaky , Andrew Gauger " error-code,https://github.com/DoumanAsh/error-code,BSL-1.0,Douman +etcetera,https://github.com/lunacookies/etcetera,MIT OR Apache-2.0,The etcetera Authors event-listener,https://github.com/smol-rs/event-listener,Apache-2.0 OR MIT,Stjepan Glavina event-listener,https://github.com/smol-rs/event-listener,Apache-2.0 OR MIT,"Stjepan Glavina , John Nunley " event-listener-strategy,https://github.com/smol-rs/event-listener-strategy,Apache-2.0 OR MIT,John Nunley +evmap,https://github.com/jonhoo/rust-evmap,MIT OR Apache-2.0,Jon Gjengset executor-trait,https://github.com/amqp-rs/executor-trait,Apache-2.0 OR MIT,Marc-Antoine Perennou exitcode,https://github.com/benwilber/exitcode,Apache-2.0,Ben Wilber fakedata_generator,https://github.com/kevingimbel/fakedata_generator,MIT,Kevin Gimbel fallible-iterator,https://github.com/sfackler/rust-fallible-iterator,MIT OR Apache-2.0,Steven Fackler fancy-regex,https://github.com/fancy-regex/fancy-regex,MIT,"Raph Levien , Robin Stocker " +fancy-regex,https://github.com/fancy-regex/fancy-regex,MIT,"Raph Levien , Robin Stocker , Keith Hall " fastrand,https://github.com/smol-rs/fastrand,Apache-2.0 OR MIT,Stjepan Glavina ff,https://github.com/zkcrypto/ff,MIT OR Apache-2.0,"Sean Bowe , Jack Grigg " fiat-crypto,https://github.com/mit-plv/fiat-crypto,MIT OR Apache-2.0 OR BSD-1-Clause,Fiat Crypto library authors -filetime,https://github.com/alexcrichton/filetime,MIT OR Apache-2.0,Alex Crichton finl_unicode,https://github.com/dahosek/finl_unicode,MIT OR Apache-2.0,The finl_unicode Authors -flagset,https://github.com/enarx/flagset,Apache-2.0,Nathaniel McCallum flate2,https://github.com/rust-lang/flate2-rs,MIT OR Apache-2.0,"Alex Crichton , Josh Triplett " float_eq,https://github.com/jtempest/float_eq-rs,MIT OR Apache-2.0,jtempest fluent-uri,https://github.com/yescallop/fluent-uri-rs,MIT,Scallop Ye @@ -233,6 +243,7 @@ flume,https://github.com/zesterer/flume,Apache-2.0 OR MIT,Joshua Barretto foldhash,https://github.com/orlp/foldhash,Zlib,Orson Peters foreign-types,https://github.com/sfackler/foreign-types,MIT OR Apache-2.0,Steven Fackler +fraction,https://github.com/dnsl48/fraction,MIT OR Apache-2.0,dnsl48 fsevent-sys,https://github.com/octplane/fsevent-rust/tree/master/fsevent-sys,MIT,Pierre Baillet fslock,https://github.com/brunoczim/fslock,MIT,The fslock Authors funty,https://github.com/myrrlyn/funty,MIT,myrrlyn @@ -241,6 +252,7 @@ futures,https://github.com/rust-lang/futures-rs,MIT OR Apache-2.0,The futures Au futures-channel,https://github.com/rust-lang/futures-rs,MIT OR Apache-2.0,The futures-channel Authors futures-core,https://github.com/rust-lang/futures-rs,MIT OR Apache-2.0,The futures-core Authors futures-executor,https://github.com/rust-lang/futures-rs,MIT OR Apache-2.0,The futures-executor Authors +futures-intrusive,https://github.com/Matthias247/futures-intrusive,MIT OR Apache-2.0,Matthias Einwag futures-io,https://github.com/rust-lang/futures-rs,MIT OR Apache-2.0,The futures-io Authors futures-lite,https://github.com/smol-rs/futures-lite,Apache-2.0 OR MIT,"Stjepan Glavina , Contributors to futures-rs" futures-macro,https://github.com/rust-lang/futures-rs,MIT OR Apache-2.0,The futures-macro Authors @@ -252,6 +264,7 @@ generic-array,https://github.com/fizyk20/generic-array,MIT,"Bartłomiej Kamińsk getrandom,https://github.com/rust-random/getrandom,MIT OR Apache-2.0,The Rand Project Developers gimli,https://github.com/gimli-rs/gimli,MIT OR Apache-2.0,The gimli Authors glob,https://github.com/rust-lang/glob,MIT OR Apache-2.0,The Rust Project Developers +gloo-timers,https://github.com/rustwasm/gloo/tree/master/crates/timers,MIT OR Apache-2.0,Rust and WebAssembly Working Group goauth,https://github.com/durch/rust-goauth,MIT,Drazen Urch governor,https://github.com/boinkor-net/governor,MIT,Andreas Fuchs graphql-introspection-query,https://github.com/graphql-rust/graphql-client,Apache-2.0 OR MIT,Tom Houlé @@ -261,12 +274,14 @@ graphql_client_codegen,https://github.com/graphql-rust/graphql-client,Apache-2.0 graphql_query_derive,https://github.com/graphql-rust/graphql-client,Apache-2.0 OR MIT,Tom Houlé greptime-proto,https://github.com/GreptimeTeam/greptime-proto,Apache-2.0,The greptime-proto Authors greptimedb-ingester,https://github.com/GreptimeTeam/greptimedb-ingester-rust,Apache-2.0,The greptimedb-ingester Authors -grok,https://github.com/daschl/grok,Apache-2.0,Michael Nitschinger +grok,https://github.com/mmastrac/grok,Apache-2.0,"Matt Mastracci , Michael Nitschinger " group,https://github.com/zkcrypto/group,MIT OR Apache-2.0,"Sean Bowe , Jack Grigg " h2,https://github.com/hyperium/h2,MIT,"Carl Lerche , Sean McArthur " half,https://github.com/starkat99/half-rs,MIT OR Apache-2.0,Kathryn Long -hash_hasher,https://github.com/Fraser999/Hash-Hasher,Apache-2.0 OR MIT,Fraser Hutchison +hash_hasher,https://github.com/Fraser999/Hash-Hasher,Apache-2.0 OR MIT,Fraser Hutchison +hashbag,https://github.com/jonhoo/hashbag,MIT OR Apache-2.0,Jon Gjengset hashbrown,https://github.com/rust-lang/hashbrown,MIT OR Apache-2.0,Amanieu d'Antras +hashlink,https://github.com/kyren/hashlink,MIT OR Apache-2.0,kyren headers,https://github.com/hyperium/headers,MIT,Sean McArthur heck,https://github.com/withoutboats/heck,MIT OR Apache-2.0,The heck Authors heck,https://github.com/withoutboats/heck,MIT OR Apache-2.0,Without Boats @@ -285,7 +300,7 @@ http-serde,https://gitlab.com/kornelski/http-serde,Apache-2.0 OR MIT,Kornel httparse,https://github.com/seanmonstar/httparse,MIT OR Apache-2.0,Sean McArthur httpdate,https://github.com/pyfisch/httpdate,MIT OR Apache-2.0,Pyfisch -humantime,https://github.com/tailhook/humantime,MIT OR Apache-2.0,Paul Colomiets +humantime,https://github.com/chronotope/humantime,MIT OR Apache-2.0,The humantime Authors hyper,https://github.com/hyperium/hyper,MIT,Sean McArthur hyper-named-pipe,https://github.com/fussybeaver/hyper-named-pipe,Apache-2.0,The hyper-named-pipe Authors hyper-openssl,https://github.com/sfackler/hyper-openssl,MIT OR Apache-2.0,Steven Fackler @@ -294,7 +309,7 @@ hyper-rustls,https://github.com/rustls/hyper-rustls,Apache-2.0 OR ISC OR MIT,The hyper-timeout,https://github.com/hjr3/hyper-timeout,MIT OR Apache-2.0,Herman J. Radtke III hyper-tls,https://github.com/hyperium/hyper-tls,MIT OR Apache-2.0,Sean McArthur hyper-util,https://github.com/hyperium/hyper-util,MIT,Sean McArthur -hyperlocal-next,https://github.com/softprops/hyperlocal,MIT,softprops +hyperlocal,https://github.com/softprops/hyperlocal,MIT,softprops iana-time-zone,https://github.com/strawlab/iana-time-zone,MIT OR Apache-2.0,"Andrew Straw , René Kijewski , Ryan Lopopolo " iana-time-zone-haiku,https://github.com/strawlab/iana-time-zone,MIT OR Apache-2.0,René Kijewski icu_collections,https://github.com/unicode-org/icu4x,Unicode-3.0,The ICU4X Project Developers @@ -321,6 +336,7 @@ instability,https://github.com/ratatui-org/instability,MIT,"Stephen M. Coakley < instant,https://github.com/sebcrozet/instant,BSD-3-Clause,sebcrozet inventory,https://github.com/dtolnay/inventory,MIT OR Apache-2.0,David Tolnay io-lifetimes,https://github.com/sunfishcode/io-lifetimes,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,Dan Gohman +io-uring,https://github.com/tokio-rs/io-uring,MIT OR Apache-2.0,quininer iovec,https://github.com/carllerche/iovec,MIT OR Apache-2.0,Carl Lerche ipconfig,https://github.com/liranringel/ipconfig,MIT OR Apache-2.0,Liran Ringel ipnet,https://github.com/krisprice/ipnet,MIT OR Apache-2.0,Kris Price @@ -335,6 +351,7 @@ js-sys,https://github.com/rustwasm/wasm-bindgen/tree/master/crates/js-sys,MIT OR json-patch,https://github.com/idubrov/json-patch,MIT OR Apache-2.0,Ivan Dubrov jsonpath-rust,https://github.com/besok/jsonpath-rust,MIT,BorisZhguchev jsonptr,https://github.com/chanced/jsonptr,MIT OR Apache-2.0,chance dinkins +jsonschema,https://github.com/Stranger6667/jsonschema,MIT,Dmitry Dygalo k8s-openapi,https://github.com/Arnavion/k8s-openapi,Apache-2.0,Arnav Singh keccak,https://github.com/RustCrypto/sponges/tree/master/keccak,Apache-2.0 OR MIT,RustCrypto Developers kqueue,https://gitlab.com/rust-kqueue/rust-kqueue,MIT,William Orr @@ -347,17 +364,21 @@ lazy_static,https://github.com/rust-lang-nursery/lazy-static.rs,MIT OR Apache-2. libc,https://github.com/rust-lang/libc,MIT OR Apache-2.0,The Rust Project Developers libflate,https://github.com/sile/libflate,MIT,Takeru Ohta libm,https://github.com/rust-lang/libm,MIT OR Apache-2.0,Jorge Aparicio +libsqlite3-sys,https://github.com/rusqlite/rusqlite,MIT,The rusqlite developers +libz-rs-sys,https://github.com/trifectatechfoundation/zlib-rs,Zlib,The libz-rs-sys Authors libz-sys,https://github.com/rust-lang/libz-sys,MIT OR Apache-2.0,"Alex Crichton , Josh Triplett , Sebastian Thiel " linked-hash-map,https://github.com/contain-rs/linked-hash-map,MIT OR Apache-2.0,"Stepan Koltsov , Andrew Paseltiner " linked_hash_set,https://github.com/alexheretic/linked-hash-set,Apache-2.0,Alex Butler linux-raw-sys,https://github.com/sunfishcode/linux-raw-sys,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,Dan Gohman -listenfd,https://github.com/mitsuhiko/rust-listenfd,Apache-2.0,Armin Ronacher +listenfd,https://github.com/mitsuhiko/listenfd,Apache-2.0,Armin Ronacher litemap,https://github.com/unicode-org/icu4x,Unicode-3.0,The ICU4X Project Developers +litrs,https://github.com/LukasKalbertodt/litrs,MIT OR Apache-2.0,Lukas Kalbertodt lockfree-object-pool,https://github.com/EVaillant/lockfree-object-pool,BSL-1.0,Etienne Vaillant log,https://github.com/rust-lang/log,MIT OR Apache-2.0,The Rust Project Developers lru,https://github.com/jeromefroe/lru-rs,MIT,Jerome Froelich lru-cache,https://github.com/contain-rs/lru-cache,MIT OR Apache-2.0,Stepan Koltsov lz4,https://github.com/10xGenomics/lz4-rs,MIT,"Jens Heyens , Artem V. Navrotskiy , Patrick Marks " +lz4_flex,https://github.com/pseitz/lz4_flex,MIT,"Pascal Seitz , Arthur Silva , ticki " macaddr,https://github.com/svartalf/rust-macaddr,Apache-2.0 OR MIT,svartalf mach,https://github.com/fitzgen/mach,BSD-2-Clause,"Nick Fitzgerald , David Cuddeback , Gonzalo Brito Gadeschi " malloc_buf,https://github.com/SSheldon/malloc_buf,MIT,Steven Sheldon @@ -376,9 +397,10 @@ mime,https://github.com/hyperium/mime,MIT OR Apache-2.0,Sean McArthur minimal-lexical,https://github.com/Alexhuszagh/minimal-lexical,MIT OR Apache-2.0,Alex Huszagh miniz_oxide,https://github.com/Frommi/miniz_oxide/tree/master/miniz_oxide,MIT OR Zlib OR Apache-2.0,"Frommi , oyvindln " +miniz_oxide,https://github.com/Frommi/miniz_oxide/tree/master/miniz_oxide,MIT OR Zlib OR Apache-2.0,"Frommi , oyvindln , Rich Geldreich richgel99@gmail.com" mio,https://github.com/tokio-rs/mio,MIT,"Carl Lerche , Thomas de Zeeuw , Tokio Contributors " -mlua,https://github.com/khvzak/mlua,MIT,"Aleksandr Orlenko , kyren " -mlua-sys,https://github.com/khvzak/mlua,MIT,Aleksandr Orlenko +mlua,https://github.com/mlua-rs/mlua,MIT,"Aleksandr Orlenko , kyren " +mlua-sys,https://github.com/mlua-rs/mlua,MIT,Aleksandr Orlenko mlua_derive,https://github.com/khvzak/mlua,MIT,Aleksandr Orlenko moka,https://github.com/moka-rs/moka,MIT OR Apache-2.0,The moka Authors mongodb,https://github.com/mongodb/mongo-rust-driver,Apache-2.0,"Saghm Rossi , Patrick Freed , Isabel Atkinson , Abraham Egnor , Kaitlin Mahar " @@ -389,36 +411,44 @@ netlink-packet-core,https://github.com/rust-netlink/netlink-packet-core,MIT,Core netlink-packet-sock-diag,https://github.com/rust-netlink/netlink-packet-sock-diag,MIT,"Flier Lu , Corentin Henry " netlink-packet-utils,https://github.com/rust-netlink/netlink-packet-utils,MIT,Corentin Henry netlink-sys,https://github.com/rust-netlink/netlink-sys,MIT,Corentin Henry +newtype-uuid,https://github.com/oxidecomputer/newtype-uuid,MIT OR Apache-2.0,The newtype-uuid Authors nibble_vec,https://github.com/michaelsproul/rust_nibble_vec,MIT,Michael Sproul nix,https://github.com/nix-rust/nix,MIT,The nix-rust Project Developers nkeys,https://github.com/wasmcloud/nkeys,Apache-2.0,wasmCloud Team no-proxy,https://github.com/jdrouet/no-proxy,MIT,Jérémie Drouet -no-std-compat,https://gitlab.com/jD91mZM2/no-std-compat,MIT,jD91mZM2 +nohash,https://github.com/tetcoin/nohash,Apache-2.0 OR MIT,Parity Technologies nom,https://github.com/Geal/nom,MIT,contact@geoffroycouprie.com +nom,https://github.com/rust-bakery/nom,MIT,contact@geoffroycouprie.com nonzero_ext,https://github.com/antifuchs/nonzero_ext,Apache-2.0,Andreas Fuchs notify,https://github.com/notify-rs/notify,CC0-1.0,"Félix Saparelli , Daniel Faust , Aron Heinecke " notify-types,https://github.com/notify-rs/notify,MIT OR Apache-2.0,Daniel Faust ntapi,https://github.com/MSxDOS/ntapi,Apache-2.0 OR MIT,MSxDOS nu-ansi-term,https://github.com/nushell/nu-ansi-term,MIT,"ogham@bsago.me, Ryan Scheel (Havvy) , Josh Triplett , The Nushell Project Developers" nuid,https://github.com/casualjim/rs-nuid,Apache-2.0,Ivan Porto Carrero +num,https://github.com/rust-num/num,MIT OR Apache-2.0,The Rust Project Developers num-bigint,https://github.com/rust-num/num-bigint,MIT OR Apache-2.0,The Rust Project Developers num-bigint-dig,https://github.com/dignifiedquire/num-bigint,MIT OR Apache-2.0,"dignifiedquire , The Rust Project Developers" +num-cmp,https://github.com/lifthrasiir/num-cmp,MIT OR Apache-2.0,Kang Seonghoon +num-complex,https://github.com/rust-num/num-complex,MIT OR Apache-2.0,The Rust Project Developers num-conv,https://github.com/jhpratt/num-conv,MIT OR Apache-2.0,Jacob Pratt num-format,https://github.com/bcmyers/num-format,MIT OR Apache-2.0,Brian Myers num-integer,https://github.com/rust-num/num-integer,MIT OR Apache-2.0,The Rust Project Developers num-iter,https://github.com/rust-num/num-iter,MIT OR Apache-2.0,The Rust Project Developers num-rational,https://github.com/rust-num/num-rational,MIT OR Apache-2.0,The Rust Project Developers num-traits,https://github.com/rust-num/num-traits,MIT OR Apache-2.0,The Rust Project Developers +num_cpus,https://github.com/seanmonstar/num_cpus,MIT OR Apache-2.0,Sean McArthur num_enum,https://github.com/illicitonion/num_enum,BSD-3-Clause OR MIT OR Apache-2.0,"Daniel Wagner-Hall , Daniel Henry-Mantilla , Vincent Esche " num_threads,https://github.com/jhpratt/num_threads,MIT OR Apache-2.0,Jacob Pratt number_prefix,https://github.com/ogham/rust-number-prefix,MIT,Benjamin Sago oauth2,https://github.com/ramosbugs/oauth2-rs,MIT OR Apache-2.0,"Alex Crichton , Florin Lipan , David A. Ramos " objc,http://github.com/SSheldon/rust-objc,MIT,Steven Sheldon +objc2-core-foundation,https://github.com/madsmtm/objc2,Zlib OR Apache-2.0 OR MIT,The objc2-core-foundation Authors +objc2-io-kit,https://github.com/madsmtm/objc2,Zlib OR Apache-2.0 OR MIT,The objc2-io-kit Authors object,https://github.com/gimli-rs/object,Apache-2.0 OR MIT,The object Authors octseq,https://github.com/NLnetLabs/octets/,BSD-3-Clause,NLnet Labs ofb,https://github.com/RustCrypto/block-modes,MIT OR Apache-2.0,RustCrypto Developers once_cell,https://github.com/matklad/once_cell,MIT OR Apache-2.0,Aleksey Kladov -onig,http://github.com/iwillspeak/rust-onig,MIT,"Will Speak , Ivan Ivashchenko " +onig,https://github.com/iwillspeak/rust-onig,MIT,"Will Speak , Ivan Ivashchenko " opaque-debug,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers opendal,https://github.com/apache/opendal,Apache-2.0,Apache OpenDAL openidconnect,https://github.com/ramosbugs/openidconnect-rs,MIT,David A. Ramos @@ -428,12 +458,12 @@ openssl-probe,https://github.com/alexcrichton/openssl-probe,MIT OR Apache-2.0,Al openssl-sys,https://github.com/sfackler/rust-openssl,MIT,"Alex Crichton , Steven Fackler " ordered-float,https://github.com/reem/rust-ordered-float,MIT,"Jonathan Reem , Matt Brubeck " outref,https://github.com/Nugine/outref,MIT,The outref Authors -overload,https://github.com/danaugrs/overload,MIT,Daniel Salvadori -owo-colors,https://github.com/jam1garner/owo-colors,MIT,jam1garner <8260240+jam1garner@users.noreply.github.com> +owo-colors,https://github.com/owo-colors/owo-colors,MIT,jam1garner <8260240+jam1garner@users.noreply.github.com> p256,https://github.com/RustCrypto/elliptic-curves/tree/master/p256,Apache-2.0 OR MIT,RustCrypto Developers p384,https://github.com/RustCrypto/elliptic-curves/tree/master/p384,Apache-2.0 OR MIT,"RustCrypto Developers, Frank Denis " pad,https://github.com/ogham/rust-pad,MIT,Ben S parking,https://github.com/smol-rs/parking,Apache-2.0 OR MIT,"Stjepan Glavina , The Rust Project Developers" +parking_lot,https://github.com/Amanieu/parking_lot,Apache-2.0 OR MIT,Amanieu d'Antras parking_lot,https://github.com/Amanieu/parking_lot,MIT OR Apache-2.0,Amanieu d'Antras parse-size,https://github.com/kennytm/parse-size,MIT,kennytm passt,https://github.com/kevingimbel/passt,MIT OR Apache-2.0,Kevin Gimbel @@ -466,15 +496,12 @@ prettydiff,https://github.com/romankoblov/prettydiff,MIT,Roman Koblov -proc-macro-error,https://gitlab.com/CreepySkeleton/proc-macro-error,MIT OR Apache-2.0,CreepySkeleton proc-macro-error-attr2,https://github.com/GnomedDev/proc-macro-error-2,MIT OR Apache-2.0,"CreepySkeleton , GnomedDev " proc-macro-error2,https://github.com/GnomedDev/proc-macro-error-2,MIT OR Apache-2.0,"CreepySkeleton , GnomedDev " proc-macro-hack,https://github.com/dtolnay/proc-macro-hack,MIT OR Apache-2.0,David Tolnay proc-macro2,https://github.com/dtolnay/proc-macro2,MIT OR Apache-2.0,"David Tolnay , Alex Crichton " proptest,https://github.com/proptest-rs/proptest,MIT OR Apache-2.0,Jason Lingle -prost,https://github.com/tokio-rs/prost,Apache-2.0,"Dan Burkert , Lucio Franco " prost,https://github.com/tokio-rs/prost,Apache-2.0,"Dan Burkert , Lucio Franco , Casper Meijn , Tokio Contributors " -prost-derive,https://github.com/tokio-rs/prost,Apache-2.0,"Dan Burkert , Lucio Franco , Tokio Contributors " prost-reflect,https://github.com/andrewhickman/prost-reflect,MIT OR Apache-2.0,Andrew Hickman psl,https://github.com/addr-rs/psl,MIT OR Apache-2.0,rushmorem psl-types,https://github.com/addr-rs/psl-types,MIT OR Apache-2.0,rushmorem @@ -484,36 +511,41 @@ pulsar,https://github.com/streamnative/pulsar-rs,MIT OR Apache-2.0,"Colin Stearn quad-rand,https://github.com/not-fl3/quad-rand,MIT,not-fl3 quanta,https://github.com/metrics-rs/quanta,MIT,Toby Lawrence quick-error,http://github.com/tailhook/quick-error,MIT OR Apache-2.0,"Paul Colomiets , Colin Kiegel " +quick-junit,https://github.com/nextest-rs/quick-junit,Apache-2.0 OR MIT,The quick-junit Authors quick-xml,https://github.com/tafia/quick-xml,MIT,The quick-xml Authors quickcheck,https://github.com/BurntSushi/quickcheck,Unlicense OR MIT,Andrew Gallant +quinn,https://github.com/quinn-rs/quinn,MIT OR Apache-2.0,The quinn Authors +quinn-proto,https://github.com/quinn-rs/quinn,MIT OR Apache-2.0,The quinn-proto Authors +quinn-udp,https://github.com/quinn-rs/quinn,MIT OR Apache-2.0,The quinn-udp Authors quote,https://github.com/dtolnay/quote,MIT OR Apache-2.0,David Tolnay quoted_printable,https://github.com/staktrace/quoted-printable,0BSD,Kartikaya Gupta radium,https://github.com/bitvecto-rs/radium,MIT,"Nika Layzell , myrrlyn " radix_trie,https://github.com/michaelsproul/rust_radix_trie,MIT,Michael Sproul rand,https://github.com/rust-random/rand,MIT OR Apache-2.0,"The Rand Project Developers, The Rust Project Developers" rand_chacha,https://github.com/rust-random/rand,MIT OR Apache-2.0,"The Rand Project Developers, The Rust Project Developers, The CryptoCorrosion Contributors" -rand_distr,https://github.com/rust-random/rand,MIT OR Apache-2.0,The Rand Project Developers +rand_distr,https://github.com/rust-random/rand_distr,MIT OR Apache-2.0,The Rand Project Developers rand_hc,https://github.com/rust-random/rand,MIT OR Apache-2.0,The Rand Project Developers rand_xorshift,https://github.com/rust-random/rngs,MIT OR Apache-2.0,"The Rand Project Developers, The Rust Project Developers" ratatui,https://github.com/ratatui/ratatui,MIT,"Florian Dehau , The Ratatui Developers" raw-cpuid,https://github.com/gz/rust-cpuid,MIT,Gerd Zellweger raw-window-handle,https://github.com/rust-windowing/raw-window-handle,MIT OR Apache-2.0 OR Zlib,Osspial -rayon,https://github.com/rayon-rs/rayon,MIT OR Apache-2.0,"Niko Matsakis , Josh Stone " rdkafka,https://github.com/fede1024/rust-rdkafka,MIT,Federico Giraud redis,https://github.com/redis-rs/redis-rs,BSD-3-Clause,The redis Authors redox_syscall,https://gitlab.redox-os.org/redox-os/syscall,MIT,Jeremy Soller redox_users,https://gitlab.redox-os.org/redox-os/users,MIT,"Jose Narvaez , Wesley Hershberger " +ref-cast,https://github.com/dtolnay/ref-cast,MIT OR Apache-2.0,David Tolnay regex,https://github.com/rust-lang/regex,MIT OR Apache-2.0,"The Rust Project Developers, Andrew Gallant " -regex-automata,https://github.com/BurntSushi/regex-automata,Unlicense OR MIT,Andrew Gallant regex-automata,https://github.com/rust-lang/regex/tree/master/regex-automata,MIT OR Apache-2.0,"The Rust Project Developers, Andrew Gallant " +regex-filtered,https://github.com/ua-parser/uap-rust,BSD-3-Clause,The regex-filtered Authors regex-lite,https://github.com/rust-lang/regex/tree/master/regex-lite,MIT OR Apache-2.0,"The Rust Project Developers, Andrew Gallant " -regex-syntax,https://github.com/rust-lang/regex,MIT OR Apache-2.0,The Rust Project Developers regex-syntax,https://github.com/rust-lang/regex/tree/master/regex-syntax,MIT OR Apache-2.0,"The Rust Project Developers, Andrew Gallant " rend,https://github.com/djkoloski/rend,MIT,David Koloski reqwest,https://github.com/seanmonstar/reqwest,MIT OR Apache-2.0,Sean McArthur +reqwest-middleware,https://github.com/TrueLayer/reqwest-middleware,MIT OR Apache-2.0,Rodrigo Gryzinski resolv-conf,http://github.com/tailhook/resolv-conf,MIT OR Apache-2.0,paul@colomiets.name +retry-policies,https://github.com/TrueLayer/retry-policies,MIT OR Apache-2.0,Luca Palmieri rfc6979,https://github.com/RustCrypto/signatures/tree/master/rfc6979,Apache-2.0 OR MIT,RustCrypto Developers -ring,https://github.com/briansmith/ring,ISC AND Custom,Brian Smith +ring,https://github.com/briansmith/ring,Apache-2.0 AND ISC,The ring Authors rkyv,https://github.com/rkyv/rkyv,MIT,David Koloski rle-decode-fast,https://github.com/WanzenBug/rle-decode-helper,MIT OR Apache-2.0,Moritz Wanzenböck rmp,https://github.com/3Hren/msgpack-rust,MIT,Evgeny Safronov @@ -544,6 +576,7 @@ same-file,https://github.com/BurntSushi/same-file,Unlicense OR MIT,Andrew Gallan sasl2-sys,https://github.com/MaterializeInc/rust-sasl,Apache-2.0,"Materialize, Inc." scan_fmt,https://github.com/wlentz/scan_fmt,MIT,wlentz schannel,https://github.com/steffengy/schannel-rs,MIT,"Steven Fackler , Steffen Butzer " +schemars,https://github.com/GREsau/schemars,MIT,Graham Esau scoped-tls,https://github.com/alexcrichton/scoped-tls,MIT OR Apache-2.0,Alex Crichton scopeguard,https://github.com/bluss/scopeguard,MIT OR Apache-2.0,bluss sct,https://github.com/rustls/sct.rs,Apache-2.0 OR ISC OR MIT,Joseph Birr-Pixton @@ -593,6 +626,7 @@ spin,https://github.com/mvdnes/spin-rs,MIT,"Mathijs van de Nes , John Ericson , Joshua Barretto " spinning_top,https://github.com/rust-osdev/spinning_top,MIT OR Apache-2.0,Philipp Oppermann spki,https://github.com/RustCrypto/formats/tree/master/spki,Apache-2.0 OR MIT,RustCrypto Developers +sqlx,https://github.com/launchbadge/sqlx,MIT OR Apache-2.0,"Ryan Leckey , Austin Bonander , Chloe Ross , Daniel Akhterov " stable_deref_trait,https://github.com/storyyeller/stable_deref_trait,MIT OR Apache-2.0,Robert Grosse static_assertions,https://github.com/nvzqz/static-assertions-rs,MIT OR Apache-2.0,Nikolai Vazquez static_assertions_next,https://github.com/scuffletv/static-assertions,MIT OR Apache-2.0,Nikolai Vazquez @@ -605,7 +639,6 @@ strum,https://github.com/Peternator7/strum,MIT,Peter Glotfelty , Henry de Valence " supports-color,https://github.com/zkat/supports-color,Apache-2.0,Kat Marchán syn,https://github.com/dtolnay/syn,MIT OR Apache-2.0,David Tolnay -syn_derive,https://github.com/Kyuuhachi/syn_derive,MIT OR Apache-2.0,Kyuuhachi sync_wrapper,https://github.com/Actyx/sync_wrapper,Apache-2.0,Actyx AG synstructure,https://github.com/mystor/synstructure,MIT,Nika Layzell sysinfo,https://github.com/GuillaumeGomez/sysinfo,MIT,Guillaume Gomez @@ -638,8 +671,13 @@ tokio-retry,https://github.com/srijs/rust-tokio-retry,MIT,Sam Rijs , Alexey Galakhov " tokio-websockets,https://github.com/Gelbpunkt/tokio-websockets,MIT,The tokio-websockets Authors -toml,https://github.com/toml-rs/toml,MIT OR Apache-2.0,Alex Crichton +toml,https://github.com/toml-rs/toml,MIT OR Apache-2.0,The toml Authors +toml_datetime,https://github.com/toml-rs/toml,MIT OR Apache-2.0,The toml_datetime Authors toml_edit,https://github.com/toml-rs/toml,MIT OR Apache-2.0,"Andronik Ordian , Ed Page " +toml_edit,https://github.com/toml-rs/toml,MIT OR Apache-2.0,The toml_edit Authors +toml_parser,https://github.com/toml-rs/toml,MIT OR Apache-2.0,The toml_parser Authors +toml_write,https://github.com/toml-rs/toml,MIT OR Apache-2.0,The toml_write Authors +toml_writer,https://github.com/toml-rs/toml,MIT OR Apache-2.0,The toml_writer Authors tonic,https://github.com/hyperium/tonic,MIT,Lucio Franco tower,https://github.com/tower-rs/tower,MIT,Tower Maintainers tower-http,https://github.com/tower-rs/tower-http,MIT,Tower Maintainers @@ -661,7 +699,7 @@ typed-builder,https://github.com/idanarye/rust-typed-builder,MIT OR Apache-2.0," typenum,https://github.com/paholg/typenum,MIT OR Apache-2.0,"Paho Lurie-Gregg , Andre Bogus " typetag,https://github.com/dtolnay/typetag,MIT OR Apache-2.0,David Tolnay tz-rs,https://github.com/x-hgg-x/tz-rs,MIT OR Apache-2.0,x-hgg-x -uaparser,https://github.com/davidarmstronglewis/uap-rs,MIT,David Lewis +ua-parser,https://github.com/ua-parser/uap-rust,Apache-2.0,The ua-parser Authors ucd-trie,https://github.com/BurntSushi/ucd-generate,MIT OR Apache-2.0,Andrew Gallant unarray,https://github.com/cameron1024/unarray,MIT OR Apache-2.0,The unarray Authors unicase,https://github.com/seanmonstar/unicase,MIT OR Apache-2.0,Sean McArthur @@ -683,18 +721,19 @@ utf16_iter,https://github.com/hsivonen/utf16_iter,Apache-2.0 OR MIT,Henri Sivone utf8-width,https://github.com/magiclen/utf8-width,MIT,Magic Len utf8_iter,https://github.com/hsivonen/utf8_iter,Apache-2.0 OR MIT,Henri Sivonen uuid,https://github.com/uuid-rs/uuid,Apache-2.0 OR MIT,"Ashley Mannix, Dylan DPC, Hunar Roop Kahlon" +uuid-simd,https://github.com/Nugine/simd,MIT,The uuid-simd Authors valuable,https://github.com/tokio-rs/valuable,MIT,The valuable Authors void,https://github.com/reem/rust-void,MIT,Jonathan Reem vrl,https://github.com/vectordotdev/vrl,MPL-2.0,Vector Contributors vsimd,https://github.com/Nugine/simd,MIT,The vsimd Authors vte,https://github.com/alacritty/vte,Apache-2.0 OR MIT,"Joe Wilm , Christian Duerr " -vte_generate_state_changes,https://github.com/jwilm/vte,Apache-2.0 OR MIT,Christian Duerr wait-timeout,https://github.com/alexcrichton/wait-timeout,MIT OR Apache-2.0,Alex Crichton waker-fn,https://github.com/smol-rs/waker-fn,Apache-2.0 OR MIT,Stjepan Glavina walkdir,https://github.com/BurntSushi/walkdir,Unlicense OR MIT,Andrew Gallant want,https://github.com/seanmonstar/want,MIT,Sean McArthur warp,https://github.com/seanmonstar/warp,MIT,Sean McArthur wasi,https://github.com/bytecodealliance/wasi,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,The Cranelift Project Developers +wasi,https://github.com/bytecodealliance/wasi-rs,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,The Cranelift Project Developers wasite,https://github.com/ardaku/wasite,Apache-2.0 OR BSL-1.0 OR MIT,The wasite Authors wasm-bindgen,https://github.com/rustwasm/wasm-bindgen,MIT OR Apache-2.0,The wasm-bindgen Developers wasm-bindgen-backend,https://github.com/rustwasm/wasm-bindgen/tree/master/crates/backend,MIT OR Apache-2.0,The wasm-bindgen Developers @@ -703,7 +742,9 @@ wasm-bindgen-macro,https://github.com/rustwasm/wasm-bindgen/tree/master/crates/m wasm-bindgen-macro-support,https://github.com/rustwasm/wasm-bindgen/tree/master/crates/macro-support,MIT OR Apache-2.0,The wasm-bindgen Developers wasm-bindgen-shared,https://github.com/rustwasm/wasm-bindgen/tree/master/crates/shared,MIT OR Apache-2.0,The wasm-bindgen Developers wasm-streams,https://github.com/MattiasBuelens/wasm-streams,MIT OR Apache-2.0,Mattias Buelens +wasm-timer,https://github.com/tomaka/wasm-timer,MIT,Pierre Krieger web-sys,https://github.com/rustwasm/wasm-bindgen/tree/master/crates/web-sys,MIT OR Apache-2.0,The wasm-bindgen Developers +web-time,https://github.com/daxpedda/web-time,MIT OR Apache-2.0,The web-time Authors webbrowser,https://github.com/amodm/webbrowser-rs,MIT OR Apache-2.0,Amod Malviya @amodm webpki-roots,https://github.com/rustls/webpki-roots,MPL-2.0,The webpki-roots Authors whoami,https://github.com/ardaku/whoami,Apache-2.0 OR BSL-1.0 OR MIT,The whoami Authors @@ -712,15 +753,19 @@ widestring,https://github.com/starkat99/widestring-rs,MIT OR Apache-2.0,The wide winapi,https://github.com/retep998/winapi-rs,MIT OR Apache-2.0,Peter Atashian winapi-util,https://github.com/BurntSushi/winapi-util,Unlicense OR MIT,Andrew Gallant windows,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows-collections,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The windows-collections Authors +windows-future,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The windows-future Authors +windows-numerics,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The windows-numerics Authors windows-service,https://github.com/mullvad/windows-service-rs,MIT OR Apache-2.0,Mullvad VPN winnow,https://github.com/winnow-rs/winnow,MIT,The winnow Authors winreg,https://github.com/gentoo90/winreg-rs,MIT,Igor Shaula +wit-bindgen-rt,https://github.com/bytecodealliance/wasi-rs,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,The wit-bindgen-rt Authors woothee,https://github.com/woothee/woothee-rust,Apache-2.0,hhatto write16,https://github.com/hsivonen/write16,Apache-2.0 OR MIT,The write16 Authors writeable,https://github.com/unicode-org/icu4x,Unicode-3.0,The ICU4X Project Developers wyz,https://github.com/myrrlyn/wyz,MIT,myrrlyn xmlparser,https://github.com/RazrFalcon/xmlparser,MIT OR Apache-2.0,Yevhenii Reizner -yaml-rust,https://github.com/chyh1990/yaml-rust,MIT OR Apache-2.0,Yuheng Chen +xxhash-rust,https://github.com/DoumanAsh/xxhash-rust,BSL-1.0,Douman yoke,https://github.com/unicode-org/icu4x,Unicode-3.0,Manish Goregaokar yoke-derive,https://github.com/unicode-org/icu4x,Unicode-3.0,Manish Goregaokar zerocopy,https://github.com/google/zerocopy,BSD-2-Clause OR Apache-2.0 OR MIT,Joshua Liebow-Feeser @@ -729,6 +774,7 @@ zerofrom-derive,https://github.com/unicode-org/icu4x,Unicode-3.0,Manish Goregaok zeroize,https://github.com/RustCrypto/utils/tree/master/zeroize,Apache-2.0 OR MIT,The RustCrypto Project Developers zerovec,https://github.com/unicode-org/icu4x,Unicode-3.0,The ICU4X Project Developers zerovec-derive,https://github.com/unicode-org/icu4x,Unicode-3.0,Manish Goregaokar +zlib-rs,https://github.com/trifectatechfoundation/zlib-rs,Zlib,The zlib-rs Authors zstd,https://github.com/gyscos/zstd-rs,MIT,Alexandre Bury zstd-safe,https://github.com/gyscos/zstd-rs,MIT OR Apache-2.0,Alexandre Bury zstd-sys,https://github.com/gyscos/zstd-rs,MIT OR Apache-2.0,Alexandre Bury diff --git a/Makefile b/Makefile index afe737044b..8830387d5c 100644 --- a/Makefile +++ b/Makefile @@ -119,6 +119,9 @@ define MAYBE_ENVIRONMENT_COPY_ARTIFACTS endef endif +# docker container id file needs to live in the host machine and is later mounted into the container +CIDFILE := $(shell mktemp -u /tmp/vector-environment-docker-cid.XXXXXX) + # We use a volume here as non-Linux hosts are extremely slow to share disks, and Linux hosts tend to get permissions clobbered. define ENVIRONMENT_EXEC ${ENVIRONMENT_PREPARE} @@ -134,11 +137,13 @@ define ENVIRONMENT_EXEC $(if $(ENVIRONMENT_NETWORK),--network $(ENVIRONMENT_NETWORK),) \ --mount type=bind,source=${CURRENT_DIR},target=/git/vectordotdev/vector \ $(if $(findstring docker,$(CONTAINER_TOOL)),--mount type=bind$(COMMA)source=/var/run/docker.sock$(COMMA)target=/var/run/docker.sock,) \ + $(if $(findstring docker,$(CONTAINER_TOOL)),--cidfile $(CIDFILE),) \ + $(if $(findstring docker,$(CONTAINER_TOOL)),--mount type=bind$(COMMA)source=$(CIDFILE)$(COMMA)target=/.docker-container-id,) \ --mount type=volume,source=vector-target,target=/git/vectordotdev/vector/target \ --mount type=volume,source=vector-cargo-cache,target=/root/.cargo \ --mount type=volume,source=vector-rustup-cache,target=/root/.rustup \ $(foreach publish,$(ENVIRONMENT_PUBLISH),--publish $(publish)) \ - $(ENVIRONMENT_UPSTREAM) + $(ENVIRONMENT_UPSTREAM); rm -f $(CIDFILE) endef @@ -369,7 +374,7 @@ test-integration: test-integration-databend test-integration-docker-logs test-in test-integration: test-integration-eventstoredb test-integration-fluent test-integration-gcp test-integration-greptimedb test-integration-humio test-integration-http-client test-integration-influxdb test-integration: test-integration-kafka test-integration-logstash test-integration-loki test-integration-mongodb test-integration-nats test-integration: test-integration-nginx test-integration-opentelemetry test-integration-postgres test-integration-prometheus test-integration-pulsar -test-integration: test-integration-redis test-integration-splunk test-integration-dnstap test-integration-datadog-agent test-integration-datadog-logs test-integration-e2e-datadog-logs +test-integration: test-integration-redis test-integration-splunk test-integration-dnstap test-integration-datadog-agent test-integration-datadog-logs test-integration-e2e-datadog-logs test-integration-e2e-opentelemetry-logs test-integration: test-integration-datadog-traces test-integration-shutdown test-integration-%-cleanup: @@ -459,7 +464,7 @@ check-component-features: ## Check that all component features are setup properl .PHONY: check-clippy check-clippy: ## Check code with Clippy - ${MAYBE_ENVIRONMENT_EXEC} cargo vdev check rust --clippy + ${MAYBE_ENVIRONMENT_EXEC} cargo vdev check rust .PHONY: check-docs check-docs: ## Check that all /docs file are valid @@ -700,3 +705,7 @@ cargo-install-%: .PHONY: ci-generate-publish-metadata ci-generate-publish-metadata: ## Generates the necessary metadata required for building/publishing Vector. cargo vdev build publish-metadata + +.PHONY: clippy-fix +clippy-fix: + ${MAYBE_ENVIRONMENT_EXEC} cargo vdev check rust --fix diff --git a/PRIVACY.md b/PRIVACY.md index 5e3d210233..8c2c0c90e6 100644 --- a/PRIVACY.md +++ b/PRIVACY.md @@ -34,8 +34,7 @@ checks, capturing diagnostic information, and sharing crash reports. ## Vector Website & Docs The Vector website does collect various analytics. Aggregated analytics data is -derived from backend server logs which are anonymized. Vector uses -[Netlify analytics][netlify_analytics] for this. +derived from backend server logs which are anonymized. ## Vector Community @@ -54,5 +53,4 @@ privacy policy [here][discord_pp]. [github_pp]: https://help.github.com/en/github/site-policy/github-privacy-statement [discord_pp]: https://discord.com/privacy/ -[netlify_analytics]: https://www.netlify.com/products/analytics/ [vero_pp]: https://www.getvero.com/privacy/ diff --git a/README.md b/README.md index ae93fa4d6d..bf4db29273 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,11 @@ +[![Nightly](https://github.com/vectordotdev/vector/actions/workflows/nightly.yml/badge.svg)](https://github.com/vectordotdev/vector/actions/workflows/nightly.yml) +[![E2E Test Suite](https://github.com/vectordotdev/vector/actions/workflows/e2e.yml/badge.svg)](https://github.com/vectordotdev/vector/actions/workflows/e2e.yml) +[![Component Features](https://github.com/vectordotdev/vector/actions/workflows/component_features.yml/badge.svg)](https://github.com/vectordotdev/vector/actions/workflows/component_features.yml) + +

+ Vector +

+

Quickstart  •   @@ -9,9 +17,6 @@ Rust Crate Docs

-

- Vector -

## What is Vector? @@ -27,6 +32,8 @@ faster than every alternative in the space. To get started, follow our [**quickstart guide**][docs.quickstart] or [**install Vector**][docs.installation]. +Vector is maintained by Datadog's [Community Open Source Engineering team](https://opensource.datadoghq.com/about/#the-community-open-source-engineering-team). + ### Principles * **Reliable** - Built in [Rust][urls.rust], Vector's primary design goal is reliability. @@ -48,48 +55,23 @@ Vector**][docs.installation]. **Tuple**, **Douban**, **Visa**, **Mambu**, **Blockfi**, **Claranet**, **Instacart**, **Forcepoint**, and [many more][urls.production_users]. * Vector is **downloaded over 100,000 times per day**. -* Vector's largest user **processes over 30TB daily**. -* Vector has **over 100 contributors** and growing. +* Vector's largest user **processes over 500TB daily**. +* Vector has **over 500 contributors** and growing. -## [Documentation](https://vector.dev/docs/) +## Documentation -### About +All user documentation is available at **[vector.dev/docs](https://vector.dev/docs)**. -* [**Concepts**][docs.about.concepts] -* [**Under the hood**][docs.about.under-the-hood] - * [**Architecture**][docs.under-the-hood.architecture] - [data model][docs.architecture.data-model] ([log][docs.data-model.log], [metric][docs.data-model.metric]), [pipeline model][docs.architecture.pipeline-model], [concurrency model][docs.architecture.concurrency-model], [runtime model][docs.architecture.runtime-model] - * [**Networking**][docs.under-the-hood.networking] - [ARC][docs.networking.adaptive-request-concurrency] - * [**Guarantees**][docs.under-the-hood.guarantees] +Other Resources: -### Setup - -* [**Quickstart**][docs.setup.quickstart] -* [**Installation**][docs.setup.installation] - [operating systems][docs.installation.operating_systems], [package managers][docs.installation.package_managers], [platforms][docs.installation.platforms] ([Kubernetes][docs.platforms.kubernetes]), [manual][docs.installation.manual] -* [**Deployment**][docs.deployment] - [roles][docs.deployment.roles], [topologies][docs.deployment.topologies] - -### Reference - -* **Configuration** - * [**Sources**][docs.configuration.sources] - [docker_logs][docs.sources.docker_logs], [file][docs.sources.file], [http][docs.sources.http], [journald][docs.sources.journald], [kafka][docs.sources.kafka], [socket][docs.sources.socket], and [many more...][docs.sources] - * [**Transforms**][docs.configuration.transforms] - [dedupe][docs.transforms.dedupe], [filter][docs.transforms.filter], [geoip][docs.transforms.geoip], [log_to_metric][docs.transforms.log_to_metric], [lua][docs.transforms.lua], [remap][docs.transforms.remap], and [many more...][docs.transforms] - * [**Sinks**][docs.configuration.sinks] - [aws_cloudwatch_logs][docs.sinks.aws_cloudwatch_logs], [aws_s3][docs.sinks.aws_s3], [clickhouse][docs.sinks.clickhouse], [elasticsearch][docs.sinks.elasticsearch], [gcp_cloud_storage][docs.sinks.gcp_cloud_storage], and [many more...][docs.sinks] - * [**Unit tests**][docs.configuration.tests] -* [**Remap Language**][docs.reference.vrl] -* [**API**][docs.reference.api] -* [**CLI**][docs.reference.cli] - -### Administration - -* [**Management**][docs.administration.management] -* [**Monitoring & observing**][docs.administration.monitoring] -* [**Upgrading**][docs.administration.upgrading] -* [**Validating**][docs.administration.validating] - -### Resources - -* [**Community**][urls.vector_community] - [chat][urls.vector_chat], [calendar][urls.vector_calendar], [@vectordotdev][urls.vector_twitter] -* [**Releases**][urls.vector_releases] -* **Policies** - [Code of Conduct][urls.vector_code_of_conduct], [Privacy][urls.vector_privacy_policy], [Releases][urls.vector_releases_policy], [Security][urls.vector_security_policy], [Versioning][urls.vector_versioning_policy] +* [**Vector Calendar**][urls.vector_calendar] +* **Policies**: + * [**Code of Conduct**][urls.vector_code_of_conduct] + * [**Contributing**][urls.vector_contributing_policy] + * [**Privacy**][urls.vector_privacy_policy] + * [**Releases**][urls.vector_releases_policy] + * [**Versioning**][urls.vector_versioning_policy] + * [**Security**][urls.vector_security_policy] ## Comparisons @@ -154,22 +136,23 @@ Vector is an end-to-end, unified, open data platform. Developed with ❤️ by Datadog - Security Policy - Privacy Policy

-[docs.about.concepts]: https://vector.dev/docs/about/concepts/ -[docs.about.under-the-hood]: https://vector.dev/docs/about/under-the-hood/ +[docs.about.concepts]: https://vector.dev/docs/introduction/concepts/ +[docs.about.introduction]: https://vector.dev/docs/introduction/ [docs.administration.monitoring]: https://vector.dev/docs/administration/monitoring/ [docs.administration.management]: https://vector.dev/docs/administration/management/ [docs.administration.upgrading]: https://vector.dev/docs/administration/upgrading/ [docs.administration.validating]: https://vector.dev/docs/administration/validating/ -[docs.architecture.concurrency-model]: https://vector.dev/docs/about/under-the-hood/architecture/concurrency-model/ -[docs.architecture.data-model]: https://vector.dev/docs/about/under-the-hood/architecture/data-model/ -[docs.architecture.pipeline-model]: https://vector.dev/docs/about/under-the-hood/architecture/pipeline-model/ -[docs.architecture.runtime-model]: https://vector.dev/docs/about/under-the-hood/architecture/runtime-model/ +[docs.architecture.concurrency-model]: https://vector.dev/docs/architecture/concurrency-model/ +[docs.architecture.data-model]: https://vector.dev/docs/architecture/data-model/ +[docs.architecture.pipeline-model]: https://vector.dev/docs/architecture/pipeline-model/ +[docs.architecture.runtime-model]: https://vector.dev/docs/architecture/runtime-model/ [docs.configuration.sinks]: https://vector.dev/docs/reference/configuration/sinks/ [docs.configuration.sources]: https://vector.dev/docs/reference/configuration/sources/ [docs.configuration.tests]: https://vector.dev/docs/reference/configuration/tests/ [docs.configuration.transforms]: https://vector.dev/docs/reference/configuration/transforms/ -[docs.data-model.log]: https://vector.dev/docs/about/under-the-hood/architecture/data-model/log/ -[docs.data-model.metric]: https://vector.dev/docs/about/under-the-hood/architecture/data-model/metric/ +[docs.configuration.enrichment_tables]: https://vector.dev/docs/reference/configuration/global-options/#enrichment_tables +[docs.data-model.log]: https://vector.dev/docs/architecture/data-model/log/ +[docs.data-model.metric]: https://vector.dev/docs/architecture/data-model/metric/ [docs.deployment.roles]: https://vector.dev/docs/setup/deployment/roles/ [docs.deployment.topologies]: https://vector.dev/docs/setup/deployment/topologies/ [docs.deployment]: https://vector.dev/docs/setup/deployment/ @@ -178,7 +161,7 @@ Vector is an end-to-end, unified, open data platform. [docs.installation.package_managers]: https://vector.dev/docs/setup/installation/package-managers/ [docs.installation.platforms]: https://vector.dev/docs/setup/installation/platforms/ [docs.installation]: https://vector.dev/docs/setup/installation/ -[docs.networking.adaptive-request-concurrency]: https://vector.dev/docs/about/under-the-hood/networking/arc/ +[docs.architecture.adaptive-request-concurrency]: https://vector.dev/docs/architecture/arc/ [docs.platforms.kubernetes]: https://vector.dev/docs/setup/installation/platforms/kubernetes/ [docs.quickstart]: https://vector.dev/docs/setup/quickstart/ [docs.reference.api]: https://vector.dev/docs/reference/api/ @@ -207,14 +190,15 @@ Vector is an end-to-end, unified, open data platform. [docs.transforms.lua]: https://vector.dev/docs/reference/configuration/transforms/lua/ [docs.transforms.remap]: https://vector.dev/docs/reference/configuration/transforms/remap/ [docs.transforms]: https://vector.dev/docs/reference/configuration/transforms/ -[docs.under-the-hood.architecture]: https://vector.dev/docs/about/under-the-hood/architecture/ -[docs.under-the-hood.guarantees]: https://vector.dev/docs/about/under-the-hood/guarantees/ -[docs.under-the-hood.networking]: https://vector.dev/docs/about/under-the-hood/networking/ +[docs.introduction.architecture]: https://vector.dev/docs/architecture/ +[docs.introduction.guarantees]: https://vector.dev/docs/introduction/guarantees/ +[docs.introduction.architecture]: https://vector.dev/docs/architecture/ [urls.production_users]: https://github.com/vectordotdev/vector/issues/790 [urls.rust]: https://www.rust-lang.org/ [urls.vector_calendar]: https://calendar.vector.dev [urls.vector_chat]: https://chat.vector.dev [urls.vector_code_of_conduct]: https://github.com/vectordotdev/vector/blob/master/CODE_OF_CONDUCT.md +[urls.vector_contributing_policy]: https://github.com/vectordotdev/vector/blob/master/CONTRIBUTING.md [urls.vector_community]: https://vector.dev/community/ [urls.vector_privacy_policy]: https://github.com/vectordotdev/vector/blob/master/PRIVACY.md [urls.vector_release_policy]: https://github.com/vectordotdev/vector/blob/master/RELEASING.md @@ -225,4 +209,3 @@ Vector is an end-to-end, unified, open data platform. [urls.vector_twitter]: https://twitter.com/vectordotdev [urls.vector_versioning_policy]: https://github.com/vectordotdev/vector/blob/master/VERSIONING.md [urls.vote_feature]: https://github.com/vectordotdev/vector/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc+label%3A%22type%3A+feature%22 - diff --git a/VERSIONING.md b/VERSIONING.md index 33d37b8aa7..a84b608f6a 100644 --- a/VERSIONING.md +++ b/VERSIONING.md @@ -122,7 +122,7 @@ here. Each minor release bump will include an upgrade guide in the [CLI]: https://vector.dev/docs/reference/cli/ [configuration schema]: https://vector.dev/docs/reference/configuration/ [data directory]: https://vector.dev/docs/reference/configuration/global-options/#data_dir -[data model]: https://vector.dev/docs/about/under-the-hood/architecture/data-model/ +[data model]: https://vector.dev/docs/architecture/data-model/ [GitHub repository]: https://github.com/vectordotdev/vector [GraphQL API]: https://vector.dev/docs/reference/api/ [Installation workflows]: https://vector.dev/docs/setup/installation/ diff --git a/aqua/aqua.yaml b/aqua/aqua.yaml index e72564f079..37c8ae623b 100644 --- a/aqua/aqua.yaml +++ b/aqua/aqua.yaml @@ -6,9 +6,9 @@ registries: ref: v4.268.0 # renovate: depName=aquaproj/aqua-registry packages: - name: rustwasm/wasm-pack@v0.13.1 - - name: crates.io/cargo-deb@2.9.1 + - name: crates.io/cargo-deb@2.9.3 - name: cross-rs/cross@v0.2.5 - name: nextest-rs/nextest/cargo-nextest@cargo-nextest-0.9.47 - - name: EmbarkStudios/cargo-deny@0.16.1 + - name: EmbarkStudios/cargo-deny@0.16.2 - name: foresterre/cargo-msrv@v0.15.1 - name: crates.io/dd-rust-license-tool@1.0.1 diff --git a/benches/batch.rs b/benches/batch.rs index 02e9fc071a..cb1aaa53d6 100644 --- a/benches/batch.rs +++ b/benches/batch.rs @@ -1,13 +1,13 @@ use std::{convert::Infallible, time::Duration}; use bytes::{BufMut, Bytes, BytesMut}; -use criterion::{criterion_group, Criterion, SamplingMode, Throughput}; -use futures::{future, stream, SinkExt, StreamExt}; +use criterion::{Criterion, SamplingMode, Throughput, criterion_group}; +use futures::{SinkExt, StreamExt, future, stream}; use vector::{ sinks::util::{ - batch::{Batch, BatchConfig, BatchError, BatchSettings, BatchSize, PushResult}, BatchSink, Buffer, Compression, EncodedEvent, Merged, Partition, PartitionBatchSink, SinkBatchSettings, + batch::{Batch, BatchConfig, BatchError, BatchSettings, BatchSize, PushResult}, }, test_util::{random_lines, runtime}, }; @@ -38,7 +38,7 @@ fn benchmark_batch(c: &mut Criterion) { .collect(); for (compression, batch_size) in cases.iter() { - group.bench_function(format!("partitioned/{}_{}", compression, batch_size), |b| { + group.bench_function(format!("partitioned/{compression}_{batch_size}"), |b| { b.iter_batched( || { let rt = runtime(); @@ -68,35 +68,32 @@ fn benchmark_batch(c: &mut Criterion) { ) }); - group.bench_function( - format!("unpartitioned/{}_{}", compression, batch_size), - |b| { - b.iter_batched( - || { - let rt = runtime(); - let mut batch = BatchSettings::default(); - batch.size.bytes = *batch_size; - batch.size.events = num_events; - - let batch_sink = BatchSink::new( - tower::service_fn(|_| future::ok::<_, Infallible>(())), - Buffer::new(batch.size, *compression), - Duration::from_secs(1), - ) - .sink_map_err(|error| panic!("{}", error)); - - ( - rt, - stream::iter(input.clone()) - .map(|item| Ok(EncodedEvent::new(item, 0, JsonSize::zero()))), - batch_sink, - ) - }, - |(rt, input, batch_sink)| rt.block_on(input.forward(batch_sink)).unwrap(), - criterion::BatchSize::LargeInput, - ) - }, - ); + group.bench_function(format!("unpartitioned/{compression}_{batch_size}"), |b| { + b.iter_batched( + || { + let rt = runtime(); + let mut batch = BatchSettings::default(); + batch.size.bytes = *batch_size; + batch.size.events = num_events; + + let batch_sink = BatchSink::new( + tower::service_fn(|_| future::ok::<_, Infallible>(())), + Buffer::new(batch.size, *compression), + Duration::from_secs(1), + ) + .sink_map_err(|error| panic!("{}", error)); + + ( + rt, + stream::iter(input.clone()) + .map(|item| Ok(EncodedEvent::new(item, 0, JsonSize::zero()))), + batch_sink, + ) + }, + |(rt, input, batch_sink)| rt.block_on(input.forward(batch_sink)).unwrap(), + criterion::BatchSize::LargeInput, + ) + }); } } diff --git a/benches/codecs/character_delimited_bytes.rs b/benches/codecs/character_delimited_bytes.rs index 9714937960..38cf2d7ae7 100644 --- a/benches/codecs/character_delimited_bytes.rs +++ b/benches/codecs/character_delimited_bytes.rs @@ -2,13 +2,13 @@ use std::{fmt, time::Duration}; use bytes::BytesMut; use criterion::{ - criterion_group, measurement::WallTime, BatchSize, BenchmarkGroup, BenchmarkId, Criterion, - SamplingMode, Throughput, + BatchSize, BenchmarkGroup, BenchmarkId, Criterion, SamplingMode, Throughput, criterion_group, + measurement::WallTime, }; use tokio_util::codec::Decoder; use vector_lib::codecs::{ - decoding::{Deserializer, Framer}, BytesDeserializer, CharacterDelimitedDecoder, + decoding::{Deserializer, Framer}, }; #[derive(Debug)] diff --git a/benches/codecs/encoder.rs b/benches/codecs/encoder.rs index 47f9d9ae25..ccbf4c2b09 100644 --- a/benches/codecs/encoder.rs +++ b/benches/codecs/encoder.rs @@ -2,12 +2,12 @@ use std::time::Duration; use bytes::{BufMut, BytesMut}; use criterion::{ - criterion_group, measurement::WallTime, BatchSize, BenchmarkGroup, Criterion, SamplingMode, - Throughput, + BatchSize, BenchmarkGroup, Criterion, SamplingMode, Throughput, criterion_group, + measurement::WallTime, }; use tokio_util::codec::Encoder; use vector::event::{Event, LogEvent}; -use vector_lib::codecs::{encoding::Framer, JsonSerializerConfig, NewlineDelimitedEncoder}; +use vector_lib::codecs::{JsonSerializerConfig, NewlineDelimitedEncoder, encoding::Framer}; use vector_lib::{btreemap, byte_size_of::ByteSizeOf}; #[derive(Debug, Clone)] @@ -92,7 +92,7 @@ fn encoder(c: &mut Criterion) { b.iter_batched( || { vector::codecs::Encoder::::new( - NewlineDelimitedEncoder::new().into(), + NewlineDelimitedEncoder::default().into(), JsonSerializerConfig::default().build().into(), ) }, diff --git a/benches/codecs/newline_bytes.rs b/benches/codecs/newline_bytes.rs index 95d8962036..2c777b62e6 100644 --- a/benches/codecs/newline_bytes.rs +++ b/benches/codecs/newline_bytes.rs @@ -2,12 +2,12 @@ use std::{fmt, time::Duration}; use bytes::BytesMut; use criterion::{ - criterion_group, measurement::WallTime, BatchSize, BenchmarkGroup, BenchmarkId, Criterion, - SamplingMode, Throughput, + BatchSize, BenchmarkGroup, BenchmarkId, Criterion, SamplingMode, Throughput, criterion_group, + measurement::WallTime, }; use tokio_util::codec::Decoder; use vector_lib::codecs::{ - decoding::Deserializer, decoding::Framer, BytesDeserializer, NewlineDelimitedDecoder, + BytesDeserializer, NewlineDelimitedDecoder, decoding::Deserializer, decoding::Framer, }; #[derive(Debug)] @@ -50,8 +50,8 @@ fn decoding(c: &mut Criterion) { let framer = Framer::NewlineDelimited( param .max_length - .map(|ml| NewlineDelimitedDecoder::new_with_max_length(ml)) - .unwrap_or(NewlineDelimitedDecoder::new()), + .map(NewlineDelimitedDecoder::new_with_max_length) + .unwrap_or_default(), ); let deserializer = Deserializer::Bytes(BytesDeserializer); let decoder = vector::codecs::Decoder::new(framer, deserializer); diff --git a/benches/distribution_statistic.rs b/benches/distribution_statistic.rs index 31029f0e6e..af62b1b39d 100644 --- a/benches/distribution_statistic.rs +++ b/benches/distribution_statistic.rs @@ -1,13 +1,12 @@ -use criterion::{criterion_group, criterion_main, BatchSize, Criterion}; -use rand::{ - distributions::{Distribution, Uniform}, - seq::SliceRandom, -}; +use criterion::{BatchSize, Criterion, criterion_group, criterion_main}; +use rand::distr::Distribution; +use rand::{distr::Uniform, seq::SliceRandom}; use vector::{event::metric::Sample, sinks::util::statistic::DistributionStatistic}; fn generate_samples(mut size: u32, max_bin_count: u32) -> Vec { - let mut rng = rand::thread_rng(); - let range = Uniform::from(1..=max_bin_count); + let mut rng = rand::rng(); + // Fix Uniform usage for inclusive range + let range = Uniform::new_inclusive(1, max_bin_count).unwrap(); let mut value = 1.0; let mut samples = Vec::new(); while size > 0 { @@ -28,7 +27,7 @@ fn bench_statistic(c: &mut Criterion) { let sizes = [5, 10, 50, 100, 200, 500, 1000]; for &size in &sizes { - group.bench_function(format!("small-bin-{}", size), |b| { + group.bench_function(format!("small-bin-{size}"), |b| { b.iter_batched( move || generate_samples(size, 3), |samples| { @@ -41,7 +40,7 @@ fn bench_statistic(c: &mut Criterion) { let sizes = [50, 100, 200, 500, 1000]; for &size in &sizes { - group.bench_function(format!("large-bin-{}", size), |b| { + group.bench_function(format!("large-bin-{size}"), |b| { b.iter_batched( move || generate_samples(size, 20), |samples| { diff --git a/benches/dnstap/mod.rs b/benches/dnstap/mod.rs index 37839b41ac..2c0673590b 100644 --- a/benches/dnstap/mod.rs +++ b/benches/dnstap/mod.rs @@ -1,21 +1,31 @@ +use base64::Engine; +use base64::engine::general_purpose::STANDARD; use bytes::Bytes; -use criterion::{criterion_group, criterion_main, BatchSize, Criterion, Throughput}; +use criterion::{BatchSize, Criterion, Throughput, criterion_group, criterion_main}; +use dnsmsg_parser::dns_message_parser::DnsParserOptions; +use dnstap_parser::parser::DnstapParser; use vector::event::LogEvent; -use vector::sources::dnstap::parser::DnstapParser; fn benchmark_query_parsing(c: &mut Criterion) { let mut event = LogEvent::default(); let raw_dnstap_data = "ChVqYW1lcy1WaXJ0dWFsLU1hY2hpbmUSC0JJTkQgOS4xNi4zcnoIAxACGAEiEAAAAAAAAA\ AAAAAAAAAAAAAqECABBQJwlAAAAAAAAAAAADAw8+0CODVA7+zq9wVNMU3WNlI2kwIAAAABAAAAAAABCWZhY2Vib29rMQNjb\ 20AAAEAAQAAKQIAAACAAAAMAAoACOxjCAG9zVgzWgUDY29tAHgB"; - let dnstap_data = base64::decode(raw_dnstap_data).unwrap(); + let dnstap_data = STANDARD.decode(raw_dnstap_data).unwrap(); let mut group = c.benchmark_group("dnstap"); group.throughput(Throughput::Bytes(dnstap_data.len() as u64)); group.bench_function("dns_query_parsing", |b| { b.iter_batched( || dnstap_data.clone(), - |dnstap_data| DnstapParser::parse(&mut event, Bytes::from(dnstap_data)).unwrap(), + |dnstap_data| { + DnstapParser::parse( + &mut event, + Bytes::from(dnstap_data), + DnsParserOptions::default(), + ) + .unwrap() + }, BatchSize::SmallInput, ) }); @@ -28,14 +38,21 @@ fn benchmark_update_parsing(c: &mut Criterion) { let raw_dnstap_data = "ChVqYW1lcy1WaXJ0dWFsLU1hY2hpbmUSC0JJTkQgOS4xNi4zcmsIDhABGAEiBH8AAA\ EqBH8AAAEwrG44AEC+iu73BU14gfofUh1wi6gAAAEAAAAAAAAHZXhhbXBsZQNjb20AAAYAAWC+iu73BW0agDwvch1wi6gAA\ AEAAAAAAAAHZXhhbXBsZQNjb20AAAYAAXgB"; - let dnstap_data = base64::decode(raw_dnstap_data).unwrap(); + let dnstap_data = STANDARD.decode(raw_dnstap_data).unwrap(); let mut group = c.benchmark_group("dnstap"); group.throughput(Throughput::Bytes(dnstap_data.len() as u64)); group.bench_function("dns_update_parsing", |b| { b.iter_batched( || dnstap_data.clone(), - |dnstap_data| DnstapParser::parse(&mut event, Bytes::from(dnstap_data)).unwrap(), + |dnstap_data| { + DnstapParser::parse( + &mut event, + Bytes::from(dnstap_data), + DnsParserOptions::default(), + ) + .unwrap() + }, BatchSize::SmallInput, ) }); diff --git a/benches/enrichment_tables.rs b/benches/enrichment_tables.rs index 5c9a11f157..73cfb9e938 100644 --- a/benches/enrichment_tables.rs +++ b/benches/enrichment_tables.rs @@ -1,12 +1,13 @@ use std::time::SystemTime; use chrono::prelude::*; -use criterion::{criterion_group, criterion_main, BatchSize, Criterion}; +use criterion::{BatchSize, Criterion, criterion_group, criterion_main}; +use vector::enrichment_tables::file::FileData; use vector::enrichment_tables::{ + Condition, Table, file::File, geoip::{Geoip, GeoipConfig}, mmdb::{Mmdb, MmdbConfig}, - Condition, Table, }; use vector_lib::enrichment::Case; use vrl::value::{ObjectMap, Value}; @@ -27,12 +28,11 @@ fn column(col: usize, row: usize) -> Value { // And a final column with a date, each of the above duplicated row should have // a unique date. Value::Timestamp( - Utc.ymd(2013, row as u32 % 10 + 1, 15) - .and_hms_opt(0, 0, 0) - .expect("invalid timestamp"), + Utc.with_ymd_and_hms(2013, row as u32 % 10 + 1, 15, 0, 0, 0) + .unwrap(), ) } else { - Value::from(format!("data-{}-{}", col, row)) + Value::from(format!("data-{col}-{row}")) } } @@ -49,12 +49,13 @@ fn benchmark_enrichment_tables_file(c: &mut Criterion) { let mut file = File::new( Default::default(), - SystemTime::now(), - data, - // Headers. - (0..10) - .map(|header| format!("field-{}", header)) - .collect::>(), + FileData { + data, + headers: (0..10) + .map(|header| format!("field-{header}")) + .collect::>(), + modified: SystemTime::now(), + }, ); let (condition, index, result_offset) = if date_range { @@ -67,14 +68,8 @@ fn benchmark_enrichment_tables_file(c: &mut Criterion) { }, Condition::BetweenDates { field: "field-1", - from: Utc - .ymd(2013, 6, 1) - .and_hms_opt(0, 0, 0) - .expect("invalid timestamp"), - to: Utc - .ymd(2013, 7, 1) - .and_hms_opt(0, 0, 0) - .expect("invalid timestamp"), + from: Utc.with_ymd_and_hms(2013, 6, 1, 0, 0, 0).unwrap(), + to: Utc.with_ymd_and_hms(2013, 7, 1, 0, 0, 0).unwrap(), }, ], file.add_index(case, &["field-0"]).unwrap(), @@ -100,7 +95,7 @@ fn benchmark_enrichment_tables_file(c: &mut Criterion) { let result = (0..10) .map(|idx| { ( - format!("field-{}", idx).into(), + format!("field-{idx}").into(), column(idx, size - result_offset), ) }) @@ -112,11 +107,11 @@ fn benchmark_enrichment_tables_file(c: &mut Criterion) { group.bench_function("enrichment_tables/file_date_10", |b| { let (file, index, condition, expected) = setup(10, true, Case::Sensitive); b.iter_batched( - || (&file, &condition, expected.clone()), - |(file, condition, expected)| { + || (&file, &condition, expected.clone(), Some(index)), + |(file, condition, expected, index)| { assert_eq!( Ok(expected), - file.find_table_row(Case::Sensitive, condition, None, Some(index)) + file.find_table_row(Case::Sensitive, condition, None, None, index) ) }, BatchSize::SmallInput, @@ -126,11 +121,11 @@ fn benchmark_enrichment_tables_file(c: &mut Criterion) { group.bench_function("enrichment_tables/file_hashindex_sensitive_10", |b| { let (file, index, condition, expected) = setup(10, false, Case::Sensitive); b.iter_batched( - || (&file, index, &condition, expected.clone()), - |(file, index, condition, expected)| { + || (&file, &condition, expected.clone(), Some(index)), + |(file, condition, expected, index)| { assert_eq!( Ok(expected), - file.find_table_row(Case::Sensitive, condition, None, Some(index)) + file.find_table_row(Case::Sensitive, condition, None, None, index) ) }, BatchSize::SmallInput, @@ -140,11 +135,11 @@ fn benchmark_enrichment_tables_file(c: &mut Criterion) { group.bench_function("enrichment_tables/file_hashindex_insensitive_10", |b| { let (file, index, condition, expected) = setup(10, false, Case::Insensitive); b.iter_batched( - || (&file, index, &condition, expected.clone()), - |(file, index, condition, expected)| { + || (&file, &condition, expected.clone(), Some(index)), + |(file, condition, expected, index)| { assert_eq!( Ok(expected), - file.find_table_row(Case::Insensitive, condition, None, Some(index)) + file.find_table_row(Case::Insensitive, condition, None, None, index) ) }, BatchSize::SmallInput, @@ -154,11 +149,11 @@ fn benchmark_enrichment_tables_file(c: &mut Criterion) { group.bench_function("enrichment_tables/file_date_1_000", |b| { let (file, index, condition, expected) = setup(1_000, true, Case::Sensitive); b.iter_batched( - || (&file, &condition, expected.clone()), - |(file, condition, expected)| { + || (&file, &condition, expected.clone(), Some(index)), + |(file, condition, expected, index)| { assert_eq!( Ok(expected), - file.find_table_row(Case::Sensitive, condition, None, Some(index)) + file.find_table_row(Case::Sensitive, condition, None, None, index) ) }, BatchSize::SmallInput, @@ -168,11 +163,11 @@ fn benchmark_enrichment_tables_file(c: &mut Criterion) { group.bench_function("enrichment_tables/file_hashindex_sensitive_1_000", |b| { let (file, index, condition, expected) = setup(1_000, false, Case::Sensitive); b.iter_batched( - || (&file, index, &condition, expected.clone()), - |(file, index, condition, expected)| { + || (&file, &condition, expected.clone(), Some(index)), + |(file, condition, expected, index)| { assert_eq!( Ok(expected), - file.find_table_row(Case::Sensitive, condition, None, Some(index)) + file.find_table_row(Case::Sensitive, condition, None, None, index) ) }, BatchSize::SmallInput, @@ -182,11 +177,11 @@ fn benchmark_enrichment_tables_file(c: &mut Criterion) { group.bench_function("enrichment_tables/file_hashindex_insensitive_1_000", |b| { let (file, index, condition, expected) = setup(1_000, false, Case::Insensitive); b.iter_batched( - || (&file, index, &condition, expected.clone()), - |(file, index, condition, expected)| { + || (&file, &condition, expected.clone(), Some(index)), + |(file, condition, expected, index)| { assert_eq!( Ok(expected), - file.find_table_row(Case::Insensitive, condition, None, Some(index)) + file.find_table_row(Case::Insensitive, condition, None, None, index) ) }, BatchSize::SmallInput, @@ -196,11 +191,11 @@ fn benchmark_enrichment_tables_file(c: &mut Criterion) { group.bench_function("enrichment_tables/file_date_1_000_000", |b| { let (file, index, condition, expected) = setup(1_000_000, true, Case::Sensitive); b.iter_batched( - || (&file, &condition, expected.clone()), - |(file, condition, expected)| { + || (&file, &condition, expected.clone(), Some(index)), + |(file, condition, expected, index)| { assert_eq!( Ok(expected), - file.find_table_row(Case::Sensitive, condition, None, Some(index)) + file.find_table_row(Case::Sensitive, condition, None, None, index) ) }, BatchSize::SmallInput, @@ -212,11 +207,11 @@ fn benchmark_enrichment_tables_file(c: &mut Criterion) { |b| { let (file, index, condition, expected) = setup(1_000_000, false, Case::Sensitive); b.iter_batched( - || (&file, index, &condition, expected.clone()), - |(file, index, condition, expected)| { + || (&file, &condition, expected.clone(), Some(index)), + |(file, condition, expected, index)| { assert_eq!( Ok(expected), - file.find_table_row(Case::Sensitive, condition, None, Some(index)) + file.find_table_row(Case::Sensitive, condition, None, None, index) ) }, BatchSize::SmallInput, @@ -229,11 +224,11 @@ fn benchmark_enrichment_tables_file(c: &mut Criterion) { |b| { let (file, index, condition, expected) = setup(1_000_000, false, Case::Insensitive); b.iter_batched( - || (&file, index, &condition, expected.clone()), - |(file, index, condition, expected)| { + || (&file, &condition, expected.clone(), Some(index)), + |(file, condition, expected, index)| { assert_eq!( Ok(expected), - file.find_table_row(Case::Insensitive, condition, None, Some(index)) + file.find_table_row(Case::Insensitive, condition, None, None, index) ) }, BatchSize::SmallInput, @@ -246,7 +241,7 @@ fn benchmark_enrichment_tables_geoip(c: &mut Criterion) { let mut group = c.benchmark_group("enrichment_tables_geoip"); let build = |path: &str| { Geoip::new(GeoipConfig { - path: path.to_string(), + path: path.into(), locale: "en".to_string(), }) .unwrap() @@ -268,18 +263,17 @@ fn benchmark_enrichment_tables_geoip(c: &mut Criterion) { || (&table, ip, &expected), |(table, ip, expected)| { assert_eq!( - Ok(expected), - table - .find_table_row( - Case::Insensitive, - &[Condition::Equals { - field: "ip", - value: ip.into(), - }], - None, - None, - ) - .as_ref() + Ok(expected.clone()), + table.find_table_row( + Case::Insensitive, + &[Condition::Equals { + field: "ip", + value: ip.into(), + }], + None, + None, + None, + ) ) }, BatchSize::SmallInput, @@ -306,18 +300,17 @@ fn benchmark_enrichment_tables_geoip(c: &mut Criterion) { || (&table, ip, &expected), |(table, ip, expected)| { assert_eq!( - Ok(expected), - table - .find_table_row( - Case::Insensitive, - &[Condition::Equals { - field: "ip", - value: ip.into(), - }], - None, - None, - ) - .as_ref() + Ok(expected.clone()), + table.find_table_row( + Case::Insensitive, + &[Condition::Equals { + field: "ip", + value: ip.into(), + }], + None, + None, + None, + ) ) }, BatchSize::SmallInput, @@ -327,12 +320,7 @@ fn benchmark_enrichment_tables_geoip(c: &mut Criterion) { fn benchmark_enrichment_tables_mmdb(c: &mut Criterion) { let mut group = c.benchmark_group("enrichment_tables_mmdb"); - let build = |path: &str| { - Mmdb::new(MmdbConfig { - path: path.to_string(), - }) - .unwrap() - }; + let build = |path: &str| Mmdb::new(MmdbConfig { path: path.into() }).unwrap(); group.bench_function("enrichment_tables/mmdb_isp", |b| { let table = build("tests/data/GeoIP2-ISP-Test.mmdb"); @@ -350,18 +338,17 @@ fn benchmark_enrichment_tables_mmdb(c: &mut Criterion) { || (&table, ip, &expected), |(table, ip, expected)| { assert_eq!( - Ok(expected), - table - .find_table_row( - Case::Insensitive, - &[Condition::Equals { - field: "ip", - value: ip.into(), - }], - None, - None, - ) - .as_ref() + Ok(expected.clone()), + table.find_table_row( + Case::Insensitive, + &[Condition::Equals { + field: "ip", + value: ip.into(), + }], + None, + None, + None, + ) ) }, BatchSize::SmallInput, @@ -385,21 +372,20 @@ fn benchmark_enrichment_tables_mmdb(c: &mut Criterion) { || (&table, ip, &expected), |(table, ip, expected)| { assert_eq!( - Ok(expected), - table - .find_table_row( - Case::Insensitive, - &[Condition::Equals { - field: "ip", - value: ip.into(), - }], - Some(&[ - "location.latitude".to_string(), - "location.longitude".to_string(), - ]), - None, - ) - .as_ref() + Ok(expected.clone()), + table.find_table_row( + Case::Insensitive, + &[Condition::Equals { + field: "ip", + value: ip.into(), + }], + Some(&[ + "location.latitude".to_string(), + "location.longitude".to_string(), + ]), + None, + None, + ) ) }, BatchSize::SmallInput, diff --git a/benches/event.rs b/benches/event.rs index cb48f6101a..21dbf7b69d 100644 --- a/benches/event.rs +++ b/benches/event.rs @@ -1,5 +1,5 @@ use bytes::Bytes; -use criterion::{criterion_group, BatchSize, Criterion}; +use criterion::{BatchSize, Criterion, criterion_group}; use vector::event::LogEvent; use vrl::event_path; diff --git a/benches/files.rs b/benches/files.rs index 49153a378d..5754001d15 100644 --- a/benches/files.rs +++ b/benches/files.rs @@ -1,8 +1,8 @@ use std::{convert::TryInto, path::PathBuf, time::Duration}; use bytes::Bytes; -use criterion::{criterion_group, BatchSize, Criterion, SamplingMode, Throughput}; -use futures::{stream, SinkExt, StreamExt}; +use criterion::{BatchSize, Criterion, SamplingMode, Throughput, criterion_group}; +use futures::{SinkExt, StreamExt, stream}; use tempfile::tempdir; use tokio::fs::OpenOptions; use tokio_util::codec::{BytesCodec, FramedWrite}; @@ -10,7 +10,7 @@ use vector::{ config, sinks, sources, test_util::{random_lines, runtime, start_topology}, }; -use vector_lib::codecs::{encoding::FramingConfig, TextSerializerConfig}; +use vector_lib::codecs::{TextSerializerConfig, encoding::FramingConfig}; fn benchmark_files_no_partitions(c: &mut Criterion) { let num_lines: usize = 10_000; diff --git a/benches/http.rs b/benches/http.rs index b60b25a495..e8ed333af5 100644 --- a/benches/http.rs +++ b/benches/http.rs @@ -1,20 +1,23 @@ use std::net::SocketAddr; -use criterion::{criterion_group, BatchSize, BenchmarkId, Criterion, SamplingMode, Throughput}; +use criterion::{BatchSize, BenchmarkId, Criterion, SamplingMode, Throughput, criterion_group}; use futures::TryFutureExt; use hyper::{ - service::{make_service_fn, service_fn}, Body, Response, Server, + service::{make_service_fn, service_fn}, }; use tokio::runtime::Runtime; use vector::{ - config, sinks, - sinks::util::{BatchConfig, Compression}, + Error, config, + sinks::{ + self, + util::{BatchConfig, Compression}, + }, sources, + template::Template, test_util::{next_addr, random_lines, runtime, send_lines, start_topology, wait_for_tcp}, - Error, }; -use vector_lib::codecs::{encoding::FramingConfig, TextSerializerConfig}; +use vector_lib::codecs::{TextSerializerConfig, encoding::FramingConfig}; fn benchmark_http(c: &mut Criterion) { let num_lines: usize = 1_000; @@ -48,7 +51,7 @@ fn benchmark_http(c: &mut Criterion) { "out", &["in"], sinks::http::config::HttpSinkConfig { - uri: out_addr.to_string().parse::().unwrap().into(), + uri: Template::try_from(out_addr.to_string()).unwrap(), compression: *compression, method: Default::default(), auth: Default::default(), diff --git a/benches/languages.rs b/benches/languages.rs index 2c386f786f..ae99eb81cd 100644 --- a/benches/languages.rs +++ b/benches/languages.rs @@ -1,8 +1,8 @@ -use criterion::{criterion_group, criterion_main, BatchSize, Criterion, SamplingMode, Throughput}; +use criterion::{BatchSize, Criterion, SamplingMode, Throughput, criterion_group, criterion_main}; use indoc::indoc; use vector::{ config, - test_util::{next_addr, runtime, send_lines, start_topology, wait_for_tcp, CountReceiver}, + test_util::{CountReceiver, next_addr, runtime, send_lines, start_topology, wait_for_tcp}, }; criterion_group!( @@ -256,11 +256,9 @@ fn benchmark_configs( let in_addr = next_addr(); let out_addr = next_addr(); - let lines: Vec<_> = ::std::iter::repeat(input.to_string()) - .take(num_lines) - .collect(); + let lines: Vec<_> = std::iter::repeat_n(input.to_string(), num_lines).collect(); - let mut group = criterion.benchmark_group(format!("languages/{}", benchmark_name)); + let mut group = criterion.benchmark_group(format!("languages/{benchmark_name}")); group.sampling_mode(SamplingMode::Flat); let source_config = format!( @@ -286,15 +284,15 @@ fn benchmark_configs( for (name, transform_config) in configs.into_iter() { group.throughput(Throughput::Elements(num_lines as u64)); - group.bench_function(name.clone(), |b| { + group.bench_function(name, |b| { b.iter_batched( || { let mut config = source_config.clone(); - config.push_str(&transform_config); + config.push_str(transform_config); config.push_str(&sink_config); let config = config::load_from_str(&config, config::Format::Toml) - .expect(&format!("invalid TOML configuration: {}", &config)); + .unwrap_or_else(|_| panic!("invalid TOML configuration: {}", &config)); let rt = runtime(); let (output_lines, topology) = rt.block_on(async move { let output_lines = CountReceiver::receive_lines(out_addr); @@ -322,7 +320,7 @@ fn benchmark_configs( // avoids asserting the actual == expected as the socket transform // adds dynamic keys like timestamp for (key, value) in output.iter() { - assert_eq!(Some(value), actual.get(key), "for key {}", key,); + assert_eq!(Some(value), actual.get(key), "for key {key}",); } } } diff --git a/benches/loki.rs b/benches/loki.rs index 4ff2ca4a12..e0183d10cc 100644 --- a/benches/loki.rs +++ b/benches/loki.rs @@ -1,6 +1,6 @@ use std::convert::TryFrom; -use criterion::{criterion_group, criterion_main, BatchSize, Criterion}; +use criterion::{BatchSize, Criterion, criterion_group, criterion_main}; use vector::{sinks::loki::valid_label_name, template::Template}; const VALID: [&str; 4] = ["name", " name ", "bee_bop", "a09b"]; diff --git a/benches/lua.rs b/benches/lua.rs index 0d8a409d8a..f584f47105 100644 --- a/benches/lua.rs +++ b/benches/lua.rs @@ -1,7 +1,7 @@ use std::pin::Pin; -use criterion::{criterion_group, BatchSize, Criterion, Throughput}; -use futures::{stream, SinkExt, Stream, StreamExt}; +use criterion::{BatchSize, Criterion, Throughput, criterion_group}; +use futures::{SinkExt, Stream, StreamExt, stream}; use indoc::indoc; use transforms::lua::v2::LuaConfig; use vector::{ @@ -22,7 +22,7 @@ fn bench_add_fields(c: &mut Criterion) { let benchmarks: Vec<(&str, Transform)> = vec![ ("v1", { - let source = format!("event['{}'] = '{}'", key, value); + let source = format!("event['{key}'] = '{value}'"); Transform::event_task(transforms::lua::v1::Lua::new(source, vec![]).unwrap()) }), @@ -148,9 +148,8 @@ fn bench_field_filter(c: &mut Criterion) { b.iter_batched( || (tx.clone(), events.clone()), |(mut tx, events)| { - let _ = - futures::executor::block_on(tx.send_all(&mut stream::iter(events).map(Ok))) - .unwrap(); + futures::executor::block_on(tx.send_all(&mut stream::iter(events).map(Ok))) + .unwrap(); let output = futures::executor::block_on(collect_ready(&mut rx)); diff --git a/benches/metrics_snapshot.rs b/benches/metrics_snapshot.rs index bb59ccf654..279911e5e6 100644 --- a/benches/metrics_snapshot.rs +++ b/benches/metrics_snapshot.rs @@ -1,4 +1,4 @@ -use criterion::{criterion_group, BenchmarkId, Criterion}; +use criterion::{BenchmarkId, Criterion, criterion_group}; fn benchmark(c: &mut Criterion) { let mut group = c.benchmark_group("metrics_snapshot"); @@ -23,7 +23,7 @@ fn prepare_metrics(cardinality: usize) -> &'static vector::metrics::Controller { controller.reset(); for idx in 0..cardinality { - metrics::counter!("test", 1, "idx" => idx.to_string()); + metrics::counter!("test", "idx" => idx.to_string()).increment(1); } controller diff --git a/benches/remap.rs b/benches/remap.rs index 5eedd24445..518d2cc4d4 100644 --- a/benches/remap.rs +++ b/benches/remap.rs @@ -1,13 +1,13 @@ use std::collections::HashMap; use chrono::{DateTime, Utc}; -use criterion::{criterion_group, criterion_main, BatchSize, Criterion}; +use criterion::{BatchSize, Criterion, criterion_group, criterion_main}; use vector::{ config::{DataType, TransformOutput}, event::{Event, LogEvent, Value}, transforms::{ - remap::{Remap, RemapConfig}, SyncTransform, TransformOutputsBuf, + remap::{Remap, RemapConfig}, }, }; use vrl::event_path; @@ -27,7 +27,7 @@ fn benchmark_remap(c: &mut Criterion) { let add_fields_runner = |tform: &mut Box, event: Event| { let mut outputs = TransformOutputsBuf::new_with_capacity( - vec![TransformOutput::new(DataType::all(), HashMap::new())], + vec![TransformOutput::new(DataType::all_bits(), HashMap::new())], 1, ); tform.transform(event, &mut outputs); @@ -90,7 +90,7 @@ fn benchmark_remap(c: &mut Criterion) { let json_parser_runner = |tform: &mut Box, event: Event| { let mut outputs = TransformOutputsBuf::new_with_capacity( - vec![TransformOutput::new(DataType::all(), HashMap::new())], + vec![TransformOutput::new(DataType::all_bits(), HashMap::new())], 1, ); tform.transform(event, &mut outputs); @@ -144,7 +144,7 @@ fn benchmark_remap(c: &mut Criterion) { let coerce_runner = |tform: &mut Box, event: Event, timestamp: DateTime| { let mut outputs = TransformOutputsBuf::new_with_capacity( - vec![TransformOutput::new(DataType::all(), HashMap::new())], + vec![TransformOutput::new(DataType::all_bits(), HashMap::new())], 1, ); tform.transform(event, &mut outputs); diff --git a/benches/template.rs b/benches/template.rs index 008df426c9..ae34292a7e 100644 --- a/benches/template.rs +++ b/benches/template.rs @@ -1,7 +1,7 @@ use std::convert::TryFrom; use chrono::Utc; -use criterion::{criterion_group, BatchSize, Criterion}; +use criterion::{BatchSize, Criterion, criterion_group}; use vector::{config::log_schema, event::Event, event::LogEvent}; fn bench_elasticsearch_index(c: &mut Criterion) { diff --git a/benches/transform/common.rs b/benches/transform/common.rs index 1760c5e6a5..d56c2e46ec 100644 --- a/benches/transform/common.rs +++ b/benches/transform/common.rs @@ -6,7 +6,7 @@ use std::{ task::{Context, Poll}, }; -use futures::{task::noop_waker, Stream}; +use futures::{Stream, task::noop_waker}; use vector::event::{Event, LogEvent}; // == Streams == @@ -38,7 +38,7 @@ impl FixedLogStream { let mut events = Vec::with_capacity(total.get()); let mut cycle = 0; for _ in 0..total.get() { - events.push(Event::Log(LogEvent::from(format!("event{}", cycle)))); + events.push(Event::Log(LogEvent::from(format!("event{cycle}")))); cycle = (cycle + 1) % cycle_size; } Self::new_from_vec(events) diff --git a/benches/transform/dedupe.rs b/benches/transform/dedupe.rs index 5df5a5cf96..743b66e209 100644 --- a/benches/transform/dedupe.rs +++ b/benches/transform/dedupe.rs @@ -1,15 +1,16 @@ use core::fmt; use std::{num::NonZeroUsize, time::Duration}; +use crate::common::{FixedLogStream, consume}; use criterion::{ - criterion_group, measurement::WallTime, BatchSize, BenchmarkGroup, BenchmarkId, Criterion, - SamplingMode, Throughput, + BatchSize, BenchmarkGroup, BenchmarkId, Criterion, SamplingMode, Throughput, criterion_group, + measurement::WallTime, }; -use vector::transforms::dedupe::{CacheConfig, Dedupe, DedupeConfig, FieldMatchConfig}; +use vector::transforms::dedupe::common::{CacheConfig, FieldMatchConfig, TimedCacheConfig}; +use vector::transforms::dedupe::config::DedupeConfig; +use vector::transforms::dedupe::transform::Dedupe; use vector_lib::transform::Transform; -use crate::common::{consume, FixedLogStream}; - #[derive(Debug)] struct Param { slug: &'static str, @@ -44,6 +45,7 @@ fn dedupe(c: &mut Criterion) { dedupe_config: DedupeConfig { fields: Some(FieldMatchConfig::IgnoreFields(vec!["message".into()])), cache: cache.clone(), + time_settings: None, }, }, // Modification of previous where field "message" is matched. @@ -53,6 +55,33 @@ fn dedupe(c: &mut Criterion) { dedupe_config: DedupeConfig { fields: Some(FieldMatchConfig::MatchFields(vec!["message".into()])), cache: cache.clone(), + time_settings: None, + }, + }, + // Modification of previous where deduplication with max age is used. + Param { + slug: "field_match_message_timed", + input: fixed_stream.clone(), + dedupe_config: DedupeConfig { + fields: Some(FieldMatchConfig::MatchFields(vec!["message".into()])), + cache: cache.clone(), + time_settings: Some(TimedCacheConfig { + max_age_ms: Duration::from_secs(5), + refresh_on_drop: false, + }), + }, + }, + // Modification of previous where refresh on drop is enabled. + Param { + slug: "field_match_message_timed_refresh_on_drop", + input: fixed_stream.clone(), + dedupe_config: DedupeConfig { + fields: Some(FieldMatchConfig::MatchFields(vec!["message".into()])), + cache: cache.clone(), + time_settings: Some(TimedCacheConfig { + max_age_ms: Duration::from_secs(5), + refresh_on_drop: true, + }), }, }, // Measurement where ignore fields do not exist in the event. @@ -68,6 +97,7 @@ fn dedupe(c: &mut Criterion) { "cdeab".into(), "bcdea".into(), ])), + time_settings: None, }, }, // Modification of previous where match fields do not exist in the @@ -84,6 +114,7 @@ fn dedupe(c: &mut Criterion) { "cdeab".into(), "bcdea".into(), ])), + time_settings: None, }, }, ] { @@ -91,8 +122,12 @@ fn dedupe(c: &mut Criterion) { group.bench_with_input(BenchmarkId::new("transform", param), ¶m, |b, param| { b.iter_batched( || { - let dedupe = - Transform::event_task(Dedupe::new(param.dedupe_config.clone())).into_task(); + let config = param.dedupe_config.clone(); + let dedupe = Transform::event_task(Dedupe::new( + config.cache.num_events, + config.fields.unwrap(), + )) + .into_task(); (Box::new(dedupe), Box::pin(param.input.clone())) }, |(dedupe, input)| { diff --git a/benches/transform/filter.rs b/benches/transform/filter.rs index c2e7b2f722..a1538cd090 100644 --- a/benches/transform/filter.rs +++ b/benches/transform/filter.rs @@ -1,12 +1,12 @@ use std::time::Duration; use criterion::{ - criterion_group, measurement::WallTime, BatchSize, BenchmarkGroup, Criterion, SamplingMode, - Throughput, + BatchSize, BenchmarkGroup, Criterion, SamplingMode, Throughput, criterion_group, + measurement::WallTime, }; use vector::{ conditions::Condition, - transforms::{filter::Filter, FunctionTransform, OutputBuffer}, + transforms::{FunctionTransform, OutputBuffer, filter::Filter}, }; use vector_lib::event::{Event, LogEvent}; diff --git a/benches/transform/reduce.rs b/benches/transform/reduce.rs index 9f0e4ed60f..0e074d5c08 100644 --- a/benches/transform/reduce.rs +++ b/benches/transform/reduce.rs @@ -1,16 +1,16 @@ use core::fmt; use std::{num::NonZeroUsize, time::Duration}; +use crate::common::{FixedLogStream, consume}; use criterion::{ - criterion_group, measurement::WallTime, BatchSize, BenchmarkGroup, BenchmarkId, Criterion, - SamplingMode, Throughput, + BatchSize, BenchmarkGroup, BenchmarkId, Criterion, SamplingMode, Throughput, criterion_group, + measurement::WallTime, }; use indexmap::IndexMap; -use vector::transforms::reduce::{Reduce, ReduceConfig}; +use vector::transforms::reduce::config::ReduceConfig; +use vector::transforms::reduce::transform::Reduce; use vector_lib::transform::Transform; -use crate::common::{consume, FixedLogStream}; - #[derive(Debug)] struct Param { slug: &'static str, @@ -33,17 +33,13 @@ fn reduce(c: &mut Criterion) { NonZeroUsize::new(128).unwrap(), NonZeroUsize::new(2).unwrap(), ); - for param in &[ - // The `Reduce` transform has a high configuration surface. For now we - // only benchmark the "proof of concept" configuration, demonstrating - // that the benchmark does minimally work. Once we have soak tests with - // reduces in them we should extend this array to include those - // configurations. - Param { + { + let param = &Param { slug: "proof_of_concept", input: fixed_stream.clone(), reduce_config: ReduceConfig { expire_after_ms: Duration::from_secs(30), + end_every_period_ms: None, flush_period_ms: Duration::from_secs(1), group_by: vec![String::from("message")], merge_strategies: IndexMap::default(), @@ -51,8 +47,7 @@ fn reduce(c: &mut Criterion) { starts_when: None, max_events: None, }, - }, - ] { + }; group.throughput(Throughput::Elements(param.input.len() as u64)); group.bench_with_input(BenchmarkId::new("transform", param), ¶m, |b, param| { b.to_async(tokio::runtime::Runtime::new().unwrap()) diff --git a/benches/transform/route.rs b/benches/transform/route.rs index 0ef6efa30b..33b0caa52e 100644 --- a/benches/transform/route.rs +++ b/benches/transform/route.rs @@ -3,13 +3,13 @@ use std::time::Duration; use bytes::Bytes; use criterion::{ - black_box, criterion_group, measurement::WallTime, BatchSize, BenchmarkGroup, BenchmarkId, - Criterion, SamplingMode, Throughput, + BatchSize, BenchmarkGroup, BenchmarkId, Criterion, SamplingMode, Throughput, criterion_group, + measurement::WallTime, }; use vector::config::TransformContext; use vector::transforms::{ - route::{Route, RouteConfig}, TransformOutputsBuf, + route::{Route, RouteConfig}, }; use vector_lib::{ config::{DataType, TransformOutput}, @@ -159,7 +159,8 @@ fn route(c: &mut Criterion) { (route, param.input.clone(), param.output_buffer.clone()) }, |(mut route, input, mut output_buffer)| { - black_box(route.transform(input, &mut output_buffer)); + route.transform(input, &mut output_buffer); + std::hint::black_box(()); }, BatchSize::SmallInput, ) diff --git a/build.rs b/build.rs index b603074f90..cf35846cbe 100644 --- a/build.rs +++ b/build.rs @@ -20,7 +20,7 @@ impl TrackedEnv { pub fn emit_rerun_stanzas(&self) { for env_var in &self.tracked { - println!("cargo:rerun-if-env-changed={}", env_var); + println!("cargo:rerun-if-env-changed={env_var}"); } } } @@ -33,9 +33,9 @@ enum ConstantValue { impl ConstantValue { pub fn as_parts(&self) -> (&'static str, String) { match &self { - ConstantValue::Required(value) => ("&str", format!("\"{}\"", value)), + ConstantValue::Required(value) => ("&str", format!("\"{value}\"")), ConstantValue::Optional(value) => match value { - Some(value) => ("Option<&str>", format!("Some(\"{}\")", value)), + Some(value) => ("Option<&str>", format!("Some(\"{value}\")")), None => ("Option<&str>", "None".to_string()), }, } @@ -79,10 +79,8 @@ impl BuildConstants { for (name, desc, value) in self.values { let (const_type, const_val) = value.as_parts(); - let full = format!( - "#[doc=r#\"{}\"#]\npub const {}: {} = {};\n", - desc, name, const_type, const_val - ); + let full = + format!("#[doc=r#\"{desc}\"#]\npub const {name}: {const_type} = {const_val};\n"); output_file.write_all(full.as_ref())?; } @@ -202,10 +200,7 @@ fn main() { .map_err(|e| { #[allow(clippy::print_stderr)] { - eprintln!( - "Unable to determine git short hash from rev-parse command: {}", - e - ); + eprintln!("Unable to determine git short hash from rev-parse command: {e}"); } }) .expect("git hash detection failed"); diff --git a/changelog.d/21756_drain_events_when_shutdown_nats_source.enhancement.md b/changelog.d/21756_drain_events_when_shutdown_nats_source.enhancement.md new file mode 100644 index 0000000000..05a3ff849e --- /dev/null +++ b/changelog.d/21756_drain_events_when_shutdown_nats_source.enhancement.md @@ -0,0 +1,3 @@ +The `nats` source now drains subscriptions during shutdown, ensuring that in-flight and pending messages are processed. + +authors: benjamin-awd diff --git a/changelog.d/21972-add-tcp-collector-host-metrics.feature.md b/changelog.d/21972-add-tcp-collector-host-metrics.feature.md deleted file mode 100644 index 04eab2b916..0000000000 --- a/changelog.d/21972-add-tcp-collector-host-metrics.feature.md +++ /dev/null @@ -1,14 +0,0 @@ -The `host_metrics` source has a new collector, `tcp`. The `tcp` -collector exposes three metrics related to the TCP stack of the -system: - -* `tcp_connections_total`: The total number of TCP connections. It - includes the `state` of the connection as a tag. -* `tcp_tx_queued_bytes_total`: The sum of the number of bytes in the - send queue across all connections. -* `tcp_rx_queued_bytes_total`: The sum of the number of bytes in the - receive queue across all connections. - -This collector is enabled only on Linux systems. - -authors: aryan9600 diff --git a/changelog.d/22007_chronicle_ingest_header.fix.md b/changelog.d/22007_chronicle_ingest_header.fix.md deleted file mode 100644 index 970453dfd4..0000000000 --- a/changelog.d/22007_chronicle_ingest_header.fix.md +++ /dev/null @@ -1,3 +0,0 @@ -The `chronicle_unstructured` sink now sets the `content-encoding` header when compression is enabled. - -authors: chocpanda diff --git a/changelog.d/22029_support_jetstream_in_nats_source.enhancement.md b/changelog.d/22029_support_jetstream_in_nats_source.enhancement.md new file mode 100644 index 0000000000..f147b3ef42 --- /dev/null +++ b/changelog.d/22029_support_jetstream_in_nats_source.enhancement.md @@ -0,0 +1,3 @@ +Adds JetStream support to the `nats` source. + +authors: benjamin-awd diff --git a/changelog.d/22968_okta.feature.md b/changelog.d/22968_okta.feature.md new file mode 100644 index 0000000000..6c69ebcdbf --- /dev/null +++ b/changelog.d/22968_okta.feature.md @@ -0,0 +1,3 @@ +Introduced a new `okta` source for consuming [Okta system logs](https://developer.okta.com/docs/api/openapi/okta-management/management/tag/SystemLog/) + +authors: sonnens diff --git a/changelog.d/add-session-name-config-for-aws.fix.md b/changelog.d/add-session-name-config-for-aws.fix.md deleted file mode 100644 index 2a3a758c87..0000000000 --- a/changelog.d/add-session-name-config-for-aws.fix.md +++ /dev/null @@ -1,3 +0,0 @@ -Allow users to specify `session_name` when using aws auth. - -authors: akutta diff --git a/changelog.d/add_chronicle_regional_endpoints.enhancement.md b/changelog.d/add_chronicle_regional_endpoints.enhancement.md deleted file mode 100644 index 91e03f2fd8..0000000000 --- a/changelog.d/add_chronicle_regional_endpoints.enhancement.md +++ /dev/null @@ -1,3 +0,0 @@ -Add support for more chronicle regional endpoints as listed - https://cloud.google.com/chronicle/docs/reference/ingestion-api#regional_endpoints - -authors: chocpanda diff --git a/changelog.d/add_support_v1_1_secrets.feature.md b/changelog.d/add_support_v1_1_secrets.feature.md new file mode 100644 index 0000000000..bfe45f2f56 --- /dev/null +++ b/changelog.d/add_support_v1_1_secrets.feature.md @@ -0,0 +1,3 @@ +Add support for v1.1 of the secrets manager protocol + +authors: graphcareful diff --git a/changelog.d/add_virtual_memory_process_metric.feature.md b/changelog.d/add_virtual_memory_process_metric.feature.md deleted file mode 100644 index 422232018e..0000000000 --- a/changelog.d/add_virtual_memory_process_metric.feature.md +++ /dev/null @@ -1,3 +0,0 @@ -Add virtual memory metric to the process host metrics collector. - -authors: nionata diff --git a/changelog.d/incremental_to_absolute_transform.feature.md b/changelog.d/incremental_to_absolute_transform.feature.md new file mode 100644 index 0000000000..623ee48bab --- /dev/null +++ b/changelog.d/incremental_to_absolute_transform.feature.md @@ -0,0 +1,6 @@ +Add a new `incremental_to_absolute` transform which converts incremental metrics to absolute metrics. This is useful for +use cases when sending metrics to a sink is lossy or you want to get a historical record of metrics, in which case +incremental metrics may be inaccurate since any gaps in metrics sent will result in an inaccurate reading of the ending +value. + +authors: GreyLilac09 diff --git a/changelog.d/log_global_config_diff.feature.md b/changelog.d/log_global_config_diff.feature.md new file mode 100644 index 0000000000..4eb87cc0d8 --- /dev/null +++ b/changelog.d/log_global_config_diff.feature.md @@ -0,0 +1,3 @@ +When config reload is aborted due to `GlobalOptions` changes, the specific top-level fields that differ are now logged to help debugging. + +authors: suikammd diff --git a/changelog.d/otlp_decoding.enhancement.md b/changelog.d/otlp_decoding.enhancement.md new file mode 100644 index 0000000000..3376717795 --- /dev/null +++ b/changelog.d/otlp_decoding.enhancement.md @@ -0,0 +1,8 @@ +The `opentelemetry` source now supports a new decoding mode which can be enabled by setting `use_otlp_decoding` to `true`. In this mode, +all events preserve the [OTLP](https://opentelemetry.io/docs/specs/otel/protocol/) format. These events can be forwarded directly to +the `opentelemetry` sink without modifications. + +**Note:** The OTLP metric format and the Vector metric format differ, so the `opentelemetry` source emits OTLP formatted metrics as Vector log +events. These events cannot be used with existing metrics transforms. However, they can be ingested by the OTEL collectors as metrics. + +authors: pront diff --git a/changelog.d/protobuf_varint_framing_support.enhancement.md b/changelog.d/protobuf_varint_framing_support.enhancement.md new file mode 100644 index 0000000000..0aeec4e100 --- /dev/null +++ b/changelog.d/protobuf_varint_framing_support.enhancement.md @@ -0,0 +1,5 @@ +Added support for varint length delimited framing for protobuf, which is compatible with standard protobuf streaming implementations and tools like ClickHouse. + +Users can now opt-in to varint framing by explicitly specifying `framing.method: varint_length_delimited` in their configuration. The default remains length-delimited framing for backward compatibility. + +authors: modev2301 diff --git a/changelog.d/s3-log-spam.fix.md b/changelog.d/s3-log-spam.fix.md deleted file mode 100644 index 90f019f7bf..0000000000 --- a/changelog.d/s3-log-spam.fix.md +++ /dev/null @@ -1,3 +0,0 @@ -Downgraded some noisy `info!` statements in the `aws_s3` source to `debug!`. - -authors: pront diff --git a/clippy.toml b/clippy.toml index 25937ac641..20274f646b 100644 --- a/clippy.toml +++ b/clippy.toml @@ -1,4 +1,4 @@ -cognitive-complexity-threshold = 75 +large-error-threshold = 256 # in bytes # for `disallowed_method`: # https://rust-lang.github.io/rust-clippy/master/index.html#disallowed_method diff --git a/config/examples/environment_variables.yaml b/config/examples/environment_variables.yaml index 544ce90786..2f73528397 100644 --- a/config/examples/environment_variables.yaml +++ b/config/examples/environment_variables.yaml @@ -9,7 +9,7 @@ data_dir: "/var/lib/vector" # Ingests Apache 2 log data by tailing one or more log files # Example: 194.221.90.140 - - [22/06/2019:11:55:14 -0400] "PUT /integrate" 100 2213 -# Docs: https://vector.dev/docs/reference/sources/file +# Docs: https://vector.dev/docs/reference/configuration/sources/file sources: apache_logs: type: "file" @@ -18,7 +18,7 @@ sources: ignore_older_secs: 86400 # Add a field based on the value of the HOSTNAME env var -# Docs: https://vector.dev/docs/reference/transforms/remap +# Docs: https://vector.dev/docs/reference/configuration/transforms/remap transforms: add_host: inputs: [ "apache_logs" ] @@ -29,7 +29,7 @@ transforms: ''' # Print the data to STDOUT for inspection -# Docs: https://vector.dev/docs/reference/sinks/console +# Docs: https://vector.dev/docs/reference/configuration/sinks/console sinks: out: inputs: [ "add_host" ] diff --git a/config/examples/es_s3_hybrid.yaml b/config/examples/es_s3_hybrid.yaml index df221e64cc..b298ef6592 100644 --- a/config/examples/es_s3_hybrid.yaml +++ b/config/examples/es_s3_hybrid.yaml @@ -8,7 +8,7 @@ data_dir: "/var/lib/vector" # Ingest data by tailing one or more files -# Docs: https://vector.dev/docs/reference/sources/file +# Docs: https://vector.dev/docs/reference/configuration/sources/file sources: apache_logs: type: "file" @@ -16,7 +16,7 @@ sources: ignore_older_secs: 86400 # 1 day # Optionally parse, structure and transform data here. -# Docs: https://vector.dev/docs/reference/transforms +# Docs: https://vector.dev/docs/reference/configuration/transforms # Send structured data to Elasticsearch for searching of recent data sinks: diff --git a/config/examples/varint_framing_protobuf.yaml b/config/examples/varint_framing_protobuf.yaml new file mode 100644 index 0000000000..a0000d3d2a --- /dev/null +++ b/config/examples/varint_framing_protobuf.yaml @@ -0,0 +1,56 @@ +# Example configuration demonstrating varint framing for protobuf +# This is compatible with tools like ClickHouse that use protobuf with varint length prefixes + +sources: + protobuf_source: + type: socket + mode: tcp + address: "0.0.0.0:8080" + decoding: + codec: protobuf + protobuf: + desc_file: "path/to/your/protobuf.desc" + message_type: "your.package.MessageType" + framing: + method: varint_length_delimited + + socket_source: + type: socket + mode: tcp + address: "0.0.0.0:8081" + decoding: + codec: protobuf + protobuf: + desc_file: "input.desc" + message_type: "input.Message" + framing: + method: varint_length_delimited + +sinks: + protobuf_sink: + inputs: + - protobuf_source + type: socket + mode: tcp + address: "localhost:9090" + encoding: + codec: protobuf + protobuf: + desc_file: "path/to/your/protobuf.desc" + message_type: "your.package.MessageType" + framing: + method: varint_length_delimited + + socket_output: + inputs: + - socket_source + type: socket + mode: tcp + address: "localhost:9090" + encoding: + codec: protobuf + protobuf: + desc_file: "output.desc" + message_type: "output.Message" + framing: + method: varint_length_delimited diff --git a/config/examples/wrapped_json.yaml b/config/examples/wrapped_json.yaml index fb14c07724..90fed711f2 100644 --- a/config/examples/wrapped_json.yaml +++ b/config/examples/wrapped_json.yaml @@ -8,7 +8,7 @@ data_dir: "/var/lib/vector" # Ingest data # Example: {"message": "{\"parent\": \"{\\\"child\\\": \\\"value2\\\"}\"}"} -# Docs: https://vector.dev/docs/reference/sources/file +# Docs: https://vector.dev/docs/reference/configuration/sources/file sources: logs: type: "file" @@ -16,7 +16,7 @@ sources: ignore_older_secs: 86400 # 1 day # Parse the data as JSON -# Docs: https://vector.dev/docs/reference/transforms/remap +# Docs: https://vector.dev/docs/reference/configuration/transforms/remap transforms: parse_json: inputs: [ "logs" ] @@ -33,7 +33,7 @@ transforms: . |= object!(parse_json!(string!(child))) # Print the data to STDOUT for inspection -# Docs: https://vector.dev/docs/reference/sinks/console +# Docs: https://vector.dev/docs/reference/configuration/sinks/console sinks: out: inputs: [ "parse_json" ] diff --git a/deny.toml b/deny.toml index 62104525de..7e200969f6 100644 --- a/deny.toml +++ b/deny.toml @@ -8,6 +8,7 @@ allow = [ "CC0-1.0", "ISC", "MIT", + "MIT-0", "OpenSSL", "Unicode-3.0", "Unicode-DFS-2016", @@ -36,30 +37,28 @@ license-files = [ [advisories] ignore = [ - # Vulnerability in `rsa` crate: https://rustsec.org/advisories/RUSTSEC-2023-0071.html - # There is not fix available yet. - # https://github.com/vectordotdev/vector/issues/19262 - "RUSTSEC-2023-0071", - - # Vulnerability in `tonic` crate: https://rustsec.org/advisories/RUSTSEC-2024-0376 - # There is a fixed version (v0.12.3) but we are blocked from upgrading to `http` v1, which - # `tonic` v0.12 depends on. See https://github.com/vectordotdev/vector/issues/19179 - "RUSTSEC-2024-0376", - - # Advisory in rustls crate: https://rustsec.org/advisories/RUSTSEC-2024-0336 If a `close_notify` - # alert is received during a handshake, `complete_io` does not terminate. - # Vulnerable version only used in dev-dependencies - "RUSTSEC-2024-0336", - - # idna accepts Punycode labels that do not produce any non-ASCII when decoded - # Need to update some direct dependencies before we can upgrade idna to fix - "RUSTSEC-2024-0421", - - # unmaintained dependencies - "RUSTSEC-2021-0139", # ansi_term - "RUSTSEC-2024-0388", # derivative - "RUSTSEC-2024-0384", # instant` - "RUSTSEC-2020-0168", # mach - "RUSTSEC-2024-0370", # proc-macro-error - "RUSTSEC-2024-0320", # yaml-rust + # Vulnerability in `rsa` crate: https://rustsec.org/advisories/RUSTSEC-2023-0071.html + # There is not fix available yet. + # https://github.com/vectordotdev/vector/issues/19262 + "RUSTSEC-2023-0071", + # Vulnerability in `tonic` crate: https://rustsec.org/advisories/RUSTSEC-2024-0376 + # There is a fixed version (v0.12.3) but we are blocked from upgrading to `http` v1, which + # `tonic` v0.12 depends on. See https://github.com/vectordotdev/vector/issues/19179 + "RUSTSEC-2024-0376", + # Advisory in rustls crate: https://rustsec.org/advisories/RUSTSEC-2024-0336 If a `close_notify` + # alert is received during a handshake, `complete_io` does not terminate. + # Vulnerable version only used in dev-dependencies + "RUSTSEC-2024-0336", + # idna accepts Punycode labels that do not produce any non-ASCII when decoded + # Need to update some direct dependencies before we can upgrade idna to fix + "RUSTSEC-2024-0421", + { id = "RUSTSEC-2021-0139", reason = " ansi_term is unmaintained" }, + { id = "RUSTSEC-2024-0388", reason = "derivative is unmaintained" }, + { id = "RUSTSEC-2024-0384", reason = "instant is unmaintained" }, + { id = "RUSTSEC-2020-0168", reason = "mach is unmaintained" }, + { id = "RUSTSEC-2024-0370", reason = "proc-macro-error is unmaintained" }, + { id = "RUSTSEC-2024-0320", reason = "yaml-rust is unmaintained" }, + { id = "RUSTSEC-2024-0436", reason = "paste is unmaintained" }, + { id = "RUSTSEC-2025-0012", reason = "backoff is unmaintained" }, + { id = "RUSTSEC-2025-0014", reason = "humantime is unmaintained" }, ] diff --git a/distribution/docker/README.md b/distribution/docker/README.md index 249e76c43b..b6e6a66ea6 100644 --- a/distribution/docker/README.md +++ b/distribution/docker/README.md @@ -120,12 +120,12 @@ Vector's Docker source files are located [docs.administration]: https://vector.dev/docs/administration/ [docs.setup.configuration]: https://vector.dev/docs/setup/configuration/ [docs.deployment]: https://vector.dev/docs/setup/deployment/ -[docs.sinks]: https://vector.dev/docs/reference/sinks/ -[docs.sources]: https://vector.dev/docs/reference/sources/ +[docs.sinks]: https://vector.dev/docs/reference/configuration/sinks/ +[docs.sources]: https://vector.dev/docs/reference/configuration/sources/ [docs.strategies#daemon]: https://vector.dev/docs/setup/deployment/strategies/#daemon [docs.strategies#service]: https://vector.dev/docs/setup/deployment/strategies/#service [docs.strategies#sidecar]: https://vector.dev/docs/setup/deployment/strategies/#sidecar -[docs.transforms]: https://vector.dev/docs/reference/transforms/ +[docs.transforms]: https://vector.dev/docs/reference/configurationtransforms/ [pages.index#correctness]: https://vector.dev/#correctness [pages.index#performance]: https://vector.dev/#performance [urls.default_configuration]: https://github.com/vectordotdev/vector/blob/master/config/vector.yaml diff --git a/distribution/docker/alpine/Dockerfile b/distribution/docker/alpine/Dockerfile index 1e56e5b6aa..45ae3194b2 100644 --- a/distribution/docker/alpine/Dockerfile +++ b/distribution/docker/alpine/Dockerfile @@ -12,7 +12,13 @@ RUN ARCH=$(if [ "$TARGETPLATFORM" = "linux/arm/v6" ]; then echo "arm"; else cat RUN mkdir -p /var/lib/vector -FROM docker.io/alpine:3.21 +FROM docker.io/alpine:3.22 + +# https://github.com/opencontainers/image-spec/blob/main/annotations.md +LABEL org.opencontainers.image.url="https://vector.dev" +LABEL org.opencontainers.image.source="https://github.com/vectordotdev/vector" +LABEL org.opencontainers.image.documentation="https://vector.dev/docs" + # we want the latest versions of these # hadolint ignore=DL3018 RUN apk --no-cache add ca-certificates tzdata diff --git a/distribution/docker/debian/Dockerfile b/distribution/docker/debian/Dockerfile index c678dfd308..f037a7f06e 100644 --- a/distribution/docker/debian/Dockerfile +++ b/distribution/docker/debian/Dockerfile @@ -9,6 +9,11 @@ RUN mkdir -p /var/lib/vector FROM docker.io/debian:bookworm-slim +# https://github.com/opencontainers/image-spec/blob/main/annotations.md +LABEL org.opencontainers.image.url="https://vector.dev" +LABEL org.opencontainers.image.source="https://github.com/vectordotdev/vector" +LABEL org.opencontainers.image.documentation="https://vector.dev/docs" + # we want the latest versions of these # hadolint ignore=DL3008 RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates tzdata systemd && rm -rf /var/lib/apt/lists/* diff --git a/distribution/docker/distroless-libc/Dockerfile b/distribution/docker/distroless-libc/Dockerfile index 2a0f7084c9..b9dae8ff56 100644 --- a/distribution/docker/distroless-libc/Dockerfile +++ b/distribution/docker/distroless-libc/Dockerfile @@ -11,6 +11,11 @@ RUN mkdir -p /var/lib/vector # hadolint ignore=DL3007 FROM gcr.io/distroless/cc-debian12:latest +# https://github.com/opencontainers/image-spec/blob/main/annotations.md +LABEL org.opencontainers.image.url="https://vector.dev" +LABEL org.opencontainers.image.source="https://github.com/vectordotdev/vector" +LABEL org.opencontainers.image.documentation="https://vector.dev/docs" + COPY --from=builder /usr/bin/vector /usr/bin/vector COPY --from=builder /usr/share/doc/vector /usr/share/doc/vector COPY --from=builder /usr/share/vector /usr/share/vector diff --git a/distribution/docker/distroless-static/Dockerfile b/distribution/docker/distroless-static/Dockerfile index 0c54991b93..2764d89934 100644 --- a/distribution/docker/distroless-static/Dockerfile +++ b/distribution/docker/distroless-static/Dockerfile @@ -11,6 +11,11 @@ RUN mkdir -p /var/lib/vector # hadolint ignore=DL3007 FROM gcr.io/distroless/static:latest +# https://github.com/opencontainers/image-spec/blob/main/annotations.md +LABEL org.opencontainers.image.url="https://vector.dev" +LABEL org.opencontainers.image.source="https://github.com/vectordotdev/vector" +LABEL org.opencontainers.image.documentation="https://vector.dev/docs" + COPY --from=builder /vector/bin/* /usr/local/bin/ COPY --from=builder /vector/config/vector.yaml /etc/vector/vector.yaml COPY --from=builder /var/lib/vector /var/lib/vector diff --git a/distribution/install.sh b/distribution/install.sh index c14713c482..ceaad6bae3 100755 --- a/distribution/install.sh +++ b/distribution/install.sh @@ -13,7 +13,7 @@ set -u # If PACKAGE_ROOT is unset or empty, default it. PACKAGE_ROOT="${PACKAGE_ROOT:-"https://packages.timber.io/vector"}" # If VECTOR_VERSION is unset or empty, default it. -VECTOR_VERSION="${VECTOR_VERSION:-"0.44.0"}" +VECTOR_VERSION="${VECTOR_VERSION:-"0.49.0"}" _divider="--------------------------------------------------------------------------------" _prompt=">>>" _indent=" " @@ -594,21 +594,25 @@ downloader() { if [ "$1" = --check ]; then need_cmd "$_dld" elif [ "$_dld" = curl ]; then - check_curl_for_retry_support - _retry="$RETVAL" + if check_curl_for_retry_support; then + _retry=(--retry 3) + else + _retry=() + fi + get_ciphersuites_for_curl _ciphersuites="$RETVAL" if [ -n "$_ciphersuites" ]; then - _err=$(curl $_retry --proto '=https' --tlsv1.2 --ciphers "$_ciphersuites" --silent --show-error --fail --location "$1" --output "$2" 2>&1) + _err=$(curl "${_retry[@]}" --proto '=https' --tlsv1.2 --ciphers "$_ciphersuites" --silent --show-error --fail --location "$1" --output "$2" 2>&1) _status=$? else echo "Warning: Not enforcing strong cipher suites for TLS, this is potentially less secure" if ! check_help_for "$3" curl --proto --tlsv1.2; then echo "Warning: Not enforcing TLS v1.2, this is potentially less secure" - _err=$(curl $_retry --silent --show-error --fail --location "$1" --output "$2" 2>&1) + _err=$(curl "${_retry[@]}" --silent --show-error --fail --location "$1" --output "$2" 2>&1) _status=$? else - _err=$(curl $_retry --proto '=https' --tlsv1.2 --silent --show-error --fail --location "$1" --output "$2" 2>&1) + _err=$(curl "${_retry[@]}" --proto '=https' --tlsv1.2 --silent --show-error --fail --location "$1" --output "$2" 2>&1) _status=$? fi fi @@ -707,16 +711,13 @@ check_help_for() { true # not strictly needed } -# Check if curl supports the --retry flag, then pass it to the curl invocation. +# Check if curl supports the --retry flag check_curl_for_retry_support() { - local _retry_supported="" - # "unspecified" is for arch, allows for possibility old OS using macports, homebrew, etc. if check_help_for "notspecified" "curl" "--retry"; then - _retry_supported="--retry 3" + return 0 + else + return 1 fi - - RETVAL="$_retry_supported" - } # Return cipher suite string specified by user, otherwise return strong TLS 1.2-1.3 cipher suites diff --git a/distribution/kubernetes/vector-agent/README.md b/distribution/kubernetes/vector-agent/README.md index b38f5f345b..4c0cfbf8fc 100644 --- a/distribution/kubernetes/vector-agent/README.md +++ b/distribution/kubernetes/vector-agent/README.md @@ -1,6 +1,6 @@ The kubernetes manifests found in this directory have been automatically generated from the [helm chart `vector/vector`](https://github.com/vectordotdev/helm-charts/tree/master/charts/vector) -version 0.40.0 with the following `values.yaml`: +version 0.45.0 with the following `values.yaml`: ```yaml role: Agent diff --git a/distribution/kubernetes/vector-agent/configmap.yaml b/distribution/kubernetes/vector-agent/configmap.yaml index c88067f675..1e0f0d561d 100644 --- a/distribution/kubernetes/vector-agent/configmap.yaml +++ b/distribution/kubernetes/vector-agent/configmap.yaml @@ -4,12 +4,12 @@ apiVersion: v1 kind: ConfigMap metadata: name: vector - namespace: default + namespace: "default" labels: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Agent - app.kubernetes.io/version: "0.44.0-distroless-libc" + app.kubernetes.io/version: "0.49.0-distroless-libc" data: agent.yaml: | data_dir: /vector-data-dir diff --git a/distribution/kubernetes/vector-agent/daemonset.yaml b/distribution/kubernetes/vector-agent/daemonset.yaml index a6f5a9a13f..910d1b0428 100644 --- a/distribution/kubernetes/vector-agent/daemonset.yaml +++ b/distribution/kubernetes/vector-agent/daemonset.yaml @@ -4,12 +4,12 @@ apiVersion: apps/v1 kind: DaemonSet metadata: name: vector - namespace: default + namespace: "default" labels: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Agent - app.kubernetes.io/version: "0.44.0-distroless-libc" + app.kubernetes.io/version: "0.49.0-distroless-libc" spec: selector: matchLabels: @@ -30,7 +30,7 @@ spec: dnsPolicy: ClusterFirst containers: - name: vector - image: "timberio/vector:0.44.0-distroless-libc" + image: "timberio/vector:0.49.0-distroless-libc" imagePullPolicy: IfNotPresent args: - --config-dir diff --git a/distribution/kubernetes/vector-agent/kustomization.yaml b/distribution/kubernetes/vector-agent/kustomization.yaml index b80393851f..f50f3bfe08 100644 --- a/distribution/kubernetes/vector-agent/kustomization.yaml +++ b/distribution/kubernetes/vector-agent/kustomization.yaml @@ -8,3 +8,4 @@ resources: - daemonset.yaml - rbac.yaml - serviceaccount.yaml + - service-headless.yaml diff --git a/distribution/kubernetes/vector-agent/rbac.yaml b/distribution/kubernetes/vector-agent/rbac.yaml index 781f84980f..d1bdf5b57f 100644 --- a/distribution/kubernetes/vector-agent/rbac.yaml +++ b/distribution/kubernetes/vector-agent/rbac.yaml @@ -10,7 +10,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Agent - app.kubernetes.io/version: "0.44.0-distroless-libc" + app.kubernetes.io/version: "0.49.0-distroless-libc" rules: - apiGroups: - "" @@ -31,7 +31,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Agent - app.kubernetes.io/version: "0.44.0-distroless-libc" + app.kubernetes.io/version: "0.49.0-distroless-libc" roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -39,4 +39,4 @@ roleRef: subjects: - kind: ServiceAccount name: vector - namespace: default + namespace: "default" diff --git a/distribution/kubernetes/vector-agent/service-headless.yaml b/distribution/kubernetes/vector-agent/service-headless.yaml index 11261e449e..f9c1ed731f 100644 --- a/distribution/kubernetes/vector-agent/service-headless.yaml +++ b/distribution/kubernetes/vector-agent/service-headless.yaml @@ -4,12 +4,12 @@ apiVersion: v1 kind: Service metadata: name: vector-headless - namespace: default + namespace: "default" labels: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Agent - app.kubernetes.io/version: "0.44.0-distroless-libc" + app.kubernetes.io/version: "0.49.0-distroless-libc" annotations: spec: clusterIP: None diff --git a/distribution/kubernetes/vector-agent/serviceaccount.yaml b/distribution/kubernetes/vector-agent/serviceaccount.yaml index 5d9c6135e2..feccbcb04d 100644 --- a/distribution/kubernetes/vector-agent/serviceaccount.yaml +++ b/distribution/kubernetes/vector-agent/serviceaccount.yaml @@ -4,10 +4,10 @@ apiVersion: v1 kind: ServiceAccount metadata: name: vector - namespace: default + namespace: "default" labels: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Agent - app.kubernetes.io/version: "0.44.0-distroless-libc" + app.kubernetes.io/version: "0.49.0-distroless-libc" automountServiceAccountToken: true diff --git a/distribution/kubernetes/vector-aggregator/README.md b/distribution/kubernetes/vector-aggregator/README.md index 30f156f9eb..cbecb3df87 100644 --- a/distribution/kubernetes/vector-aggregator/README.md +++ b/distribution/kubernetes/vector-aggregator/README.md @@ -1,6 +1,6 @@ The kubernetes manifests found in this directory have been automatically generated from the [helm chart `vector/vector`](https://github.com/vectordotdev/helm-charts/tree/master/charts/vector) -version 0.40.0 with the following `values.yaml`: +version 0.45.0 with the following `values.yaml`: ```yaml diff --git a/distribution/kubernetes/vector-aggregator/configmap.yaml b/distribution/kubernetes/vector-aggregator/configmap.yaml index 462482ce20..020bc89eed 100644 --- a/distribution/kubernetes/vector-aggregator/configmap.yaml +++ b/distribution/kubernetes/vector-aggregator/configmap.yaml @@ -4,12 +4,12 @@ apiVersion: v1 kind: ConfigMap metadata: name: vector - namespace: default + namespace: "default" labels: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Aggregator - app.kubernetes.io/version: "0.44.0-distroless-libc" + app.kubernetes.io/version: "0.49.0-distroless-libc" data: aggregator.yaml: | data_dir: /vector-data-dir diff --git a/distribution/kubernetes/vector-aggregator/service-headless.yaml b/distribution/kubernetes/vector-aggregator/service-headless.yaml index ac74dc0f29..554ced0faa 100644 --- a/distribution/kubernetes/vector-aggregator/service-headless.yaml +++ b/distribution/kubernetes/vector-aggregator/service-headless.yaml @@ -4,12 +4,12 @@ apiVersion: v1 kind: Service metadata: name: vector-headless - namespace: default + namespace: "default" labels: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Aggregator - app.kubernetes.io/version: "0.44.0-distroless-libc" + app.kubernetes.io/version: "0.49.0-distroless-libc" annotations: spec: clusterIP: None diff --git a/distribution/kubernetes/vector-aggregator/service.yaml b/distribution/kubernetes/vector-aggregator/service.yaml index b254a4f889..8daa6a12b4 100644 --- a/distribution/kubernetes/vector-aggregator/service.yaml +++ b/distribution/kubernetes/vector-aggregator/service.yaml @@ -4,12 +4,12 @@ apiVersion: v1 kind: Service metadata: name: vector - namespace: default + namespace: "default" labels: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Aggregator - app.kubernetes.io/version: "0.44.0-distroless-libc" + app.kubernetes.io/version: "0.49.0-distroless-libc" annotations: spec: ports: diff --git a/distribution/kubernetes/vector-aggregator/serviceaccount.yaml b/distribution/kubernetes/vector-aggregator/serviceaccount.yaml index 1b37f896bc..bfac691054 100644 --- a/distribution/kubernetes/vector-aggregator/serviceaccount.yaml +++ b/distribution/kubernetes/vector-aggregator/serviceaccount.yaml @@ -4,10 +4,10 @@ apiVersion: v1 kind: ServiceAccount metadata: name: vector - namespace: default + namespace: "default" labels: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Aggregator - app.kubernetes.io/version: "0.44.0-distroless-libc" + app.kubernetes.io/version: "0.49.0-distroless-libc" automountServiceAccountToken: true diff --git a/distribution/kubernetes/vector-aggregator/statefulset.yaml b/distribution/kubernetes/vector-aggregator/statefulset.yaml index cd341a8bf3..ac25b64588 100644 --- a/distribution/kubernetes/vector-aggregator/statefulset.yaml +++ b/distribution/kubernetes/vector-aggregator/statefulset.yaml @@ -4,12 +4,12 @@ apiVersion: apps/v1 kind: StatefulSet metadata: name: vector - namespace: default + namespace: "default" labels: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Aggregator - app.kubernetes.io/version: "0.44.0-distroless-libc" + app.kubernetes.io/version: "0.49.0-distroless-libc" annotations: {} spec: replicas: 1 @@ -34,7 +34,7 @@ spec: dnsPolicy: ClusterFirst containers: - name: vector - image: "timberio/vector:0.44.0-distroless-libc" + image: "timberio/vector:0.49.0-distroless-libc" imagePullPolicy: IfNotPresent args: - --config-dir diff --git a/distribution/kubernetes/vector-stateless-aggregator/README.md b/distribution/kubernetes/vector-stateless-aggregator/README.md index 1290a26a3a..a2e9e8fe81 100644 --- a/distribution/kubernetes/vector-stateless-aggregator/README.md +++ b/distribution/kubernetes/vector-stateless-aggregator/README.md @@ -1,6 +1,6 @@ The kubernetes manifests found in this directory have been automatically generated from the [helm chart `vector/vector`](https://github.com/vectordotdev/helm-charts/tree/master/charts/vector) -version 0.40.0 with the following `values.yaml`: +version 0.45.0 with the following `values.yaml`: ```yaml role: Stateless-Aggregator diff --git a/distribution/kubernetes/vector-stateless-aggregator/configmap.yaml b/distribution/kubernetes/vector-stateless-aggregator/configmap.yaml index 82e31c71ba..07174c2028 100644 --- a/distribution/kubernetes/vector-stateless-aggregator/configmap.yaml +++ b/distribution/kubernetes/vector-stateless-aggregator/configmap.yaml @@ -4,12 +4,12 @@ apiVersion: v1 kind: ConfigMap metadata: name: vector - namespace: default + namespace: "default" labels: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Stateless-Aggregator - app.kubernetes.io/version: "0.44.0-distroless-libc" + app.kubernetes.io/version: "0.49.0-distroless-libc" data: aggregator.yaml: | data_dir: /vector-data-dir diff --git a/distribution/kubernetes/vector-stateless-aggregator/deployment.yaml b/distribution/kubernetes/vector-stateless-aggregator/deployment.yaml index 53b8dc0cbd..4805960f16 100644 --- a/distribution/kubernetes/vector-stateless-aggregator/deployment.yaml +++ b/distribution/kubernetes/vector-stateless-aggregator/deployment.yaml @@ -4,12 +4,12 @@ apiVersion: apps/v1 kind: Deployment metadata: name: vector - namespace: default + namespace: "default" labels: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Stateless-Aggregator - app.kubernetes.io/version: "0.44.0-distroless-libc" + app.kubernetes.io/version: "0.49.0-distroless-libc" annotations: {} spec: replicas: 1 @@ -32,7 +32,7 @@ spec: dnsPolicy: ClusterFirst containers: - name: vector - image: "timberio/vector:0.44.0-distroless-libc" + image: "timberio/vector:0.49.0-distroless-libc" imagePullPolicy: IfNotPresent args: - --config-dir diff --git a/distribution/kubernetes/vector-stateless-aggregator/service-headless.yaml b/distribution/kubernetes/vector-stateless-aggregator/service-headless.yaml index bc096d915d..39b4aee2d0 100644 --- a/distribution/kubernetes/vector-stateless-aggregator/service-headless.yaml +++ b/distribution/kubernetes/vector-stateless-aggregator/service-headless.yaml @@ -4,12 +4,12 @@ apiVersion: v1 kind: Service metadata: name: vector-headless - namespace: default + namespace: "default" labels: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Stateless-Aggregator - app.kubernetes.io/version: "0.44.0-distroless-libc" + app.kubernetes.io/version: "0.49.0-distroless-libc" annotations: spec: clusterIP: None diff --git a/distribution/kubernetes/vector-stateless-aggregator/service.yaml b/distribution/kubernetes/vector-stateless-aggregator/service.yaml index 07f710225e..b3ea61a5f1 100644 --- a/distribution/kubernetes/vector-stateless-aggregator/service.yaml +++ b/distribution/kubernetes/vector-stateless-aggregator/service.yaml @@ -4,12 +4,12 @@ apiVersion: v1 kind: Service metadata: name: vector - namespace: default + namespace: "default" labels: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Stateless-Aggregator - app.kubernetes.io/version: "0.44.0-distroless-libc" + app.kubernetes.io/version: "0.49.0-distroless-libc" annotations: spec: ports: diff --git a/distribution/kubernetes/vector-stateless-aggregator/serviceaccount.yaml b/distribution/kubernetes/vector-stateless-aggregator/serviceaccount.yaml index 641d2a58bd..9ff057a9ab 100644 --- a/distribution/kubernetes/vector-stateless-aggregator/serviceaccount.yaml +++ b/distribution/kubernetes/vector-stateless-aggregator/serviceaccount.yaml @@ -4,10 +4,10 @@ apiVersion: v1 kind: ServiceAccount metadata: name: vector - namespace: default + namespace: "default" labels: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Stateless-Aggregator - app.kubernetes.io/version: "0.44.0-distroless-libc" + app.kubernetes.io/version: "0.49.0-distroless-libc" automountServiceAccountToken: true diff --git a/distribution/systemd/vector.service b/distribution/systemd/vector.service index d9c6e78431..b78ff3131e 100644 --- a/distribution/systemd/vector.service +++ b/distribution/systemd/vector.service @@ -9,7 +9,7 @@ User=vector Group=vector ExecStartPre=/usr/bin/vector validate ExecStart=/usr/bin/vector -ExecReload=/usr/bin/vector validate +ExecReload=/usr/bin/vector validate --no-environment ExecReload=/bin/kill -HUP $MAINPID Restart=always AmbientCapabilities=CAP_NET_BIND_SERVICE diff --git a/docs/DEPRECATIONS.md b/docs/DEPRECATIONS.md index 3a5109eda2..fdc73468c7 100644 --- a/docs/DEPRECATIONS.md +++ b/docs/DEPRECATIONS.md @@ -14,6 +14,10 @@ For example: ## To be deprecated +- `v0.50.0` | `x86_64-apple-darwin` | macOS x86_64 (AKA x86 macOS, x86_64-apple-darwin or Intel macOS) will no longer be tested nor guaranteed to be able to build Vector. + +- `v0.49.0` | `block_interpolation` | Interpolating _multi-line_ configuration blocks using environment variables is now deprecated. + ## To be migrated ## To be removed diff --git a/docs/DEVELOPING.md b/docs/DEVELOPING.md index 7b9116a2a8..356f780e54 100644 --- a/docs/DEVELOPING.md +++ b/docs/DEVELOPING.md @@ -126,7 +126,7 @@ Loosely, you'll need the following: - **To run integration tests:** Have `docker` available, or a real live version of that service. (Use `AUTOSPAWN=false`) - **To run `make check-component-features`:** Have `remarshal` installed. - **To run `make check-licenses` or `cargo vdev build licenses`:** Have `dd-rust-license-tool` [installed](https://github.com/DataDog/rust-license-tool). -- **To run `cargo vdev build component-docs`:** Have `cue` [installed](https://cuelang.org/docs/install/). +- **To run `make generate-component-docs`:** Have `cue` [installed](https://cuelang.org/docs/install/). If you find yourself needing to run something inside the Docker environment described above, that's totally fine, they won't collide or hurt each other. In this case, you'd just run `make environment-generate`. @@ -161,7 +161,7 @@ cargo bench transforms::example make fmt cargo fmt # Build component documentation for the website -cargo vdev build component-docs +make generate-component-docs ``` If you run `make` you'll see a full list of all our tasks. Some of these will start Docker containers, sign commits, or even make releases. These are not common development commands and your mileage may vary. diff --git a/docs/REVIEWING.md b/docs/REVIEWING.md index 9075b74556..d40d311c63 100644 --- a/docs/REVIEWING.md +++ b/docs/REVIEWING.md @@ -26,7 +26,7 @@ should be used for all pull requests: - [ ] Is backward compatibility broken? If so, can it be avoided or deprecated? (see [Backward compatibility](#backward-compatibility)) - [ ] Have dependencies changed? (see [Dependencies](#dependencies)) - [ ] Has the code been explicitly reviewed for security issues? Dependencies included. (see [Security](#security)) -- [ ] Is there a risk of performance regressions? If so, have run the [Vector test harness](https://github.com/vectordotdev/vector-test-harness)? (see [Performance Testing](#performance-testing)) +- [ ] Is there a risk of performance regressions? (see [Performance Testing](#performance-testing)) - [ ] Should documentation be adjusted to reflect any of these changes? (see [Documentation](#documentation)) For component changes, especially pull requests introducing new components, the diff --git a/docs/tutorials/sinks/1_basic_sink.md b/docs/tutorials/sinks/1_basic_sink.md index 65bb7b4078..a099a55306 100644 --- a/docs/tutorials/sinks/1_basic_sink.md +++ b/docs/tutorials/sinks/1_basic_sink.md @@ -348,7 +348,7 @@ Our sink works! [event_streams_tracking]: https://github.com/vectordotdev/vector/issues/9261 [vdev_install]: https://github.com/vectordotdev/vector/tree/master/vdev#installation -[acknowledgements]: https://vector.dev/docs/about/under-the-hood/architecture/end-to-end-acknowledgements/ +[acknowledgements]: https://vector.dev/docs/architecture/end-to-end-acknowledgements/ [configurable_component]: https://rust-doc.vector.dev/vector_config/attr.configurable_component.html [generate_config]: https://rust-doc.vector.dev/vector/config/trait.generateconfig [sink_config]: https://rust-doc.vector.dev/vector/config/trait.sinkconfig diff --git a/docs/tutorials/sinks/2_http_sink.md b/docs/tutorials/sinks/2_http_sink.md index ca25791136..4c100fcb8e 100644 --- a/docs/tutorials/sinks/2_http_sink.md +++ b/docs/tutorials/sinks/2_http_sink.md @@ -150,7 +150,7 @@ generated by our `BasicEncoder`. *finalizers* - [`EventFinalizers`][event_finalizers] is a collection of `EventFinalizer`s. An [`EventFinalizer`][event_finalizer] is used to track the status of a given event and is used to support [end to end acknowledgements] -(https://vector.dev/docs/about/under-the-hood/guarantees/#acknowledgement- +(https://vector.dev/docs/architecture/guarantees/#acknowledgement- guarantees). *metadata* - the metadata contains additional data that is used to emit various metrics when diff --git a/lib/codecs/Cargo.toml b/lib/codecs/Cargo.toml index 992c162e89..28f5411ef3 100644 --- a/lib/codecs/Cargo.toml +++ b/lib/codecs/Cargo.toml @@ -2,7 +2,7 @@ name = "codecs" version = "0.1.0" authors = ["Vector Contributors "] -edition = "2021" +edition = "2024" publish = false [[bin]] @@ -11,29 +11,29 @@ path = "tests/bin/generate-avro-fixtures.rs" [dependencies] apache-avro = { version = "0.16.0", default-features = false } -bytes = { version = "1", default-features = false } +bytes.workspace = true chrono.workspace = true -csv-core = { version = "0.1.10", default-features = false } -derivative = { version = "2", default-features = false } +csv-core = { version = "0.1.12", default-features = false } +derivative.workspace = true dyn-clone = { version = "1", default-features = false } flate2.workspace = true influxdb-line-protocol = { version = "2", default-features = false } lookup = { package = "vector-lookup", path = "../vector-lookup", default-features = false, features = ["test"] } memchr = { version = "2", default-features = false } -ordered-float = { version = "4.6.0", default-features = false } +ordered-float.workspace = true prost.workspace = true prost-reflect.workspace = true rand.workspace = true -regex = { version = "1.11.1", default-features = false, features = ["std", "perf"] } +regex.workspace = true serde.workspace = true -serde_with = { version = "3.12.0", default-features = false, features = ["std", "macros", "chrono_0_4"] } +serde_with = { version = "3.14.0", default-features = false, features = ["std", "macros", "chrono_0_4"] } serde_json.workspace = true smallvec = { version = "1", default-features = false, features = ["union"] } snafu.workspace = true -syslog_loose = { version = "0.21", default-features = false, optional = true } +syslog_loose = { version = "0.23", default-features = false, optional = true } tokio-util = { version = "0.7", default-features = false, features = ["codec"] } -tokio.workspace = true -tracing = { version = "0.1", default-features = false } +tokio = { workspace = true, features = ["full"] } +tracing.workspace = true vrl.workspace = true vector-common = { path = "../vector-common", default-features = false } vector-config = { path = "../vector-config", default-features = false } @@ -42,11 +42,11 @@ vector-core = { path = "../vector-core", default-features = false, features = [" [dev-dependencies] futures.workspace = true -indoc = { version = "2", default-features = false } -tokio = { version = "1", features = ["test-util"] } -similar-asserts = "1.6.0" +indoc.workspace = true +tokio = { workspace = true, features = ["test-util"] } +similar-asserts = "1.7.0" vector-core = { path = "../vector-core", default-features = false, features = ["vrl", "test"] } -rstest = "0.24.0" +rstest = "0.26.1" tracing-test = "0.2.5" uuid.workspace = true vrl.workspace = true diff --git a/lib/codecs/src/decoding/format/avro.rs b/lib/codecs/src/decoding/format/avro.rs index 7a49763179..cc15c632bb 100644 --- a/lib/codecs/src/decoding/format/avro.rs +++ b/lib/codecs/src/decoding/format/avro.rs @@ -5,10 +5,10 @@ use bytes::Bytes; use chrono::Utc; use lookup::event_path; use serde::{Deserialize, Serialize}; -use smallvec::{smallvec, SmallVec}; +use smallvec::{SmallVec, smallvec}; use vector_config::configurable_component; use vector_core::{ - config::{log_schema, DataType, LogNamespace}, + config::{DataType, LogNamespace, log_schema}, event::{Event, LogEvent}, schema, }; @@ -41,7 +41,7 @@ impl AvroDeserializerConfig { /// Build the `AvroDeserializer` from this configuration. pub fn build(&self) -> AvroDeserializer { let schema = apache_avro::Schema::parse_str(&self.avro_options.schema) - .map_err(|error| format!("Failed building Avro serializer: {}", error)) + .map_err(|error| format!("Failed building Avro serializer: {error}")) .unwrap(); AvroDeserializer { schema, @@ -90,7 +90,7 @@ impl From<&AvroDeserializerOptions> for AvroSerializerOptions { #[derive(Clone, Debug)] pub struct AvroDeserializerOptions { /// The Avro schema definition. - /// Please note that the following [`apache_avro::types::Value`] variants are currently *not* supported: + /// **Note**: The following [`apache_avro::types::Value`] variants are *not* supported: /// * `Date` /// * `Decimal` /// * `Duration` @@ -103,7 +103,7 @@ pub struct AvroDeserializerOptions { ))] pub schema: String, - /// For Avro datum encoded in Kafka messages, the bytes are prefixed with the schema ID. Set this to true to strip the schema ID prefix. + /// For Avro datum encoded in Kafka messages, the bytes are prefixed with the schema ID. Set this to `true` to strip the schema ID prefix. /// According to [Confluent Kafka's document](https://docs.confluent.io/platform/current/schema-registry/fundamentals/serdes-develop/index.html#wire-format). pub strip_schema_id_prefix: bool, } diff --git a/lib/codecs/src/decoding/format/bytes.rs b/lib/codecs/src/decoding/format/bytes.rs index 06a97b67c2..bf8389de9f 100644 --- a/lib/codecs/src/decoding/format/bytes.rs +++ b/lib/codecs/src/decoding/format/bytes.rs @@ -1,11 +1,11 @@ use bytes::Bytes; use lookup::OwnedTargetPath; use serde::{Deserialize, Serialize}; -use smallvec::{smallvec, SmallVec}; +use smallvec::{SmallVec, smallvec}; use vector_core::config::LogNamespace; use vector_core::schema::meaning; use vector_core::{ - config::{log_schema, DataType}, + config::{DataType, log_schema}, event::{Event, LogEvent}, schema, }; diff --git a/lib/codecs/src/decoding/format/gelf.rs b/lib/codecs/src/decoding/format/gelf.rs index c37924072f..03cb82eaee 100644 --- a/lib/codecs/src/decoding/format/gelf.rs +++ b/lib/codecs/src/decoding/format/gelf.rs @@ -3,13 +3,13 @@ use chrono::{DateTime, Utc}; use derivative::Derivative; use lookup::{event_path, owned_value_path}; use serde::{Deserialize, Serialize}; -use serde_with::{serde_as, TimestampSecondsWithFrac}; -use smallvec::{smallvec, SmallVec}; +use serde_with::{TimestampSecondsWithFrac, serde_as}; +use smallvec::{SmallVec, smallvec}; use std::collections::HashMap; use vector_config::configurable_component; use vector_core::config::LogNamespace; use vector_core::{ - config::{log_schema, DataType}, + config::{DataType, log_schema}, event::Event, event::LogEvent, schema, @@ -17,9 +17,9 @@ use vector_core::{ use vrl::value::kind::Collection; use vrl::value::{Kind, Value}; -use super::{default_lossy, Deserializer}; +use super::{Deserializer, default_lossy}; use crate::gelf::GELF_TARGET_PATHS; -use crate::{gelf_fields::*, VALID_FIELD_REGEX}; +use crate::{VALID_FIELD_REGEX, gelf_fields::*}; // On GELF decoding behavior: // Graylog has a relaxed decoding. They are much more lenient than the spec would @@ -84,7 +84,7 @@ impl GelfDeserializerConfig { #[derive(Debug, Clone, PartialEq, Eq, Derivative)] #[derivative(Default)] pub struct GelfDeserializerOptions { - /// Determines whether or not to replace invalid UTF-8 sequences instead of failing. + /// Determines whether to replace invalid UTF-8 sequences instead of failing. /// /// When true, invalid UTF-8 sequences are replaced with the [`U+FFFD REPLACEMENT CHARACTER`][U+FFFD]. /// @@ -118,11 +118,9 @@ impl GelfDeserializer { // GELF spec defines the version as 1.1 which has not changed since 2013 if parsed.version != GELF_VERSION { - return Err(format!( - "{} does not match GELF spec version ({})", - VERSION, GELF_VERSION - ) - .into()); + return Err( + format!("{VERSION} does not match GELF spec version ({GELF_VERSION})").into(), + ); } log.insert(&GELF_TARGET_PATHS.version, parsed.version.to_string()); @@ -166,16 +164,18 @@ impl GelfDeserializer { // per GELF spec, Additional field names must be prefixed with an underscore if !key.starts_with('_') { return Err(format!( - "'{}' field is invalid. \ - Additional field names must be prefixed with an underscore.", - key + "'{key}' field is invalid. \ + Additional field names must be prefixed with an underscore." ) .into()); } // per GELF spec, Additional field names must be characters dashes or dots if !VALID_FIELD_REGEX.is_match(key) { - return Err(format!("'{}' field contains invalid characters. Field names may \ - contain only letters, numbers, underscores, dashes and dots.", key).into()); + return Err(format!( + "'{key}' field contains invalid characters. Field names may \ + contain only letters, numbers, underscores, dashes and dots." + ) + .into()); } // per GELF spec, Additional field values must be either strings or numbers @@ -191,8 +191,8 @@ impl GelfDeserializer { serde_json::Value::Array(_) => "array", serde_json::Value::Object(_) => "object", }; - return Err(format!("The value type for field {} is an invalid type ({}). Additional field values \ - should be either strings or numbers.", key, type_).into()); + return Err(format!("The value type for field {key} is an invalid type ({type_}). Additional field values \ + should be either strings or numbers.").into()); } } } diff --git a/lib/codecs/src/decoding/format/influxdb.rs b/lib/codecs/src/decoding/format/influxdb.rs index a4cb5cbfe7..622f032d9a 100644 --- a/lib/codecs/src/decoding/format/influxdb.rs +++ b/lib/codecs/src/decoding/format/influxdb.rs @@ -9,8 +9,8 @@ use vector_config::configurable_component; use vector_core::config::LogNamespace; use vector_core::event::{Event, Metric, MetricKind, MetricTags, MetricValue}; use vector_core::{config::DataType, schema}; -use vrl::value::kind::Collection; use vrl::value::Kind; +use vrl::value::kind::Collection; use crate::decoding::format::default_lossy; @@ -56,7 +56,7 @@ impl InfluxdbDeserializerConfig { #[derive(Debug, Clone, PartialEq, Eq, Derivative)] #[derivative(Default)] pub struct InfluxdbDeserializerOptions { - /// Determines whether or not to replace invalid UTF-8 sequences instead of failing. + /// Determines whether to replace invalid UTF-8 sequences instead of failing. /// /// When true, invalid UTF-8 sequences are replaced with the [`U+FFFD REPLACEMENT CHARACTER`][U+FFFD]. /// diff --git a/lib/codecs/src/decoding/format/json.rs b/lib/codecs/src/decoding/format/json.rs index a017cba3c8..a2f9b0c450 100644 --- a/lib/codecs/src/decoding/format/json.rs +++ b/lib/codecs/src/decoding/format/json.rs @@ -1,16 +1,16 @@ use bytes::Bytes; use chrono::Utc; use derivative::Derivative; -use smallvec::{smallvec, SmallVec}; +use smallvec::{SmallVec, smallvec}; use vector_config::configurable_component; use vector_core::{ - config::{log_schema, DataType, LogNamespace}, + config::{DataType, LogNamespace, log_schema}, event::Event, schema, }; use vrl::value::Kind; -use super::{default_lossy, Deserializer}; +use super::{Deserializer, default_lossy}; /// Config used to build a `JsonDeserializer`. #[configurable_component] @@ -67,7 +67,7 @@ impl JsonDeserializerConfig { #[derive(Debug, Clone, PartialEq, Eq, Derivative)] #[derivative(Default)] pub struct JsonDeserializerOptions { - /// Determines whether or not to replace invalid UTF-8 sequences instead of failing. + /// Determines whether to replace invalid UTF-8 sequences instead of failing. /// /// When true, invalid UTF-8 sequences are replaced with the [`U+FFFD REPLACEMENT CHARACTER`][U+FFFD]. /// @@ -111,7 +111,7 @@ impl Deserializer for JsonDeserializer { true => serde_json::from_str(&String::from_utf8_lossy(&bytes)), false => serde_json::from_slice(&bytes), } - .map_err(|error| format!("Error parsing JSON: {:?}", error))?; + .map_err(|error| format!("Error parsing JSON: {error:?}"))?; // If the root is an Array, split it into multiple events let mut events = match json { diff --git a/lib/codecs/src/decoding/format/mod.rs b/lib/codecs/src/decoding/format/mod.rs index 9e2dee7de1..783aa5d79d 100644 --- a/lib/codecs/src/decoding/format/mod.rs +++ b/lib/codecs/src/decoding/format/mod.rs @@ -44,11 +44,18 @@ pub trait Deserializer: DynClone + Send + Sync { /// frame can potentially hold multiple events, e.g. when parsing a JSON /// array. However, we optimize the most common case of emitting one event /// by not requiring heap allocations for it. + /// + /// **Note**: The type of the produced events depends on the implementation. fn parse( &self, bytes: Bytes, log_namespace: LogNamespace, ) -> vector_common::Result>; + + /// Parses trace events from bytes. + fn parse_traces(&self, _bytes: Bytes) -> vector_common::Result> { + unimplemented!() + } } dyn_clone::clone_trait_object!(Deserializer); diff --git a/lib/codecs/src/decoding/format/native.rs b/lib/codecs/src/decoding/format/native.rs index f94683f14d..264a5dc4e5 100644 --- a/lib/codecs/src/decoding/format/native.rs +++ b/lib/codecs/src/decoding/format/native.rs @@ -1,11 +1,11 @@ use bytes::Bytes; use prost::Message; use serde::{Deserialize, Serialize}; -use smallvec::{smallvec, SmallVec}; +use smallvec::{SmallVec, smallvec}; use vector_core::config::LogNamespace; use vector_core::{ config::DataType, - event::{proto, Event, EventArray, EventContainer}, + event::{Event, EventArray, EventContainer, proto}, schema, }; use vrl::value::Kind; diff --git a/lib/codecs/src/decoding/format/native_json.rs b/lib/codecs/src/decoding/format/native_json.rs index eac15b3052..1512dbabb5 100644 --- a/lib/codecs/src/decoding/format/native_json.rs +++ b/lib/codecs/src/decoding/format/native_json.rs @@ -1,12 +1,12 @@ use bytes::Bytes; use derivative::Derivative; -use smallvec::{smallvec, SmallVec}; +use smallvec::{SmallVec, smallvec}; use vector_config::configurable_component; use vector_core::{config::DataType, event::Event, schema}; -use vrl::value::kind::Collection; use vrl::value::Kind; +use vrl::value::kind::Collection; -use super::{default_lossy, Deserializer}; +use super::{Deserializer, default_lossy}; use vector_core::config::LogNamespace; /// Config used to build a `NativeJsonDeserializer`. @@ -57,7 +57,7 @@ impl NativeJsonDeserializerConfig { #[derive(Debug, Clone, PartialEq, Eq, Derivative)] #[derivative(Default)] pub struct NativeJsonDeserializerOptions { - /// Determines whether or not to replace invalid UTF-8 sequences instead of failing. + /// Determines whether to replace invalid UTF-8 sequences instead of failing. /// /// When true, invalid UTF-8 sequences are replaced with the [`U+FFFD REPLACEMENT CHARACTER`][U+FFFD]. /// @@ -97,7 +97,7 @@ impl Deserializer for NativeJsonDeserializer { true => serde_json::from_str(&String::from_utf8_lossy(&bytes)), false => serde_json::from_slice(&bytes), } - .map_err(|error| format!("Error parsing JSON: {:?}", error))?; + .map_err(|error| format!("Error parsing JSON: {error:?}"))?; let events = match json { serde_json::Value::Array(values) => values diff --git a/lib/codecs/src/decoding/format/protobuf.rs b/lib/codecs/src/decoding/format/protobuf.rs index 42f8665891..beece2053a 100644 --- a/lib/codecs/src/decoding/format/protobuf.rs +++ b/lib/codecs/src/decoding/format/protobuf.rs @@ -4,15 +4,17 @@ use bytes::Bytes; use chrono::Utc; use derivative::Derivative; use prost_reflect::{DynamicMessage, MessageDescriptor}; -use smallvec::{smallvec, SmallVec}; +use smallvec::{SmallVec, smallvec}; use vector_config::configurable_component; -use vector_core::event::LogEvent; +use vector_core::event::{LogEvent, TraceEvent}; use vector_core::{ - config::{log_schema, DataType, LogNamespace}, + config::{DataType, LogNamespace, log_schema}, event::Event, schema, }; -use vrl::value::Kind; +use vrl::protobuf::descriptor::{get_message_descriptor, get_message_descriptor_from_bytes}; +use vrl::protobuf::parse::{Options, proto_to_value}; +use vrl::value::{Kind, Value}; use super::Deserializer; @@ -66,10 +68,15 @@ impl ProtobufDeserializerConfig { #[derive(Debug, Clone, PartialEq, Eq, Derivative)] #[derivative(Default)] pub struct ProtobufDeserializerOptions { - /// Path to desc file + /// The path to the protobuf descriptor set file. + /// + /// This file is the output of `protoc -I -o ` + /// + /// You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). pub desc_file: PathBuf, - /// message type. e.g package.message + /// The name of the message type to use for serializing. + #[configurable(metadata(docs::examples = "package.Message"))] pub message_type: String, } @@ -77,27 +84,56 @@ pub struct ProtobufDeserializerOptions { #[derive(Debug, Clone)] pub struct ProtobufDeserializer { message_descriptor: MessageDescriptor, + options: Options, } impl ProtobufDeserializer { /// Creates a new `ProtobufDeserializer`. pub fn new(message_descriptor: MessageDescriptor) -> Self { - Self { message_descriptor } + Self { + message_descriptor, + options: Default::default(), + } + } + + /// Creates a new deserializer instance using the descriptor bytes directly. + pub fn new_from_bytes( + desc_bytes: &[u8], + message_type: &str, + options: Options, + ) -> vector_common::Result { + let message_descriptor = get_message_descriptor_from_bytes(desc_bytes, message_type)?; + Ok(Self { + message_descriptor, + options, + }) } } +fn extract_vrl_value( + bytes: Bytes, + message_descriptor: &MessageDescriptor, + options: &Options, +) -> vector_common::Result { + let dynamic_message = DynamicMessage::decode(message_descriptor.clone(), bytes) + .map_err(|error| format!("Error parsing protobuf: {error:?}"))?; + + Ok(proto_to_value( + &prost_reflect::Value::Message(dynamic_message), + None, + options, + )?) +} + impl Deserializer for ProtobufDeserializer { fn parse( &self, bytes: Bytes, log_namespace: LogNamespace, ) -> vector_common::Result> { - let dynamic_message = DynamicMessage::decode(self.message_descriptor.clone(), bytes) - .map_err(|error| format!("Error parsing protobuf: {:?}", error))?; + let vrl_value = extract_vrl_value(bytes, &self.message_descriptor, &self.options)?; + let mut event = Event::Log(LogEvent::from(vrl_value)); - let proto_vrl = - vrl::protobuf::proto_to_value(&prost_reflect::Value::Message(dynamic_message), None)?; - let mut event = Event::Log(LogEvent::from(proto_vrl)); let event = match log_namespace { LogNamespace::Vector => event, LogNamespace::Legacy => { @@ -114,15 +150,19 @@ impl Deserializer for ProtobufDeserializer { Ok(smallvec![event]) } + + fn parse_traces(&self, bytes: Bytes) -> vector_common::Result> { + let vrl_value = extract_vrl_value(bytes, &self.message_descriptor, &self.options)?; + let trace_event = Event::Trace(TraceEvent::from(vrl_value)); + Ok(smallvec![trace_event]) + } } impl TryFrom<&ProtobufDeserializerConfig> for ProtobufDeserializer { type Error = vector_common::Error; fn try_from(config: &ProtobufDeserializerConfig) -> vector_common::Result { - let message_descriptor = vrl::protobuf::get_message_descriptor( - &config.protobuf.desc_file, - &config.protobuf.message_type, - )?; + let message_descriptor = + get_message_descriptor(&config.protobuf.desc_file, &config.protobuf.message_type)?; Ok(Self::new(message_descriptor)) } } @@ -148,8 +188,7 @@ mod tests { validate_log: fn(&LogEvent), ) { let input = Bytes::from(protobuf_bin_message); - let message_descriptor = - vrl::protobuf::get_message_descriptor(&protobuf_desc_path, message_type).unwrap(); + let message_descriptor = get_message_descriptor(&protobuf_desc_path, message_type).unwrap(); let deserializer = ProtobufDeserializer::new(message_descriptor); for namespace in [LogNamespace::Legacy, LogNamespace::Vector] { @@ -245,7 +284,7 @@ mod tests { #[test] fn deserialize_error_invalid_protobuf() { let input = Bytes::from("{ foo"); - let message_descriptor = vrl::protobuf::get_message_descriptor( + let message_descriptor = get_message_descriptor( &test_data_dir().join("protos/test_protobuf.desc"), "test_protobuf.Person", ) diff --git a/lib/codecs/src/decoding/format/syslog.rs b/lib/codecs/src/decoding/format/syslog.rs index 6a5d7e135e..524ba432ca 100644 --- a/lib/codecs/src/decoding/format/syslog.rs +++ b/lib/codecs/src/decoding/format/syslog.rs @@ -1,20 +1,20 @@ use bytes::Bytes; use chrono::{DateTime, Datelike, Utc}; use derivative::Derivative; -use lookup::{event_path, owned_value_path, OwnedTargetPath, OwnedValuePath}; -use smallvec::{smallvec, SmallVec}; +use lookup::{OwnedTargetPath, OwnedValuePath, event_path, owned_value_path}; +use smallvec::{SmallVec, smallvec}; use std::borrow::Cow; use syslog_loose::{IncompleteDate, Message, ProcId, Protocol, Variant}; use vector_config::configurable_component; use vector_core::config::{LegacyKey, LogNamespace}; use vector_core::{ - config::{log_schema, DataType}, + config::{DataType, log_schema}, event::{Event, LogEvent, ObjectMap, Value}, schema, }; -use vrl::value::{kind::Collection, Kind}; +use vrl::value::{Kind, kind::Collection}; -use super::{default_lossy, Deserializer}; +use super::{Deserializer, default_lossy}; /// Config used to build a `SyslogDeserializer`. #[configurable_component] @@ -241,7 +241,7 @@ impl SyslogDeserializerConfig { #[derive(Debug, Clone, PartialEq, Eq, Derivative)] #[derivative(Default)] pub struct SyslogDeserializerOptions { - /// Determines whether or not to replace invalid UTF-8 sequences instead of failing. + /// Determines whether to replace invalid UTF-8 sequences instead of failing. /// /// When true, invalid UTF-8 sequences are replaced with the [`U+FFFD REPLACEMENT CHARACTER`][U+FFFD]. /// @@ -481,7 +481,7 @@ fn insert_fields_from_syslog( #[cfg(test)] mod tests { use super::*; - use vector_core::config::{init_log_schema, log_schema, LogSchema}; + use vector_core::config::{LogSchema, init_log_schema, log_schema}; #[test] fn deserialize_syslog_legacy_namespace() { diff --git a/lib/codecs/src/decoding/format/vrl.rs b/lib/codecs/src/decoding/format/vrl.rs index 63a127955e..4fdd82cbf3 100644 --- a/lib/codecs/src/decoding/format/vrl.rs +++ b/lib/codecs/src/decoding/format/vrl.rs @@ -1,14 +1,14 @@ -use crate::decoding::format::Deserializer; use crate::BytesDeserializerConfig; +use crate::decoding::format::Deserializer; use bytes::Bytes; use derivative::Derivative; -use smallvec::{smallvec, SmallVec}; +use smallvec::{SmallVec, smallvec}; use vector_config_macros::configurable_component; use vector_core::config::{DataType, LogNamespace}; use vector_core::event::{Event, TargetEvents, VrlTarget}; use vector_core::{compile_vrl, schema}; use vrl::compiler::state::ExternalEnv; -use vrl::compiler::{runtime::Runtime, CompileConfig, Program, TimeZone, TypeState}; +use vrl::compiler::{CompileConfig, Program, TimeZone, TypeState, runtime::Runtime}; use vrl::diagnostic::Formatter; use vrl::value::Kind; @@ -37,7 +37,7 @@ pub struct VrlDeserializerOptions { /// time zone. The time zone name may be any name in the [TZ database][tz_database], or `local` /// to indicate system local time. /// - /// If not set, `local` will be used. + /// If not set, `local` is used. /// /// [tz_database]: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones #[serde(default)] @@ -257,7 +257,9 @@ mod tests { ); // CEF input - let cef_bytes = Bytes::from("CEF:0|Security|Threat Manager|1.0|100|worm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2 spt=1232"); + let cef_bytes = Bytes::from( + "CEF:0|Security|Threat Manager|1.0|100|worm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2 spt=1232", + ); let result = decoder.parse(cef_bytes, LogNamespace::Vector).unwrap(); assert_eq!(result.len(), 1); let cef_event = result.first().unwrap(); diff --git a/lib/codecs/src/decoding/framing/chunked_gelf.rs b/lib/codecs/src/decoding/framing/chunked_gelf.rs index bbddc457a3..a8a53c2178 100644 --- a/lib/codecs/src/decoding/framing/chunked_gelf.rs +++ b/lib/codecs/src/decoding/framing/chunked_gelf.rs @@ -3,7 +3,7 @@ use crate::{BytesDecoder, StreamDecodingError}; use bytes::{Buf, Bytes, BytesMut}; use derivative::Derivative; use flate2::read::{MultiGzDecoder, ZlibDecoder}; -use snafu::{ensure, ResultExt, Snafu}; +use snafu::{ResultExt, Snafu, ensure}; use std::any::Any; use std::collections::HashMap; use std::io::Read; @@ -67,7 +67,7 @@ pub struct ChunkedGelfDecoderOptions { /// be dropped. If this option is not set, the decoder does not limit the length of messages and /// the per-message memory is unbounded. /// - /// Note that a message can be composed of multiple chunks and this limit is applied to the whole + /// **Note**: A message can be composed of multiple chunks and this limit is applied to the whole /// message, not to individual chunks. /// /// This limit takes only into account the message's payload and the GELF header bytes are excluded from the calculation. @@ -178,11 +178,11 @@ impl ChunkedGelfDecompression { if data.starts_with(ZLIB_MAGIC) { // Based on https://datatracker.ietf.org/doc/html/rfc1950#section-2.2 - if let Some([first_byte, second_byte]) = data.get(0..2) { - if (*first_byte as u16 * 256 + *second_byte as u16) % 31 == 0 { - trace!("Detected Zlib compression"); - return Self::Zlib; - } + if let Some([first_byte, second_byte]) = data.get(0..2) + && (*first_byte as u16 * 256 + *second_byte as u16) % 31 == 0 + { + trace!("Detected Zlib compression"); + return Self::Zlib; }; warn!( @@ -231,32 +231,42 @@ pub enum ChunkedGelfDecompressionError { pub enum ChunkedGelfDecoderError { #[snafu(display("Invalid chunk header with less than 10 bytes: 0x{header:0x}"))] InvalidChunkHeader { header: Bytes }, - #[snafu(display("Received chunk with message id {message_id} and sequence number {sequence_number} has an invalid total chunks value of {total_chunks}. It must be between 1 and {GELF_MAX_TOTAL_CHUNKS}."))] + #[snafu(display( + "Received chunk with message id {message_id} and sequence number {sequence_number} has an invalid total chunks value of {total_chunks}. It must be between 1 and {GELF_MAX_TOTAL_CHUNKS}." + ))] InvalidTotalChunks { message_id: u64, sequence_number: u8, total_chunks: u8, }, - #[snafu(display("Received chunk with message id {message_id} and sequence number {sequence_number} has a sequence number greater than its total chunks value of {total_chunks}"))] + #[snafu(display( + "Received chunk with message id {message_id} and sequence number {sequence_number} has a sequence number greater than its total chunks value of {total_chunks}" + ))] InvalidSequenceNumber { message_id: u64, sequence_number: u8, total_chunks: u8, }, - #[snafu(display("Pending messages limit of {pending_messages_limit} reached while processing chunk with message id {message_id} and sequence number {sequence_number}"))] + #[snafu(display( + "Pending messages limit of {pending_messages_limit} reached while processing chunk with message id {message_id} and sequence number {sequence_number}" + ))] PendingMessagesLimitReached { message_id: u64, sequence_number: u8, pending_messages_limit: usize, }, - #[snafu(display("Received chunk with message id {message_id} and sequence number {sequence_number} has different total chunks values: original total chunks value is {original_total_chunks} and received total chunks value is {received_total_chunks}"))] + #[snafu(display( + "Received chunk with message id {message_id} and sequence number {sequence_number} has different total chunks values: original total chunks value is {original_total_chunks} and received total chunks value is {received_total_chunks}" + ))] TotalChunksMismatch { message_id: u64, sequence_number: u8, original_total_chunks: u8, received_total_chunks: u8, }, - #[snafu(display("Message with id {message_id} has exceeded the maximum message length and it will be dropped: got {length} bytes and max message length is {max_length} bytes. Discarding all buffered chunks of that message"))] + #[snafu(display( + "Message with id {message_id} has exceeded the maximum message length and it will be dropped: got {length} bytes and max message length is {max_length} bytes. Discarding all buffered chunks of that message" + ))] MaxLengthExceed { message_id: u64, sequence_number: u8, @@ -511,11 +521,10 @@ impl Decoder for ChunkedGelfDecoder { #[cfg(test)] mod tests { - use super::*; use bytes::{BufMut, BytesMut}; use flate2::{write::GzEncoder, write::ZlibEncoder}; - use rand::{rngs::SmallRng, seq::SliceRandom, SeedableRng}; + use rand::{SeedableRng, rngs::SmallRng, seq::SliceRandom}; use rstest::{fixture, rstest}; use std::fmt::Write as FmtWrite; use std::io::Write as IoWrite; @@ -734,7 +743,7 @@ mod tests { #[tokio::test] async fn decode_shuffled_messages() { - let mut rng = SmallRng::seed_from_u64(420); + let mut rng = SmallRng::seed_from_u64(42); let total_chunks = 100u8; let first_message_id = 1u64; let first_payload = "first payload"; @@ -1065,7 +1074,7 @@ mod tests { async fn decode_malformed_zlib_message() { let mut compressed_payload = BytesMut::new(); compressed_payload.extend(ZLIB_MAGIC); - compressed_payload.extend(&[0x9c, 0x12, 0x34, 0x56]); + compressed_payload.extend(&[0x9c, 0x12, 0x00, 0xFF]); let mut decoder = ChunkedGelfDecoder::default(); let error = decoder diff --git a/lib/codecs/src/decoding/framing/mod.rs b/lib/codecs/src/decoding/framing/mod.rs index 86740a5012..034f061db0 100644 --- a/lib/codecs/src/decoding/framing/mod.rs +++ b/lib/codecs/src/decoding/framing/mod.rs @@ -9,6 +9,7 @@ mod chunked_gelf; mod length_delimited; mod newline_delimited; mod octet_counting; +mod varint_length_delimited; use std::{any::Any, fmt::Debug}; @@ -26,6 +27,9 @@ pub use octet_counting::{ OctetCountingDecoder, OctetCountingDecoderConfig, OctetCountingDecoderOptions, }; use tokio_util::codec::LinesCodecError; +pub use varint_length_delimited::{ + VarintLengthDelimitedDecoder, VarintLengthDelimitedDecoderConfig, +}; pub use self::bytes::{BytesDecoder, BytesDecoderConfig}; use super::StreamDecodingError; diff --git a/lib/codecs/src/decoding/framing/octet_counting.rs b/lib/codecs/src/decoding/framing/octet_counting.rs index b5248b26b6..749d66b8ff 100644 --- a/lib/codecs/src/decoding/framing/octet_counting.rs +++ b/lib/codecs/src/decoding/framing/octet_counting.rs @@ -97,8 +97,7 @@ impl OctetCountingDecoder { // There are enough chars in this frame to discard src.advance(chars); self.octet_decoding = None; - Err(LinesCodecError::Io(io::Error::new( - io::ErrorKind::Other, + Err(LinesCodecError::Io(io::Error::other( "Frame length limit exceeded", ))) } @@ -117,8 +116,7 @@ impl OctetCountingDecoder { // When discarding we keep discarding to the next newline. src.advance(offset + 1); self.octet_decoding = None; - Err(LinesCodecError::Io(io::Error::new( - io::ErrorKind::Other, + Err(LinesCodecError::Io(io::Error::other( "Frame length limit exceeded", ))) } @@ -203,8 +201,7 @@ impl OctetCountingDecoder { (State::NotDiscarding, Some(newline_pos), _) => { // Beyond maximum length, advance to the newline. src.advance(newline_pos + 1); - Err(LinesCodecError::Io(io::Error::new( - io::ErrorKind::Other, + Err(LinesCodecError::Io(io::Error::other( "Frame length limit exceeded", ))) } @@ -235,13 +232,13 @@ impl OctetCountingDecoder { &mut self, src: &mut BytesMut, ) -> Option, LinesCodecError>> { - if let Some(&first_byte) = src.first() { - if (49..=57).contains(&first_byte) { - // First character is non zero number so we can assume that - // octet count framing is used. - trace!("Octet counting encoded event detected."); - self.octet_decoding = Some(State::NotDiscarding); - } + if let Some(&first_byte) = src.first() + && (49..=57).contains(&first_byte) + { + // First character is non zero number so we can assume that + // octet count framing is used. + trace!("Octet counting encoded event detected."); + self.octet_decoding = Some(State::NotDiscarding); } self.octet_decoding @@ -392,7 +389,7 @@ mod tests { buffer.put(&b"defghijklmnopqrstuvwxyzand here we are"[..]); let result = decoder.decode(&mut buffer); - println!("{:?}", result); + println!("{result:?}"); assert!(result.is_err()); assert_eq!(b"and here we are"[..], buffer); } diff --git a/lib/codecs/src/decoding/framing/varint_length_delimited.rs b/lib/codecs/src/decoding/framing/varint_length_delimited.rs new file mode 100644 index 0000000000..7b9d26f0a0 --- /dev/null +++ b/lib/codecs/src/decoding/framing/varint_length_delimited.rs @@ -0,0 +1,229 @@ +use bytes::{Buf, Bytes, BytesMut}; +use derivative::Derivative; +use snafu::Snafu; +use tokio_util::codec::Decoder; +use vector_config::configurable_component; + +use super::{BoxedFramingError, FramingError, StreamDecodingError}; + +/// Errors that can occur during varint length delimited framing. +#[derive(Debug, Snafu)] +pub enum VarintFramingError { + #[snafu(display("Varint too large"))] + VarintOverflow, + + #[snafu(display("Frame too large: {length} bytes (max: {max})"))] + FrameTooLarge { length: usize, max: usize }, + + #[snafu(display("Trailing data at EOF"))] + TrailingData, +} + +impl StreamDecodingError for VarintFramingError { + fn can_continue(&self) -> bool { + match self { + // Varint overflow and frame too large are not recoverable + Self::VarintOverflow | Self::FrameTooLarge { .. } => false, + // Trailing data at EOF is not recoverable + Self::TrailingData => false, + } + } +} + +impl FramingError for VarintFramingError { + fn as_any(&self) -> &dyn std::any::Any { + self as &dyn std::any::Any + } +} + +/// Config used to build a `VarintLengthDelimitedDecoder`. +#[configurable_component] +#[derive(Debug, Clone, Derivative)] +#[derivative(Default)] +pub struct VarintLengthDelimitedDecoderConfig { + /// Maximum frame length + #[serde(default = "default_max_frame_length")] + pub max_frame_length: usize, +} + +const fn default_max_frame_length() -> usize { + 8 * 1_024 * 1_024 +} + +impl VarintLengthDelimitedDecoderConfig { + /// Build the `VarintLengthDelimitedDecoder` from this configuration. + pub fn build(&self) -> VarintLengthDelimitedDecoder { + VarintLengthDelimitedDecoder::new(self.max_frame_length) + } +} + +/// A codec for handling bytes sequences whose length is encoded as a varint prefix. +/// This is compatible with protobuf's length-delimited encoding. +#[derive(Debug, Clone)] +pub struct VarintLengthDelimitedDecoder { + max_frame_length: usize, +} + +impl VarintLengthDelimitedDecoder { + /// Creates a new `VarintLengthDelimitedDecoder`. + pub fn new(max_frame_length: usize) -> Self { + Self { max_frame_length } + } + + /// Decode a varint from the buffer + fn decode_varint(&self, buf: &mut BytesMut) -> Result, BoxedFramingError> { + if buf.is_empty() { + return Ok(None); + } + + let mut value: u64 = 0; + let mut shift: u8 = 0; + let mut bytes_read = 0; + + for byte in buf.iter() { + bytes_read += 1; + let byte_value = (*byte & 0x7F) as u64; + value |= byte_value << shift; + + if *byte & 0x80 == 0 { + // Last byte of varint + buf.advance(bytes_read); + return Ok(Some(value)); + } + + shift += 7; + if shift >= 64 { + return Err(VarintFramingError::VarintOverflow.into()); + } + } + + // Incomplete varint + Ok(None) + } +} + +impl Default for VarintLengthDelimitedDecoder { + fn default() -> Self { + Self::new(default_max_frame_length()) + } +} + +impl Decoder for VarintLengthDelimitedDecoder { + type Item = Bytes; + type Error = BoxedFramingError; + + fn decode(&mut self, src: &mut BytesMut) -> Result, Self::Error> { + // First, try to decode the varint length + let length = match self.decode_varint(src)? { + Some(len) => len as usize, + None => return Ok(None), // Incomplete varint + }; + + // Check if the length is reasonable + if length > self.max_frame_length { + return Err(VarintFramingError::FrameTooLarge { + length, + max: self.max_frame_length, + } + .into()); + } + + // Check if we have enough data for the complete frame + if src.len() < length { + return Ok(None); // Incomplete frame + } + + // Extract the frame + let frame = src.split_to(length).freeze(); + Ok(Some(frame)) + } + + fn decode_eof(&mut self, src: &mut BytesMut) -> Result, Self::Error> { + if src.is_empty() { + Ok(None) + } else { + // Try to decode what we have, even if incomplete + match self.decode(src)? { + Some(frame) => Ok(Some(frame)), + None => { + // If we have data but couldn't decode it, it's trailing data + if !src.is_empty() { + Err(VarintFramingError::TrailingData.into()) + } else { + Ok(None) + } + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn decode_single_byte_varint() { + let mut input = BytesMut::from(&[0x03, b'f', b'o', b'o'][..]); + let mut decoder = VarintLengthDelimitedDecoder::default(); + + assert_eq!( + decoder.decode(&mut input).unwrap().unwrap(), + Bytes::from("foo") + ); + assert_eq!(decoder.decode(&mut input).unwrap(), None); + } + + #[test] + fn decode_multi_byte_varint() { + // 300 in varint encoding: 0xAC 0x02 + let mut input = BytesMut::from(&[0xAC, 0x02][..]); + // Add 300 bytes of data + input.extend_from_slice(&vec![b'x'; 300]); + let mut decoder = VarintLengthDelimitedDecoder::default(); + + let result = decoder.decode(&mut input).unwrap().unwrap(); + assert_eq!(result.len(), 300); + assert_eq!(decoder.decode(&mut input).unwrap(), None); + } + + #[test] + fn decode_incomplete_varint() { + let mut input = BytesMut::from(&[0x80][..]); // Incomplete varint + let mut decoder = VarintLengthDelimitedDecoder::default(); + + assert_eq!(decoder.decode(&mut input).unwrap(), None); + } + + #[test] + fn decode_incomplete_frame() { + let mut input = BytesMut::from(&[0x05, b'f', b'o'][..]); // Length 5, but only 2 bytes + let mut decoder = VarintLengthDelimitedDecoder::default(); + + assert_eq!(decoder.decode(&mut input).unwrap(), None); + } + + #[test] + fn decode_frame_too_large() { + let mut input = + BytesMut::from(&[0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01][..]); + let mut decoder = VarintLengthDelimitedDecoder::new(1000); + + assert!(decoder.decode(&mut input).is_err()); + } + + #[test] + fn decode_trailing_data_at_eof() { + let mut input = BytesMut::from(&[0x03, b'f', b'o', b'o', b'e', b'x', b't', b'r', b'a'][..]); + let mut decoder = VarintLengthDelimitedDecoder::default(); + + // First decode should succeed + assert_eq!( + decoder.decode(&mut input).unwrap().unwrap(), + Bytes::from("foo") + ); + + // Second decode should fail with trailing data + assert!(decoder.decode_eof(&mut input).is_err()); + } +} diff --git a/lib/codecs/src/decoding/mod.rs b/lib/codecs/src/decoding/mod.rs index 0e14722dd3..6b762a14f2 100644 --- a/lib/codecs/src/decoding/mod.rs +++ b/lib/codecs/src/decoding/mod.rs @@ -24,7 +24,7 @@ pub use framing::{ ChunkedGelfDecoderConfig, ChunkedGelfDecoderOptions, FramingError, LengthDelimitedDecoder, LengthDelimitedDecoderConfig, NewlineDelimitedDecoder, NewlineDelimitedDecoderConfig, NewlineDelimitedDecoderOptions, OctetCountingDecoder, OctetCountingDecoderConfig, - OctetCountingDecoderOptions, + OctetCountingDecoderOptions, VarintLengthDelimitedDecoder, VarintLengthDelimitedDecoderConfig, }; use smallvec::SmallVec; use std::fmt::Debug; @@ -51,8 +51,8 @@ pub enum Error { impl std::fmt::Display for Error { fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - Self::FramingError(error) => write!(formatter, "FramingError({})", error), - Self::ParsingError(error) => write!(formatter, "ParsingError({})", error), + Self::FramingError(error) => write!(formatter, "FramingError({error})"), + Self::ParsingError(error) => write!(formatter, "ParsingError({error})"), } } } @@ -105,6 +105,10 @@ pub enum FramingConfig { /// /// [chunked_gelf]: https://go2docs.graylog.org/current/getting_in_log_data/gelf.html ChunkedGelf(ChunkedGelfDecoderConfig), + + /// Byte frames which are prefixed by a varint indicating the length. + /// This is compatible with protobuf's length-delimited encoding. + VarintLengthDelimited(VarintLengthDelimitedDecoderConfig), } impl From for FramingConfig { @@ -143,6 +147,12 @@ impl From for FramingConfig { } } +impl From for FramingConfig { + fn from(config: VarintLengthDelimitedDecoderConfig) -> Self { + Self::VarintLengthDelimited(config) + } +} + impl FramingConfig { /// Build the `Framer` from this configuration. pub fn build(&self) -> Framer { @@ -153,6 +163,9 @@ impl FramingConfig { FramingConfig::NewlineDelimited(config) => Framer::NewlineDelimited(config.build()), FramingConfig::OctetCounting(config) => Framer::OctetCounting(config.build()), FramingConfig::ChunkedGelf(config) => Framer::ChunkedGelf(config.build()), + FramingConfig::VarintLengthDelimited(config) => { + Framer::VarintLengthDelimited(config.build()) + } } } } @@ -174,6 +187,8 @@ pub enum Framer { Boxed(BoxedFramer), /// Uses a `ChunkedGelfDecoder` for framing. ChunkedGelf(ChunkedGelfDecoder), + /// Uses a `VarintLengthDelimitedDecoder` for framing. + VarintLengthDelimited(VarintLengthDelimitedDecoder), } impl tokio_util::codec::Decoder for Framer { @@ -189,6 +204,7 @@ impl tokio_util::codec::Decoder for Framer { Framer::OctetCounting(framer) => framer.decode(src), Framer::Boxed(framer) => framer.decode(src), Framer::ChunkedGelf(framer) => framer.decode(src), + Framer::VarintLengthDelimited(framer) => framer.decode(src), } } @@ -201,15 +217,16 @@ impl tokio_util::codec::Decoder for Framer { Framer::OctetCounting(framer) => framer.decode_eof(src), Framer::Boxed(framer) => framer.decode_eof(src), Framer::ChunkedGelf(framer) => framer.decode_eof(src), + Framer::VarintLengthDelimited(framer) => framer.decode_eof(src), } } } -/// Deserializer configuration. +/// Configures how events are decoded from raw bytes. Note some decoders can also determine the event output +/// type (log, metric, trace). #[configurable_component] #[derive(Clone, Debug)] #[serde(tag = "codec", rename_all = "snake_case")] -#[configurable(description = "Configures how events are decoded from raw bytes.")] #[configurable(metadata(docs::enum_tag_description = "The codec to use for decoding events."))] pub enum DeserializerConfig { /// Uses the raw bytes as-is. @@ -237,6 +254,8 @@ pub enum DeserializerConfig { /// Decodes the raw bytes as [native Protocol Buffers format][vector_native_protobuf]. /// + /// This decoder can output all types of events (logs, metrics, traces). + /// /// This codec is **[experimental][experimental]**. /// /// [vector_native_protobuf]: https://github.com/vectordotdev/vector/blob/master/lib/vector-core/proto/event.proto @@ -245,6 +264,8 @@ pub enum DeserializerConfig { /// Decodes the raw bytes as [native JSON format][vector_native_json]. /// + /// This decoder can output all types of events (logs, metrics, traces). + /// /// This codec is **[experimental][experimental]**. /// /// [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue @@ -256,7 +277,7 @@ pub enum DeserializerConfig { /// This codec is experimental for the following reason: /// /// The GELF specification is more strict than the actual Graylog receiver. - /// Vector's decoder currently adheres more strictly to the GELF spec, with + /// Vector's decoder adheres more strictly to the GELF spec, with /// the exception that some characters such as `@` are allowed in field names. /// /// Other GELF codecs such as Loki's, use a [Go SDK][implementation] that is maintained @@ -464,6 +485,7 @@ impl DeserializerConfig { } /// Parse structured events from bytes. +#[allow(clippy::large_enum_variant)] #[derive(Clone)] pub enum Deserializer { /// Uses a `AvroDeserializer` for deserialization. diff --git a/lib/codecs/src/encoding/format/avro.rs b/lib/codecs/src/encoding/format/avro.rs index bfac175f18..c06bdee229 100644 --- a/lib/codecs/src/encoding/format/avro.rs +++ b/lib/codecs/src/encoding/format/avro.rs @@ -23,7 +23,7 @@ impl AvroSerializerConfig { /// Build the `AvroSerializer` from this configuration. pub fn build(&self) -> Result { let schema = apache_avro::Schema::parse_str(&self.avro.schema) - .map_err(|error| format!("Failed building Avro serializer: {}", error))?; + .map_err(|error| format!("Failed building Avro serializer: {error}"))?; Ok(AvroSerializer { schema }) } diff --git a/lib/codecs/src/encoding/format/cef.rs b/lib/codecs/src/encoding/format/cef.rs index d686495da6..78056e03c5 100644 --- a/lib/codecs/src/encoding/format/cef.rs +++ b/lib/codecs/src/encoding/format/cef.rs @@ -195,7 +195,7 @@ impl std::fmt::Display for Version { #[derive(Debug, Clone)] pub struct CefSerializerOptions { /// CEF Version. Can be either 0 or 1. - /// Equals to "0" by default. + /// Set to "0" by default. pub version: Version, /// Identifies the vendor of the product. @@ -208,7 +208,7 @@ pub struct CefSerializerOptions { /// The value length must be less than or equal to 63. pub device_product: String, - /// Identifies the version of the problem. In combination with device product and vendor, it composes the unique id of the device that sends messages. + /// Identifies the version of the problem. The combination of the device product, vendor and this value make up the unique id of the device that sends messages. /// The value length must be less than or equal to 31. pub device_version: String, @@ -220,8 +220,8 @@ pub struct CefSerializerOptions { /// Reflects importance of the event. /// /// It must point to a number from 0 to 10. - /// 0 = Lowest, 10 = Highest. - /// Equals to "cef.severity" by default. + /// 0 = lowest_importance, 10 = highest_importance. + /// Set to "cef.severity" by default. pub severity: ConfigTargetPath, /// This is a path that points to the human-readable description of a log event. @@ -321,7 +321,7 @@ impl Encoder for CefSerializer { continue; } let value = escape_extension(&value); - formatted_extensions.push(format!("{}={}", extension, value)); + formatted_extensions.push(format!("{extension}={value}")); } buffer.write_fmt(format_args!( @@ -368,7 +368,7 @@ fn escape_extension(s: &str) -> String { fn escape_special_chars(s: &str, extra_char: char) -> String { s.replace('\\', r#"\\"#) - .replace(extra_char, &format!(r#"\{}"#, extra_char)) + .replace(extra_char, &format!(r#"\{extra_char}"#)) } fn validate_length(field: &str, field_name: &str, max_length: usize) -> Result { diff --git a/lib/codecs/src/encoding/format/csv.rs b/lib/codecs/src/encoding/format/csv.rs index dd808d2d0f..560b29558a 100644 --- a/lib/codecs/src/encoding/format/csv.rs +++ b/lib/codecs/src/encoding/format/csv.rs @@ -27,8 +27,8 @@ pub enum QuoteStyle { Necessary, /// Puts quotes around all fields that are non-numeric. - /// Namely, when writing a field that does not parse as a valid float or integer, - /// then quotes are used even if they aren't strictly necessary. + /// This means that when writing a field that does not parse as a valid float or integer, + /// quotes are used even if they aren't strictly necessary. NonNumeric, /// Never writes quotes, even if it produces invalid CSV data. @@ -84,9 +84,9 @@ pub struct CsvSerializerOptions { )] pub delimiter: u8, - /// Enable double quote escapes. + /// Enables double quote escapes. /// - /// This is enabled by default, but it may be disabled. When disabled, quotes in + /// This is enabled by default, but you can disable it. When disabled, quotes in /// field data are escaped instead of doubled. #[serde( default = "default_double_quote", @@ -99,7 +99,7 @@ pub struct CsvSerializerOptions { /// In some variants of CSV, quotes are escaped using a special escape character /// like \ (instead of escaping quotes by doubling them). /// - /// To use this, `double_quotes` needs to be disabled as well otherwise it is ignored. + /// To use this, `double_quotes` needs to be disabled as well; otherwise, this setting is ignored. #[configurable(metadata(docs::type_override = "ascii_char"))] #[serde( default = "default_escape", @@ -121,18 +121,18 @@ pub struct CsvSerializerOptions { #[serde(default, skip_serializing_if = "vector_core::serde::is_default")] pub quote_style: QuoteStyle, - /// Set the capacity (in bytes) of the internal buffer used in the CSV writer. - /// This defaults to a reasonable setting. + /// Sets the capacity (in bytes) of the internal buffer used in the CSV writer. + /// This defaults to 8KB. #[serde(default = "default_capacity")] pub capacity: usize, - /// Configures the fields that will be encoded, as well as the order in which they + /// Configures the fields that are encoded, as well as the order in which they /// appear in the output. /// - /// If a field is not present in the event, the output will be an empty string. + /// If a field is not present in the event, the output for that field is an empty string. /// - /// Values of type `Array`, `Object`, and `Regex` are not supported and the - /// output will be an empty string. + /// Values of type `Array`, `Object`, and `Regex` are not supported, and the + /// output for any of these types is an empty string. pub fields: Vec, } diff --git a/lib/codecs/src/encoding/format/gelf.rs b/lib/codecs/src/encoding/format/gelf.rs index be5c861548..b47adb2941 100644 --- a/lib/codecs/src/encoding/format/gelf.rs +++ b/lib/codecs/src/encoding/format/gelf.rs @@ -1,5 +1,5 @@ use crate::gelf::GELF_TARGET_PATHS; -use crate::{gelf_fields::*, VALID_FIELD_REGEX}; +use crate::{VALID_FIELD_REGEX, gelf_fields::*}; use bytes::{BufMut, BytesMut}; use lookup::event_path; use ordered_float::NotNan; @@ -7,7 +7,7 @@ use serde::{Deserialize, Serialize}; use snafu::Snafu; use tokio_util::codec::Encoder; use vector_core::{ - config::{log_schema, DataType}, + config::{DataType, log_schema}, event::{Event, KeyString, LogEvent, Value}, schema, }; @@ -25,6 +25,12 @@ use vector_core::{ pub enum GelfSerializerError { #[snafu(display(r#"LogEvent does not contain required field: "{}""#, field))] MissingField { field: KeyString }, + #[snafu(display( + r#"LogEvent contains field with invalid name not matching pattern '{}': "{}""#, + pattern, + field, + ))] + InvalidField { field: KeyString, pattern: String }, #[snafu(display( r#"LogEvent contains a value with an invalid type. field = "{}" type = "{}" expected type = "{}""#, field, @@ -135,13 +141,13 @@ fn coerce_required_fields(mut log: LogEvent) -> vector_common::Result err_missing_field(HOST)?; } - if !log.contains(&GELF_TARGET_PATHS.short_message) { - if let Some(message_key) = log_schema().message_key_target_path() { - if log.contains(message_key) { - log.rename_key(message_key, &GELF_TARGET_PATHS.short_message); - } else { - err_missing_field(SHORT_MESSAGE)?; - } + if !log.contains(&GELF_TARGET_PATHS.short_message) + && let Some(message_key) = log_schema().message_key_target_path() + { + if log.contains(message_key) { + log.rename_key(message_key, &GELF_TARGET_PATHS.short_message); + } else { + err_missing_field(SHORT_MESSAGE)?; } } Ok(log) @@ -190,8 +196,9 @@ fn coerce_field_names_and_values( _ => { // additional fields must be only word chars, dashes and periods. if !VALID_FIELD_REGEX.is_match(field) { - return MissingFieldSnafu { + return InvalidFieldSnafu { field: field.clone(), + pattern: VALID_FIELD_REGEX.to_string(), } .fail() .map_err(|e| e.to_string().into()); diff --git a/lib/codecs/src/encoding/format/native.rs b/lib/codecs/src/encoding/format/native.rs index 0998f1a4d6..c3af9544b7 100644 --- a/lib/codecs/src/encoding/format/native.rs +++ b/lib/codecs/src/encoding/format/native.rs @@ -4,7 +4,7 @@ use serde::{Deserialize, Serialize}; use tokio_util::codec::Encoder; use vector_core::{ config::DataType, - event::{proto, Event, EventArray}, + event::{Event, EventArray, proto}, schema, }; diff --git a/lib/codecs/src/encoding/format/protobuf.rs b/lib/codecs/src/encoding/format/protobuf.rs index d685b68cf6..78e2842b74 100644 --- a/lib/codecs/src/encoding/format/protobuf.rs +++ b/lib/codecs/src/encoding/format/protobuf.rs @@ -1,6 +1,6 @@ use crate::encoding::BuildError; use bytes::BytesMut; -use prost_reflect::{prost::Message as _, MessageDescriptor}; +use prost_reflect::{MessageDescriptor, prost::Message as _}; use std::path::PathBuf; use tokio_util::codec::Encoder; use vector_config_macros::configurable_component; @@ -9,6 +9,8 @@ use vector_core::{ event::{Event, Value}, schema, }; +use vrl::protobuf::descriptor::get_message_descriptor; +use vrl::protobuf::encode::encode_message; /// Config used to build a `ProtobufSerializer`. #[configurable_component] @@ -21,16 +23,14 @@ pub struct ProtobufSerializerConfig { impl ProtobufSerializerConfig { /// Build the `ProtobufSerializer` from this configuration. pub fn build(&self) -> Result { - let message_descriptor = vrl::protobuf::get_message_descriptor( - &self.protobuf.desc_file, - &self.protobuf.message_type, - )?; + let message_descriptor = + get_message_descriptor(&self.protobuf.desc_file, &self.protobuf.message_type)?; Ok(ProtobufSerializer { message_descriptor }) } /// The data type of events that are accepted by `ProtobufSerializer`. pub fn input_type(&self) -> DataType { - DataType::Log + DataType::Log | DataType::Trace } /// The schema required by the serializer. @@ -47,7 +47,9 @@ impl ProtobufSerializerConfig { pub struct ProtobufSerializerOptions { /// The path to the protobuf descriptor set file. /// - /// This file is the output of `protoc -o ...` + /// This file is the output of `protoc -I -o ` + /// + /// You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). #[configurable(metadata(docs::examples = "/etc/vector/protobuf_descriptor_set.desc"))] pub desc_file: PathBuf, @@ -80,11 +82,9 @@ impl Encoder for ProtobufSerializer { fn encode(&mut self, event: Event, buffer: &mut BytesMut) -> Result<(), Self::Error> { let message = match event { - Event::Log(log) => { - vrl::protobuf::encode_message(&self.message_descriptor, log.into_parts().0) - } + Event::Log(log) => encode_message(&self.message_descriptor, log.into_parts().0), Event::Metric(_) => unimplemented!(), - Event::Trace(trace) => vrl::protobuf::encode_message( + Event::Trace(trace) => encode_message( &self.message_descriptor, Value::Object(trace.into_parts().0), ), diff --git a/lib/codecs/src/encoding/framing/length_delimited.rs b/lib/codecs/src/encoding/framing/length_delimited.rs index 1e450073b9..bd38089ba6 100644 --- a/lib/codecs/src/encoding/framing/length_delimited.rs +++ b/lib/codecs/src/encoding/framing/length_delimited.rs @@ -26,18 +26,27 @@ impl LengthDelimitedEncoderConfig { /// An encoder for handling bytes that are delimited by a length header. #[derive(Debug, Clone)] -pub struct LengthDelimitedEncoder(LengthDelimitedCodec); +pub struct LengthDelimitedEncoder { + codec: LengthDelimitedCodec, + inner_buffer: BytesMut, +} impl LengthDelimitedEncoder { /// Creates a new `LengthDelimitedEncoder`. pub fn new(config: &LengthDelimitedCoderOptions) -> Self { - Self(config.build_codec()) + Self { + codec: config.build_codec(), + inner_buffer: BytesMut::new(), + } } } impl Default for LengthDelimitedEncoder { fn default() -> Self { - Self(LengthDelimitedCodec::new()) + Self { + codec: LengthDelimitedCodec::new(), + inner_buffer: BytesMut::new(), + } } } @@ -45,8 +54,11 @@ impl Encoder<()> for LengthDelimitedEncoder { type Error = BoxedFramingError; fn encode(&mut self, _: (), buffer: &mut BytesMut) -> Result<(), BoxedFramingError> { - let bytes = buffer.split().freeze(); - self.0.encode(bytes, buffer)?; + self.inner_buffer.clear(); + self.inner_buffer.extend_from_slice(buffer); + buffer.clear(); + let bytes = self.inner_buffer.split().freeze(); + self.codec.encode(bytes, buffer)?; Ok(()) } } diff --git a/lib/codecs/src/encoding/framing/mod.rs b/lib/codecs/src/encoding/framing/mod.rs index 103b0aaf18..6eaca98858 100644 --- a/lib/codecs/src/encoding/framing/mod.rs +++ b/lib/codecs/src/encoding/framing/mod.rs @@ -7,6 +7,7 @@ mod bytes; mod character_delimited; mod length_delimited; mod newline_delimited; +mod varint_length_delimited; use std::fmt::Debug; @@ -19,6 +20,9 @@ pub use newline_delimited::{NewlineDelimitedEncoder, NewlineDelimitedEncoderConf use tokio_util::codec::LinesCodecError; pub use self::bytes::{BytesEncoder, BytesEncoderConfig}; +pub use self::varint_length_delimited::{ + VarintLengthDelimitedEncoder, VarintLengthDelimitedEncoderConfig, +}; /// An error that occurred while framing bytes. pub trait FramingError: std::error::Error + Send + Sync {} @@ -35,6 +39,12 @@ impl From for BoxedFramingError { } } +impl From for BoxedFramingError { + fn from(error: varint_length_delimited::VarintFramingError) -> Self { + Box::new(error) + } +} + /// A `Box` containing a `FramingError`. pub type BoxedFramingError = Box; diff --git a/lib/codecs/src/encoding/framing/varint_length_delimited.rs b/lib/codecs/src/encoding/framing/varint_length_delimited.rs new file mode 100644 index 0000000000..87f3737fa3 --- /dev/null +++ b/lib/codecs/src/encoding/framing/varint_length_delimited.rs @@ -0,0 +1,147 @@ +use bytes::{BufMut, BytesMut}; +use derivative::Derivative; +use snafu::Snafu; +use tokio_util::codec::Encoder; +use vector_config::configurable_component; + +use super::{BoxedFramingError, FramingError}; + +/// Errors that can occur during varint length delimited framing. +#[derive(Debug, Snafu)] +pub enum VarintFramingError { + #[snafu(display("Frame too large: {length} bytes (max: {max})"))] + FrameTooLarge { length: usize, max: usize }, +} + +impl FramingError for VarintFramingError {} + +/// Config used to build a `VarintLengthDelimitedEncoder`. +#[configurable_component] +#[derive(Debug, Clone, PartialEq, Eq, Derivative)] +#[derivative(Default)] +pub struct VarintLengthDelimitedEncoderConfig { + /// Maximum frame length + #[serde(default = "default_max_frame_length")] + pub max_frame_length: usize, +} + +const fn default_max_frame_length() -> usize { + 8 * 1_024 * 1_024 +} + +impl VarintLengthDelimitedEncoderConfig { + /// Build the `VarintLengthDelimitedEncoder` from this configuration. + pub fn build(&self) -> VarintLengthDelimitedEncoder { + VarintLengthDelimitedEncoder::new(self.max_frame_length) + } +} + +/// A codec for handling bytes sequences whose length is encoded as a varint prefix. +/// This is compatible with protobuf's length-delimited encoding. +#[derive(Debug, Clone)] +pub struct VarintLengthDelimitedEncoder { + max_frame_length: usize, +} + +impl VarintLengthDelimitedEncoder { + /// Creates a new `VarintLengthDelimitedEncoder`. + pub fn new(max_frame_length: usize) -> Self { + Self { max_frame_length } + } + + /// Encode a varint into the buffer + fn encode_varint(&self, value: usize, buf: &mut BytesMut) -> Result<(), BoxedFramingError> { + if value > self.max_frame_length { + return Err(VarintFramingError::FrameTooLarge { + length: value, + max: self.max_frame_length, + } + .into()); + } + + let mut val = value; + while val >= 0x80 { + buf.put_u8((val as u8) | 0x80); + val >>= 7; + } + buf.put_u8(val as u8); + Ok(()) + } +} + +impl Default for VarintLengthDelimitedEncoder { + fn default() -> Self { + Self::new(default_max_frame_length()) + } +} + +impl Encoder<()> for VarintLengthDelimitedEncoder { + type Error = BoxedFramingError; + + fn encode(&mut self, _: (), buffer: &mut BytesMut) -> Result<(), Self::Error> { + // This encoder expects the data to already be in the buffer + // We just need to prepend the varint length + let data_length = buffer.len(); + if data_length == 0 { + return Ok(()); + } + + // Create a temporary buffer to hold the varint + let mut varint_buffer = BytesMut::new(); + self.encode_varint(data_length, &mut varint_buffer)?; + + // Prepend the varint to the buffer + let varint_bytes = varint_buffer.freeze(); + let data_bytes = buffer.split_to(buffer.len()); + buffer.extend_from_slice(&varint_bytes); + buffer.extend_from_slice(&data_bytes); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn encode_single_byte_varint() { + let mut buffer = BytesMut::from(&b"foo"[..]); + let mut encoder = VarintLengthDelimitedEncoder::default(); + + encoder.encode((), &mut buffer).unwrap(); + assert_eq!(buffer, &[0x03, b'f', b'o', b'o'][..]); + } + + #[test] + fn encode_multi_byte_varint() { + let mut buffer = BytesMut::from(&b"foo"[..]); + let mut encoder = VarintLengthDelimitedEncoder::new(1000); + + // Set a larger frame to trigger multi-byte varint + buffer.clear(); + buffer.extend_from_slice(&vec![b'x'; 300]); + encoder.encode((), &mut buffer).unwrap(); + + // 300 in varint encoding: 0xAC 0x02 + assert_eq!(buffer[0..2], [0xAC, 0x02]); + assert_eq!(buffer.len(), 302); // 2 bytes varint + 300 bytes data + } + + #[test] + fn encode_frame_too_large() { + let large_data = vec![b'x'; 1001]; + let mut buffer = BytesMut::from(&large_data[..]); + let mut encoder = VarintLengthDelimitedEncoder::new(1000); + + assert!(encoder.encode((), &mut buffer).is_err()); + } + + #[test] + fn encode_empty_buffer() { + let mut buffer = BytesMut::new(); + let mut encoder = VarintLengthDelimitedEncoder::default(); + + encoder.encode((), &mut buffer).unwrap(); + assert_eq!(buffer.len(), 0); + } +} diff --git a/lib/codecs/src/encoding/mod.rs b/lib/codecs/src/encoding/mod.rs index 5a23deb7ad..7c9209b974 100644 --- a/lib/codecs/src/encoding/mod.rs +++ b/lib/codecs/src/encoding/mod.rs @@ -20,6 +20,7 @@ pub use framing::{ BoxedFramer, BoxedFramingError, BytesEncoder, BytesEncoderConfig, CharacterDelimitedEncoder, CharacterDelimitedEncoderConfig, CharacterDelimitedEncoderOptions, LengthDelimitedEncoder, LengthDelimitedEncoderConfig, NewlineDelimitedEncoder, NewlineDelimitedEncoderConfig, + VarintLengthDelimitedEncoder, VarintLengthDelimitedEncoderConfig, }; use vector_config::configurable_component; use vector_core::{config::DataType, event::Event, schema}; @@ -39,8 +40,8 @@ pub enum Error { impl std::fmt::Display for Error { fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - Self::FramingError(error) => write!(formatter, "FramingError({})", error), - Self::SerializingError(error) => write!(formatter, "SerializingError({})", error), + Self::FramingError(error) => write!(formatter, "FramingError({error})"), + Self::SerializingError(error) => write!(formatter, "SerializingError({error})"), } } } @@ -72,6 +73,11 @@ pub enum FramingConfig { /// Event data is delimited by a newline (LF) character. NewlineDelimited, + + /// Event data is prefixed with its length in bytes as a varint. + /// + /// This is compatible with protobuf's length-delimited encoding. + VarintLengthDelimited(VarintLengthDelimitedEncoderConfig), } impl From for FramingConfig { @@ -98,6 +104,12 @@ impl From for FramingConfig { } } +impl From for FramingConfig { + fn from(config: VarintLengthDelimitedEncoderConfig) -> Self { + Self::VarintLengthDelimited(config) + } +} + impl FramingConfig { /// Build the `Framer` from this configuration. pub fn build(&self) -> Framer { @@ -108,6 +120,9 @@ impl FramingConfig { FramingConfig::NewlineDelimited => { Framer::NewlineDelimited(NewlineDelimitedEncoderConfig.build()) } + FramingConfig::VarintLengthDelimited(config) => { + Framer::VarintLengthDelimited(config.build()) + } } } } @@ -123,6 +138,8 @@ pub enum Framer { LengthDelimited(LengthDelimitedEncoder), /// Uses a `NewlineDelimitedEncoder` for framing. NewlineDelimited(NewlineDelimitedEncoder), + /// Uses a `VarintLengthDelimitedEncoder` for framing. + VarintLengthDelimited(VarintLengthDelimitedEncoder), /// Uses an opaque `Encoder` implementation for framing. Boxed(BoxedFramer), } @@ -151,6 +168,12 @@ impl From for Framer { } } +impl From for Framer { + fn from(encoder: VarintLengthDelimitedEncoder) -> Self { + Self::VarintLengthDelimited(encoder) + } +} + impl From for Framer { fn from(encoder: BoxedFramer) -> Self { Self::Boxed(encoder) @@ -166,6 +189,7 @@ impl tokio_util::codec::Encoder<()> for Framer { Framer::CharacterDelimited(framer) => framer.encode((), buffer), Framer::LengthDelimited(framer) => framer.encode((), buffer), Framer::NewlineDelimited(framer) => framer.encode((), buffer), + Framer::VarintLengthDelimited(framer) => framer.encode((), buffer), Framer::Boxed(framer) => framer.encode((), buffer), } } @@ -206,11 +230,11 @@ pub enum SerializerConfig { /// Vector's encoder currently adheres more strictly to the GELF spec, with /// the exception that some characters such as `@` are allowed in field names. /// - /// Other GELF codecs such as Loki's, use a [Go SDK][implementation] that is maintained - /// by Graylog, and is much more relaxed than the GELF spec. + /// Other GELF codecs, such as Loki's, use a [Go SDK][implementation] that is maintained + /// by Graylog and is much more relaxed than the GELF spec. /// /// Going forward, Vector will use that [Go SDK][implementation] as the reference implementation, which means - /// the codec may continue to relax the enforcement of specification. + /// the codec might continue to relax the enforcement of the specification. /// /// [gelf]: https://docs.graylog.org/docs/gelf /// [implementation]: https://github.com/Graylog2/go-gelf/blob/v2/gelf/reader.go @@ -371,11 +395,12 @@ impl SerializerConfig { // we should do so accurately, even if practically it doesn't need to be. // // [1]: https://avro.apache.org/docs/1.11.1/specification/_print/#message-framing - SerializerConfig::Avro { .. } - | SerializerConfig::Native - | SerializerConfig::Protobuf(_) => { + SerializerConfig::Avro { .. } | SerializerConfig::Native => { FramingConfig::LengthDelimited(LengthDelimitedEncoderConfig::default()) } + SerializerConfig::Protobuf(_) => { + FramingConfig::VarintLengthDelimited(VarintLengthDelimitedEncoderConfig::default()) + } SerializerConfig::Cef(_) | SerializerConfig::Csv(_) | SerializerConfig::Json(_) @@ -397,7 +422,7 @@ impl SerializerConfig { } SerializerConfig::Cef(config) => config.input_type(), SerializerConfig::Csv(config) => config.input_type(), - SerializerConfig::Gelf { .. } => GelfSerializerConfig::input_type(), + SerializerConfig::Gelf => GelfSerializerConfig::input_type(), SerializerConfig::Json(config) => config.input_type(), SerializerConfig::Logfmt => LogfmtSerializerConfig.input_type(), SerializerConfig::Native => NativeSerializerConfig.input_type(), @@ -416,7 +441,7 @@ impl SerializerConfig { } SerializerConfig::Cef(config) => config.schema_requirement(), SerializerConfig::Csv(config) => config.schema_requirement(), - SerializerConfig::Gelf { .. } => GelfSerializerConfig::schema_requirement(), + SerializerConfig::Gelf => GelfSerializerConfig::schema_requirement(), SerializerConfig::Json(config) => config.schema_requirement(), SerializerConfig::Logfmt => LogfmtSerializerConfig.schema_requirement(), SerializerConfig::Native => NativeSerializerConfig.schema_requirement(), diff --git a/lib/codecs/src/lib.rs b/lib/codecs/src/lib.rs index 61a9c9e1bb..e386bb30f3 100644 --- a/lib/codecs/src/lib.rs +++ b/lib/codecs/src/lib.rs @@ -16,7 +16,7 @@ pub use decoding::{ LengthDelimitedDecoderConfig, NativeDeserializer, NativeDeserializerConfig, NativeJsonDeserializer, NativeJsonDeserializerConfig, NewlineDelimitedDecoder, NewlineDelimitedDecoderConfig, OctetCountingDecoder, OctetCountingDecoderConfig, - StreamDecodingError, + StreamDecodingError, VarintLengthDelimitedDecoder, VarintLengthDelimitedDecoderConfig, }; #[cfg(feature = "syslog")] pub use decoding::{SyslogDeserializer, SyslogDeserializerConfig}; @@ -28,7 +28,7 @@ pub use encoding::{ NativeSerializerConfig, NewlineDelimitedEncoder, NewlineDelimitedEncoderConfig, RawMessageSerializer, RawMessageSerializerConfig, TextSerializer, TextSerializerConfig, }; -pub use gelf::{gelf_fields, VALID_FIELD_REGEX}; +pub use gelf::{VALID_FIELD_REGEX, gelf_fields}; use vector_config_macros::configurable_component; /// The user configuration to choose the metric tag strategy. diff --git a/lib/codecs/tests/bin/generate-avro-fixtures.rs b/lib/codecs/tests/bin/generate-avro-fixtures.rs index 18fb6a60d9..0537272910 100644 --- a/lib/codecs/tests/bin/generate-avro-fixtures.rs +++ b/lib/codecs/tests/bin/generate-avro-fixtures.rs @@ -1,4 +1,4 @@ -use apache_avro::{types::Value, Decimal, Schema}; +use apache_avro::{Decimal, Schema, types::Value}; use serde::{Deserialize, Serialize}; use std::fs::File; use std::io::Write; diff --git a/lib/codecs/tests/native.rs b/lib/codecs/tests/native.rs index c39a02ea5f..4f6d61049a 100644 --- a/lib/codecs/tests/native.rs +++ b/lib/codecs/tests/native.rs @@ -6,8 +6,8 @@ use std::{ use bytes::{Bytes, BytesMut}; use codecs::{ - decoding::format::Deserializer, encoding::format::Serializer, NativeDeserializerConfig, - NativeJsonDeserializerConfig, NativeJsonSerializerConfig, NativeSerializerConfig, + NativeDeserializerConfig, NativeJsonDeserializerConfig, NativeJsonSerializerConfig, + NativeSerializerConfig, decoding::format::Deserializer, encoding::format::Serializer, }; use similar_asserts::assert_eq; use vector_core::{config::LogNamespace, event::Event}; @@ -314,7 +314,7 @@ fn rebuild_fixtures(proto: &str, deserializer: &dyn Deserializer, serializer: &m .into_iter() .collect(); let mut out = File::create(&new_path).unwrap_or_else(|error| { - panic!("Could not create rebuilt file {:?}: {:?}", new_path, error) + panic!("Could not create rebuilt file {new_path:?}: {error:?}") }); out.write_all(&buf).expect("Could not write rebuilt data"); out.flush().expect("Could not write rebuilt data"); diff --git a/lib/codecs/tests/varint_framing.rs b/lib/codecs/tests/varint_framing.rs new file mode 100644 index 0000000000..5bced68a8e --- /dev/null +++ b/lib/codecs/tests/varint_framing.rs @@ -0,0 +1,121 @@ +use bytes::BytesMut; +use codecs::{ + VarintLengthDelimitedDecoder, VarintLengthDelimitedDecoderConfig, + encoding::VarintLengthDelimitedEncoder, encoding::VarintLengthDelimitedEncoderConfig, +}; +use tokio_util::codec::{Decoder, Encoder}; + +#[test] +fn test_varint_framing_roundtrip() { + let test_data = vec![b"hello".to_vec(), b"world".to_vec(), b"protobuf".to_vec()]; + + let mut encoder = VarintLengthDelimitedEncoder::default(); + let mut decoder = VarintLengthDelimitedDecoder::default(); + let mut encoded_buffer = BytesMut::new(); + + // Encode all test data + for data in &test_data { + let mut frame_buffer = BytesMut::from(data.as_slice()); + encoder.encode((), &mut frame_buffer).unwrap(); + encoded_buffer.extend_from_slice(&frame_buffer); + } + + // Decode all test data + let mut decoded_data = Vec::new(); + let mut decode_buffer = encoded_buffer; + + while !decode_buffer.is_empty() { + match decoder.decode(&mut decode_buffer) { + Ok(Some(frame)) => { + decoded_data.push(frame.to_vec()); + } + Ok(None) => { + // Need more data, but we've provided all data + break; + } + Err(e) => { + panic!("Decoding error: {e:?}"); + } + } + } + + // Verify roundtrip + assert_eq!(decoded_data.len(), test_data.len()); + for (original, decoded) in test_data.iter().zip(decoded_data.iter()) { + assert_eq!(original, decoded); + } +} + +#[test] +fn test_varint_framing_large_frame() { + let large_data = vec![b'x'; 300]; // 300 bytes + let mut encoder = VarintLengthDelimitedEncoder::default(); + let mut decoder = VarintLengthDelimitedDecoder::default(); + + // Encode + let mut frame_buffer = BytesMut::from(&large_data[..]); + encoder.encode((), &mut frame_buffer).unwrap(); + + // Verify varint encoding (300 = 0xAC 0x02) + assert_eq!(frame_buffer[0], 0xAC); + assert_eq!(frame_buffer[1], 0x02); + assert_eq!(frame_buffer.len(), 302); // 2 bytes varint + 300 bytes data + + // Decode + let decoded = decoder.decode(&mut frame_buffer).unwrap().unwrap(); + assert_eq!(decoded.to_vec(), large_data); +} + +#[test] +fn test_varint_framing_incomplete_frame() { + let mut decoder = VarintLengthDelimitedDecoder::default(); + let mut buffer = BytesMut::from(&[0x05, b'f', b'o'][..]); // Length 5, but only 2 bytes + + // Should return None for incomplete frame + assert_eq!(decoder.decode(&mut buffer).unwrap(), None); +} + +#[test] +fn test_varint_framing_incomplete_varint() { + let mut decoder = VarintLengthDelimitedDecoder::default(); + let mut buffer = BytesMut::from(&[0x80][..]); // Incomplete varint + + // Should return None for incomplete varint + assert_eq!(decoder.decode(&mut buffer).unwrap(), None); +} + +#[test] +fn test_varint_framing_frame_too_large() { + let mut encoder = VarintLengthDelimitedEncoder::new(1000); + let large_data = vec![b'x'; 1001]; // Exceeds max frame length + + let mut frame_buffer = BytesMut::from(&large_data[..]); + assert!(encoder.encode((), &mut frame_buffer).is_err()); +} + +#[test] +fn test_varint_framing_empty_frame() { + let mut encoder = VarintLengthDelimitedEncoder::default(); + let mut decoder = VarintLengthDelimitedDecoder::default(); + let mut buffer = BytesMut::new(); + + // Encode empty frame + encoder.encode((), &mut buffer).unwrap(); + assert_eq!(buffer.len(), 0); // Empty frames are not encoded + + // Try to decode empty buffer + assert_eq!(decoder.decode(&mut buffer).unwrap(), None); +} + +#[test] +fn test_varint_framing_config() { + let config = VarintLengthDelimitedEncoderConfig { + max_frame_length: 1000, + }; + let _encoder = config.build(); + + let decoder_config = VarintLengthDelimitedDecoderConfig { + max_frame_length: 1000, + }; + let _decoder = decoder_config.build(); +} diff --git a/lib/dnsmsg-parser/Cargo.toml b/lib/dnsmsg-parser/Cargo.toml index ceab56f024..5dd9051955 100644 --- a/lib/dnsmsg-parser/Cargo.toml +++ b/lib/dnsmsg-parser/Cargo.toml @@ -2,17 +2,17 @@ name = "dnsmsg-parser" version = "0.1.0" authors = ["Vector Contributors "] -edition = "2021" +edition = "2024" publish = false license = "MIT" [dependencies] -data-encoding = "2.6" +data-encoding = "2.9" hickory-proto.workspace = true snafu.workspace = true [dev-dependencies] -criterion = "0.5" +criterion = "0.7" [lib] bench = false diff --git a/lib/dnsmsg-parser/benches/benches.rs b/lib/dnsmsg-parser/benches/benches.rs index 89b7147ac5..29247accde 100644 --- a/lib/dnsmsg-parser/benches/benches.rs +++ b/lib/dnsmsg-parser/benches/benches.rs @@ -1,4 +1,4 @@ -use criterion::{criterion_group, criterion_main, BatchSize, Criterion, Throughput}; +use criterion::{BatchSize, Criterion, Throughput, criterion_group, criterion_main}; use data_encoding::BASE64; use dnsmsg_parser::dns_message_parser::DnsMessageParser; use hickory_proto::rr::rdata::NULL; diff --git a/lib/dnsmsg-parser/src/dns_message.rs b/lib/dnsmsg-parser/src/dns_message.rs index d22bff2806..fd0cc8f613 100644 --- a/lib/dnsmsg-parser/src/dns_message.rs +++ b/lib/dnsmsg-parser/src/dns_message.rs @@ -16,7 +16,6 @@ pub(super) const RTYPE_NSAP: u16 = 22; pub(super) const RTYPE_PX: u16 = 26; pub(super) const RTYPE_LOC: u16 = 29; pub(super) const RTYPE_KX: u16 = 36; -pub(super) const RTYPE_CERT: u16 = 37; pub(super) const RTYPE_A6: u16 = 38; pub(super) const RTYPE_SINK: u16 = 40; pub(super) const RTYPE_APL: u16 = 42; diff --git a/lib/dnsmsg-parser/src/dns_message_parser.rs b/lib/dnsmsg-parser/src/dns_message_parser.rs index efa54c372e..6dd90def9e 100644 --- a/lib/dnsmsg-parser/src/dns_message_parser.rs +++ b/lib/dnsmsg-parser/src/dns_message_parser.rs @@ -1,23 +1,22 @@ +use hickory_proto::dnssec::{PublicKey, Verifier}; +use std::fmt::Write as _; use std::str::Utf8Error; -use std::{fmt::Write as _, ops::Deref}; use data_encoding::{BASE32HEX_NOPAD, BASE64, HEXUPPER}; +use hickory_proto::dnssec::SupportedAlgorithms; +use hickory_proto::dnssec::rdata::{CDNSKEY, CDS, DNSKEY, DNSSECRData, DS}; +use hickory_proto::rr::rdata::caa::Property; use hickory_proto::{ - error::ProtoError, - op::{message::Message as TrustDnsMessage, Query}, + ProtoError, + op::{Query, message::Message as TrustDnsMessage}, rr::{ - dnssec::{ - rdata::{DNSSECRData, DNSKEY, DS}, - Algorithm, SupportedAlgorithms, - }, + Name, RecordType, rdata::{ - caa::Value, - opt::{EdnsCode, EdnsOption}, A, AAAA, NULL, OPT, SVCB, + opt::{EdnsCode, EdnsOption}, }, record_data::RData, resource::Record, - Name, RecordType, }, serialize::binary::{BinDecodable, BinDecoder}, }; @@ -188,11 +187,9 @@ impl DnsMessageParser { pub(crate) fn parse_dns_record(&mut self, record: &Record) -> DnsParserResult { let record_data = match record.data() { - Some(RData::Unknown { code, rdata }) => { - self.format_unknown_rdata((*code).into(), rdata) - } - Some(rdata) => self.format_rdata(rdata), - None => Ok((Some(String::from("")), None)), // NULL record + RData::Unknown { code, rdata } => self.format_unknown_rdata((*code).into(), rdata), + RData::Update0(_) => Ok((Some(String::from("")), None)), // Previously none value + rdata => self.format_rdata(rdata), }?; Ok(DnsRecord { @@ -244,8 +241,7 @@ impl DnsMessageParser { } for _i in 0..8 { if current_byte & 0b1000_0000 == 0b1000_0000 { - write!(port_string, "{} ", current_bit) - .expect("can always write to String"); + write!(port_string, "{current_bit} ").expect("can always write to String"); } current_byte <<= 1; current_bit += 1; @@ -266,6 +262,11 @@ impl DnsMessageParser { let mut decoder = BinDecoder::new(raw_rdata); let prefix = parse_u8(&mut decoder)?; let ipv6_address = { + if prefix > 128 { + return Err(DnsMessageParserError::SimpleError { + cause: String::from("IPV6 prefix can't be greater than 128."), + }); + } let address_length = (128 - prefix) / 8; let mut address_vec = parse_vec(&mut decoder, address_length)?; if address_vec.len() < 16 { @@ -278,10 +279,7 @@ impl DnsMessageParser { parse_ipv6_address(&mut dec)? }; let domain_name = Self::parse_domain_name(&mut decoder, &self.options)?; - Ok(( - Some(format!("{} {} {}", prefix, ipv6_address, domain_name)), - None, - )) + Ok((Some(format!("{prefix} {ipv6_address} {domain_name}")), None)) } fn parse_loc_rdata( @@ -328,8 +326,7 @@ impl DnsMessageParser { Ok(( Some(format!( - "{} {} {:.2}m {}m {}m {}m", - latitude, longitude, altitude, size, horizontal_precision, vertical_precision + "{latitude} {longitude} {altitude:.2}m {size}m {horizontal_precision}m {vertical_precision}m" )), None, )) @@ -365,12 +362,8 @@ impl DnsMessageParser { let mut dec = BinDecoder::new(&address_vec); parse_ipv6_address(&mut dec)? }; - write!( - apl_rdata, - "{}{}:{}/{}", - negation, address_family, address, prefix - ) - .expect("can always write to String"); + write!(apl_rdata, "{negation}{address_family}:{address}/{prefix}") + .expect("can always write to String"); apl_rdata.push(' '); } Ok((Some(apl_rdata.trim_end().to_string()), None)) @@ -410,7 +403,7 @@ impl DnsMessageParser { let mut decoder = self.get_rdata_decoder_with_raw_message(rdata.anything()); let rmailbx = Self::parse_domain_name(&mut decoder, &options)?; let emailbx = Self::parse_domain_name(&mut decoder, &options)?; - Ok((Some(format!("{} {}", rmailbx, emailbx)), None)) + Ok((Some(format!("{rmailbx} {emailbx}")), None)) } dns_message::RTYPE_RP => { @@ -418,7 +411,7 @@ impl DnsMessageParser { let mut decoder = self.get_rdata_decoder_with_raw_message(rdata.anything()); let mbox = Self::parse_domain_name(&mut decoder, &options)?; let txt = Self::parse_domain_name(&mut decoder, &options)?; - Ok((Some(format!("{} {}", mbox, txt)), None)) + Ok((Some(format!("{mbox} {txt}")), None)) } dns_message::RTYPE_AFSDB => { @@ -426,7 +419,7 @@ impl DnsMessageParser { let mut decoder = self.get_rdata_decoder_with_raw_message(rdata.anything()); let subtype = parse_u16(&mut decoder)?; let hostname = Self::parse_domain_name(&mut decoder, &options)?; - Ok((Some(format!("{} {}", subtype, hostname)), None)) + Ok((Some(format!("{subtype} {hostname}")), None)) } dns_message::RTYPE_X25 => { @@ -470,7 +463,7 @@ impl DnsMessageParser { let mut decoder = self.get_rdata_decoder_with_raw_message(rdata.anything()); let preference = parse_u16(&mut decoder)?; let intermediate_host = Self::parse_domain_name(&mut decoder, &options)?; - Ok((Some(format!("{} {}", preference, intermediate_host)), None)) + Ok((Some(format!("{preference} {intermediate_host}")), None)) } dns_message::RTYPE_NSAP => { @@ -478,7 +471,7 @@ impl DnsMessageParser { let mut decoder = BinDecoder::new(raw_rdata); let rdata_len = raw_rdata.len() as u16; let nsap_rdata = HEXUPPER.encode(&parse_vec_with_u16_len(&mut decoder, rdata_len)?); - Ok((Some(format!("0x{}", nsap_rdata)), None)) + Ok((Some(format!("0x{nsap_rdata}")), None)) } dns_message::RTYPE_PX => { @@ -487,7 +480,7 @@ impl DnsMessageParser { let preference = parse_u16(&mut decoder)?; let map822 = Self::parse_domain_name(&mut decoder, &options)?; let mapx400 = Self::parse_domain_name(&mut decoder, &options)?; - Ok((Some(format!("{} {} {}", preference, map822, mapx400)), None)) + Ok((Some(format!("{preference} {map822} {mapx400}")), None)) } dns_message::RTYPE_LOC => self.parse_loc_rdata(rdata.anything()), @@ -497,21 +490,7 @@ impl DnsMessageParser { let mut decoder = self.get_rdata_decoder_with_raw_message(rdata.anything()); let preference = parse_u16(&mut decoder)?; let exchanger = Self::parse_domain_name(&mut decoder, &options)?; - Ok((Some(format!("{} {}", preference, exchanger)), None)) - } - - dns_message::RTYPE_CERT => { - let raw_rdata = rdata.anything(); - let mut decoder = BinDecoder::new(raw_rdata); - let cert_type = parse_u16(&mut decoder)?; - let key_tag = parse_u16(&mut decoder)?; - let algorithm = Algorithm::from_u8(parse_u8(&mut decoder)?).as_str(); - let crl_len = raw_rdata.len() as u16 - 5; - let crl = BASE64.encode(&parse_vec_with_u16_len(&mut decoder, crl_len)?); - Ok(( - Some(format!("{} {} {} {}", cert_type, key_tag, algorithm, crl)), - None, - )) + Ok((Some(format!("{preference} {exchanger}")), None)) } dns_message::RTYPE_A6 => self.parse_a6_rdata(rdata.anything()), @@ -525,10 +504,7 @@ impl DnsMessageParser { let data_len = raw_rdata.len() as u16 - 3; let data = BASE64.encode(&parse_vec_with_u16_len(&mut decoder, data_len)?); - Ok(( - Some(format!("{} {} {} {}", meaning, coding, subcoding, data)), - None, - )) + Ok((Some(format!("{meaning} {coding} {subcoding} {data}")), None)) } dns_message::RTYPE_APL => self.parse_apl_rdata(rdata.anything()), @@ -562,9 +538,23 @@ impl DnsMessageParser { RData::AAAA(ip) => Ok((Some(ip.to_string()), None)), RData::ANAME(name) => Ok((Some(name.to_string_with_options(&self.options)), None)), RData::CNAME(name) => Ok((Some(name.to_string_with_options(&self.options)), None)), + RData::CERT(cert) => { + let crl = BASE64.encode(&cert.cert_data()); + Ok(( + Some(format!( + "{} {} {} {}", + u16::from(cert.cert_type()), + cert.key_tag(), + cert.algorithm(), + crl + )), + None, + )) + } + RData::CSYNC(csync) => { // Using CSYNC's formatter since not all data is exposed otherwise - let csync_rdata = format!("{}", csync); + let csync_rdata = format!("{csync}"); Ok((Some(csync_rdata), None)) } RData::MX(mx) => { @@ -631,11 +621,19 @@ impl DnsMessageParser { "{} {} \"{}\"", caa.issuer_critical() as u8, caa.tag().as_str(), - match caa.value() { - Value::Url(url) => { + match caa.tag() { + Property::Iodef => { + let url = caa.value_as_iodef().map_err(|source| { + DnsMessageParserError::TrustDnsError { source } + })?; url.as_str().to_string() } - Value::Issuer(option_name, vec_keyvalue) => { + Property::Issue | Property::IssueWild => { + let (option_name, vec_keyvalue) = + caa.value_as_issue().map_err(|source| { + DnsMessageParserError::TrustDnsError { source } + })?; + let mut final_issuer = String::new(); if let Some(name) = option_name { final_issuer.push_str(&name.to_string_with_options(&self.options)); @@ -648,9 +646,14 @@ impl DnsMessageParser { } final_issuer.trim_end().to_string() } - Value::Unknown(unknown) => std::str::from_utf8(unknown) - .map_err(|source| DnsMessageParserError::Utf8ParsingError { source })? - .to_string(), + Property::Unknown(_) => { + let unknown = caa.raw_value(); + std::str::from_utf8(unknown) + .map_err(|source| DnsMessageParserError::Utf8ParsingError { + source, + })? + .to_string() + } } ); Ok((Some(caa_rdata), None)) @@ -741,11 +744,9 @@ impl DnsMessageParser { RData::DNSSEC(dnssec) => match dnssec { // See https://tools.ietf.org/html/rfc4034 for details // on dnssec related rdata formats - DNSSECRData::CDS(cds) => Ok((Some(format_ds_record(cds.deref())), None)), + DNSSECRData::CDS(cds) => Ok((Some(format_cds_record(cds)), None)), DNSSECRData::DS(ds) => Ok((Some(format_ds_record(ds)), None)), - DNSSECRData::CDNSKEY(cdnskey) => { - Ok((Some(format_dnskey_record(cdnskey.deref())), None)) - } + DNSSECRData::CDNSKEY(cdnskey) => Ok((Some(format_cdnskey_record(cdnskey)), None)), DNSSECRData::DNSKEY(dnskey) => Ok((Some(format_dnskey_record(dnskey)), None)), DNSSECRData::NSEC(nsec) => { let nsec_rdata = format!( @@ -753,8 +754,7 @@ impl DnsMessageParser { nsec.next_domain_name() .to_string_with_options(&self.options), nsec.type_bit_maps() - .iter() - .flat_map(|e| format_record_type(*e)) + .flat_map(format_record_type) .collect::>() .join(" ") ); @@ -770,8 +770,7 @@ impl DnsMessageParser { BASE32HEX_NOPAD.encode(nsec3.next_hashed_owner_name()), nsec3 .type_bit_maps() - .iter() - .flat_map(|e| format_record_type(*e)) + .flat_map(format_record_type) .collect::>() .join(" ") ); @@ -798,8 +797,8 @@ impl DnsMessageParser { u8::from(sig.algorithm()), sig.num_labels(), sig.original_ttl(), - sig.sig_expiration(), // currently in epoch convert to human readable ? - sig.sig_inception(), // currently in epoch convert to human readable ? + sig.sig_expiration().get(), // currently in epoch convert to human readable ? + sig.sig_inception().get(), // currently in epoch convert to human readable ? sig.key_tag(), sig.signer_name().to_string_with_options(&self.options), BASE64.encode(sig.sig()) @@ -818,8 +817,8 @@ impl DnsMessageParser { u8::from(sig.algorithm()), sig.num_labels(), sig.original_ttl(), - sig.sig_expiration(), // currently in epoch convert to human readable ? - sig.sig_inception(), // currently in epoch convert to human readable ? + sig.sig_expiration().get(), // currently in epoch convert to human readable ? + sig.sig_inception().get(), // currently in epoch convert to human readable ? sig.key_tag(), sig.signer_name().to_string_with_options(&self.options), BASE64.encode(sig.sig()) @@ -840,11 +839,11 @@ impl DnsMessageParser { Ok((None, Some(rdata.anything().to_vec()))) } _ => Err(DnsMessageParserError::SimpleError { - cause: format!("Unsupported rdata {:?}", rdata), + cause: format!("Unsupported rdata {rdata:?}"), }), }, _ => Err(DnsMessageParserError::SimpleError { - cause: format!("Unsupported rdata {:?}", rdata), + cause: format!("Unsupported rdata {rdata:?}"), }), } } @@ -877,6 +876,25 @@ fn format_svcb_record(svcb: &SVCB, options: &DnsParserOptions) -> String { ) } +fn format_cdnskey_record(cdnskey: &CDNSKEY) -> String { + format!( + "{} 3 {} {}", + { + if cdnskey.revoke() { + 0b0000_0000_0000_0000 + } else if cdnskey.zone_key() && cdnskey.secure_entry_point() { + 0b0000_0001_0000_0001 + } else { + 0b0000_0001_0000_0000 + } + }, + cdnskey.algorithm().map_or(0, u8::from), + cdnskey + .public_key() + .map_or("".to_string(), |k| BASE64.encode(k.public_bytes())) + ) +} + fn format_dnskey_record(dnskey: &DNSKEY) -> String { format!( "{} 3 {} {}", @@ -890,7 +908,17 @@ fn format_dnskey_record(dnskey: &DNSKEY) -> String { } }, u8::from(dnskey.algorithm()), - BASE64.encode(dnskey.public_key()) + BASE64.encode(dnskey.public_key().public_bytes()) + ) +} + +fn format_cds_record(cds: &CDS) -> String { + format!( + "{} {} {} {}", + cds.key_tag(), + cds.algorithm().map_or(0, u8::from), + u8::from(cds.digest_type()), + HEXUPPER.encode(cds.digest()) ) } @@ -918,8 +946,8 @@ fn parse_response_code(rcode: u16) -> Option<&'static str> { 9 => Some("NotAuth"), // 9 NotAuth Server Not Authoritative for zone [RFC2136] 10 => Some("NotZone"), // 10 NotZone Name not contained in zone [RFC2136] // backwards compat for 4 bit ResponseCodes so far. - // 16 BADVERS Bad OPT Version [RFC6891] - 16 => Some("BADSIG"), // 16 BADSIG TSIG Signature Failure [RFC2845] + 16 => Some("BADVERS"), // 16 BADVERS Bad OPT Version [RFC6891] + // 16 BADSIG TSIG Signature Failure [RFC2845] 17 => Some("BADKEY"), // 17 BADKEY Key not recognized [RFC2845] 18 => Some("BADTIME"), // 18 BADTIME Signature out of time window [RFC2845] 19 => Some("BADMODE"), // 19 BADMODE Bad TKEY Mode [RFC2930] @@ -968,7 +996,7 @@ fn parse_edns(dns_message: &TrustDnsMessage) -> Option DnsParserResult<(Vec, Vec = edns .as_ref() .iter() - .filter(|(&code, _)| u16::from(code) != EDE_OPTION_CODE) + .filter(|(code, _)| u16::from(*code) != EDE_OPTION_CODE) .map(|(code, option)| match option { - EdnsOption::DAU(algorithms) - | EdnsOption::DHU(algorithms) - | EdnsOption::N3U(algorithms) => { - Ok(parse_edns_opt_dnssec_algorithms(*code, *algorithms)) - } + EdnsOption::DAU(algorithms) => Ok(parse_edns_opt_dnssec_algorithms(*code, *algorithms)), EdnsOption::Unknown(_, opt_data) => Ok(parse_edns_opt(*code, opt_data)), option => Vec::::try_from(option) .map(|bytes| parse_edns_opt(*code, &bytes)) @@ -1019,7 +1043,7 @@ fn parse_edns_opt_dnssec_algorithms( let algorithm_names: Vec = algorithms.iter().map(|alg| alg.to_string()).collect(); EdnsOptionEntry { opt_code: Into::::into(opt_code), - opt_name: format!("{:?}", opt_code), + opt_name: format!("{opt_code:?}"), opt_data: algorithm_names.join(" "), } } @@ -1027,7 +1051,7 @@ fn parse_edns_opt_dnssec_algorithms( fn parse_edns_opt(opt_code: EdnsCode, opt_data: &[u8]) -> EdnsOptionEntry { EdnsOptionEntry { opt_code: Into::::into(opt_code), - opt_name: format!("{:?}", opt_code), + opt_name: format!("{opt_code:?}"), opt_data: BASE64.encode(opt_data), } } @@ -1036,17 +1060,14 @@ fn parse_loc_rdata_size(data: u8) -> DnsParserResult { let base = (data & 0xF0) >> 4; if base > 9 { return Err(DnsMessageParserError::SimpleError { - cause: format!("The base shouldn't be greater than 9. Base: {}", base), + cause: format!("The base shouldn't be greater than 9. Base: {base}"), }); } let exponent = data & 0x0F; if exponent > 9 { return Err(DnsMessageParserError::SimpleError { - cause: format!( - "The exponent shouldn't be greater than 9. Exponent: {}", - exponent - ), + cause: format!("The exponent shouldn't be greater than 9. Exponent: {exponent}"), }); } @@ -1262,7 +1283,7 @@ fn parse_unknown_record_type(rtype: u16) -> Option { fn format_bytes_as_hex_string(bytes: &[u8]) -> String { bytes .iter() - .map(|e| format!("{:02X}", e)) + .map(|e| format!("{e:02X}")) .collect::>() .join(".") } @@ -1270,36 +1291,39 @@ fn format_bytes_as_hex_string(bytes: &[u8]) -> String { #[cfg(test)] mod tests { use std::{ - collections::HashMap, net::{Ipv4Addr, Ipv6Addr}, str::FromStr, }; #[allow(deprecated)] - use hickory_proto::rr::{ - dnssec::{ + use hickory_proto::dnssec::rdata::key::UpdateScope; + use hickory_proto::{ + dnssec::PublicKeyBuf, + rr::{ + domain::Name, rdata::{ - dnskey::DNSKEY, - ds::DS, - key::{KeyTrust, KeyUsage, Protocol, UpdateScope}, - nsec::NSEC, - nsec3::NSEC3, - nsec3param::NSEC3PARAM, - sig::SIG, - DNSSECRData, KEY, RRSIG, + CAA, CSYNC, HINFO, HTTPS, NAPTR, OPT, SSHFP, TLSA, TXT, + caa::KeyValue, + sshfp::{Algorithm, FingerprintType}, + svcb, + tlsa::{CertUsage, Matching, Selector}, }, + }, + }; + use hickory_proto::{ + dnssec::{ Algorithm as DNSSEC_Algorithm, DigestType, Nsec3HashAlgorithm, + rdata::{ + KEY, NSEC, NSEC3, NSEC3PARAM, RRSIG, SIG, + key::{KeyTrust, KeyUsage, Protocol}, + }, }, - domain::Name, - rdata::{ - caa::KeyValue, - sshfp::{Algorithm, FingerprintType}, - svcb, - tlsa::{CertUsage, Matching, Selector}, - CAA, CSYNC, HINFO, HTTPS, NAPTR, OPT, SSHFP, TLSA, TXT, + rr::rdata::{ + CERT, + cert::{Algorithm as CertAlgorithm, CertType}, }, + serialize::binary::Restrict, }; - use hickory_proto::serialize::binary::Restrict; use super::*; @@ -1366,8 +1390,7 @@ mod tests { #[test] fn test_parse_as_query_message_with_ede_with_extra_text() { - let raw_dns_message = - "szgAAAABAAAAAAABAmg1B2V4YW1wbGUDY29tAAAGAAEAACkE0AEBQAAAOQAPADUACW5vIFNFUCBtYXRjaGluZyB0aGUgRFMgZm91bmQgZm9yIGRuc3NlYy1mYWlsZWQub3JnLg=="; + let raw_dns_message = "szgAAAABAAAAAAABAmg1B2V4YW1wbGUDY29tAAAGAAEAACkE0AEBQAAAOQAPADUACW5vIFNFUCBtYXRjaGluZyB0aGUgRFMgZm91bmQgZm9yIGRuc3NlYy1mYWlsZWQub3JnLg=="; let raw_query_message = BASE64 .decode(raw_dns_message.as_bytes()) .expect("Invalid base64 encoded data."); @@ -1384,6 +1407,29 @@ mod tests { ); } + #[test] + fn test_parse_as_query_message_with_multiple_ede() { + let raw_dns_message = + "szgAAAABAAAAAAABAmg1B2V4YW1wbGUDY29tAAAGAAEAACkAAAEBQAAADAAPAAIAFQAPAAIAFA=="; + let raw_query_message = BASE64 + .decode(raw_dns_message.as_bytes()) + .expect("Invalid base64 encoded data."); + let parse_result = DnsMessageParser::new(raw_query_message).parse_as_query_message(); + assert!(parse_result.is_ok()); + let message = parse_result.expect("Message is not parsed."); + let opt_pseudo_section = message.opt_pseudo_section.expect("OPT section was missing"); + assert_eq!(opt_pseudo_section.ede.len(), 2); + assert_eq!(opt_pseudo_section.ede[0].info_code(), 21u16); + assert_eq!(opt_pseudo_section.ede[0].purpose(), Some("Not Supported")); + assert_eq!(opt_pseudo_section.ede[0].extra_text(), None); + assert_eq!(opt_pseudo_section.ede[1].info_code(), 20u16); + assert_eq!( + opt_pseudo_section.ede[1].purpose(), + Some("Not Authoritative") + ); + assert_eq!(opt_pseudo_section.ede[1].extra_text(), None); + } + #[test] fn test_parse_as_query_message_with_invalid_data() { let err = DnsMessageParser::new(vec![1, 2, 3]) @@ -1396,7 +1442,7 @@ mod tests { DnsMessageParserError::SimpleError { cause: e } => { panic!("Expected TrustDnsError, got {}.", &e) } - _ => panic!("{}.", err), + _ => panic!("{err}."), } } @@ -1485,9 +1531,25 @@ mod tests { let raw_update_message = BASE64 .decode(raw_dns_message.as_bytes()) .expect("Invalid base64 encoded data."); - assert!(DnsMessageParser::new(raw_update_message) - .parse_as_update_message() - .is_err()); + assert!( + DnsMessageParser::new(raw_update_message) + .parse_as_update_message() + .is_err() + ); + } + + #[test] + fn test_parse_bad_prefix_value() { + // this testcase have prefix value of 160, + let raw_dns_message = "oAAAMgABAAAAAAABAAABAAAAACYAAC8BAAAAAaAAAAAAAA=="; + let raw_query_message = BASE64 + .decode(raw_dns_message.as_bytes()) + .expect("Invalid base64 encoded data."); + assert!( + DnsMessageParser::new(raw_query_message) + .parse_as_query_message() + .is_err() + ); } #[test] @@ -1635,7 +1697,7 @@ mod tests { #[test] fn test_format_rdata_for_tlsa_type() { let rdata = RData::TLSA(TLSA::new( - CertUsage::Service, + CertUsage::PkixEe, Selector::Spki, Matching::Sha256, vec![1, 2, 3, 4, 5, 6, 7, 8], @@ -1711,8 +1773,7 @@ mod tests { true, true, false, - DNSSEC_Algorithm::RSASHA256, - vec![0, 1, 2, 3, 4, 5, 6, 7], + PublicKeyBuf::new(vec![0, 1, 2, 3, 4, 5, 6, 7], DNSSEC_Algorithm::RSASHA256), ))); let rdata_text1 = format_rdata(&rdata1); @@ -1720,8 +1781,7 @@ mod tests { true, false, false, - DNSSEC_Algorithm::RSASHA256, - vec![0, 1, 2, 3, 4, 5, 6, 7], + PublicKeyBuf::new(vec![0, 1, 2, 3, 4, 5, 6, 7], DNSSEC_Algorithm::RSASHA256), ))); let rdata_text2 = format_rdata(&rdata2); @@ -1729,8 +1789,7 @@ mod tests { true, true, true, - DNSSEC_Algorithm::RSASHA256, - vec![0, 1, 2, 3, 4, 5, 6, 7], + PublicKeyBuf::new(vec![0, 1, 2, 3, 4, 5, 6, 7], DNSSEC_Algorithm::RSASHA256), ))); let rdata_text3 = format_rdata(&rdata3); @@ -1987,11 +2046,10 @@ mod tests { #[test] fn test_format_rdata_for_opt_type() { - let mut options = HashMap::new(); - options.insert( + let options = vec![( EdnsCode::LLQ, EdnsOption::Unknown(u16::from(EdnsCode::LLQ), vec![0x01; 18]), - ); + )]; let rdata = RData::OPT(OPT::new(options)); let rdata_text = format_rdata(&rdata); assert!(rdata_text.is_ok()); @@ -2001,13 +2059,38 @@ mod tests { } } + #[test] + fn test_format_rdata_for_cert_type() { + let rdata = RData::CERT(CERT::new( + CertType::Experimental(65534), + 65535, + CertAlgorithm::RSASHA1, + BASE64 + .decode( + b"MxFcby9k/yvedMfQgKzhH5er0Mu/vILz4\ + 5IkskceFGgiWCn/GxHhai6VAuHAoNUz4YoU1tVfSCSqQYn6//11U6Nld80jEeC8aTrO+KKmCaY=", + ) + .unwrap(), + )); + let rdata_text = format_rdata(&rdata); + assert!(rdata_text.is_ok()); + if let Ok((parsed, raw_rdata)) = rdata_text { + assert!(raw_rdata.is_none()); + assert_eq!( + "65534 65535 RSASHA1 MxFcby9k/yvedMfQgKzhH5er0Mu/vILz4\ + 5IkskceFGgiWCn/GxHhai6VAuHAoNUz4YoU1tVfSCSqQYn6//11U6Nld80jEeC8aTrO+KKmCaY=", + parsed.unwrap() + ); + } + } + #[test] fn test_format_rdata_for_minfo_type() { test_format_rdata_with_compressed_domain_names( "5ZWBgAABAAEAAAABBm1pbmZvbwhleGFtcGxlMQNjb20AAA4AAcAMAA4AAQAADGsADQRmcmVkwBMDam9lwBMAACkQAAAAAAAAHAAKABgZ5zwJEK3VJQEAAABfSBqpS2bKf9CNBXg=", "BGZyZWTAEwNqb2XAEw==", 14, - "fred.example1.com. joe.example1.com." + "fred.example1.com. joe.example1.com.", ); } @@ -2132,17 +2215,6 @@ mod tests { ); } - #[test] - fn test_format_rdata_for_cert_type() { - test_format_rdata( - "//7//wUzEVxvL2T/K950x9CArOEfl6vQy7+8gvPjkiSyRx4UaCJYKf8bEeFq\ - LpUC4cCg1TPhihTW1V9IJKpBifr//XVTo2V3zSMR4LxpOs74oqYJpg==", - 37, - "65534 65535 RSASHA1 MxFcby9k/yvedMfQgKzhH5er0Mu/vILz4\ - 5IkskceFGgiWCn/GxHhai6VAuHAoNUz4YoU1tVfSCSqQYn6//11U6Nld80jEeC8aTrO+KKmCaY=", - ); - } - #[test] fn test_format_rdata_for_a6_type() { test_format_rdata( diff --git a/lib/dnsmsg-parser/src/ede.rs b/lib/dnsmsg-parser/src/ede.rs index a6c30f5f13..0c3ee6c2f1 100644 --- a/lib/dnsmsg-parser/src/ede.rs +++ b/lib/dnsmsg-parser/src/ede.rs @@ -1,5 +1,5 @@ use hickory_proto::{ - error::ProtoResult, + ProtoError, serialize::binary::{BinDecodable, BinDecoder, BinEncodable, BinEncoder}, }; @@ -67,7 +67,7 @@ impl EDE { } impl BinEncodable for EDE { - fn emit(&self, encoder: &mut BinEncoder<'_>) -> ProtoResult<()> { + fn emit(&self, encoder: &mut BinEncoder<'_>) -> Result<(), ProtoError> { encoder.emit_u16(self.info_code)?; if let Some(extra_text) = &self.extra_text { encoder.emit_vec(extra_text.as_bytes())?; @@ -77,7 +77,7 @@ impl BinEncodable for EDE { } impl<'a> BinDecodable<'a> for EDE { - fn read(decoder: &mut BinDecoder<'a>) -> ProtoResult { + fn read(decoder: &mut BinDecoder<'a>) -> Result { let info_code = decoder.read_u16()?.unverified(); let extra_text = if decoder.is_empty() { None diff --git a/lib/dnstap-parser/Cargo.toml b/lib/dnstap-parser/Cargo.toml index db86200cb1..cc5527f2e5 100644 --- a/lib/dnstap-parser/Cargo.toml +++ b/lib/dnstap-parser/Cargo.toml @@ -2,21 +2,22 @@ name = "dnstap-parser" version = "0.1.0" authors = ["Vector Contributors "] -edition = "2021" +edition = "2024" publish = false license = "MIT" [dependencies] -base64 = { version = "0.22.1", default-features = false } -bytes = { version = "1.9.0", default-features = false, features = ["serde"] } +base64.workspace = true +bytes = { workspace = true, features = ["serde"] } chrono.workspace = true dnsmsg-parser = { path = "../dnsmsg-parser" } hickory-proto.workspace = true prost.workspace = true snafu.workspace = true -tracing = { version = "0.1.34", default-features = false } +tracing.workspace = true vector-lib = { path = "../vector-lib" } vrl.workspace = true +paste.workspace = true [build-dependencies] prost-build.workspace = true diff --git a/lib/dnstap-parser/proto/dnstap.proto b/lib/dnstap-parser/proto/dnstap.proto index aa7229b9b5..9bb56d48d0 100644 --- a/lib/dnstap-parser/proto/dnstap.proto +++ b/lib/dnstap-parser/proto/dnstap.proto @@ -3,7 +3,7 @@ // This file contains the protobuf schemas for the "dnstap" structured event // replication format for DNS software. -// Written in 2013-2014 by Farsight Security, Inc. +// Written in 2013-2025 by the dnstap contributors. // // To the extent possible under law, the author(s) have dedicated all // copyright and related and neighboring rights to this file to the public @@ -12,7 +12,7 @@ // You should have received a copy of the CC0 Public Domain Dedication along // with this file. If not, see: // -// . +// . syntax = "proto2"; package dnstap; @@ -22,93 +22,102 @@ package dnstap; // of dnstap payload is defined. // See: https://developers.google.com/protocol-buffers/docs/techniques#union message Dnstap { - // DNS server identity. - // If enabled, this is the identity string of the DNS server which generated - // this message. Typically this would be the same string as returned by an - // "NSID" (RFC 5001) query. - optional bytes identity = 1; - - // DNS server version. - // If enabled, this is the version string of the DNS server which generated - // this message. Typically this would be the same string as returned by a - // "version.bind" query. - optional bytes version = 2; - - // Extra data for this payload. - // This field can be used for adding an arbitrary byte-string annotation to - // the payload. No encoding or interpretation is applied or enforced. - optional bytes extra = 3; - - // Identifies which field below is filled in. - enum Type { - MESSAGE = 1; - } - required Type type = 15; - - // One of the following will be filled in. - optional Message message = 14; + // DNS server identity. + // If enabled, this is the identity string of the DNS server which generated + // this message. Typically this would be the same string as returned by an + // "NSID" (RFC 5001) query. + optional bytes identity = 1; + + // DNS server version. + // If enabled, this is the version string of the DNS server which generated + // this message. Typically this would be the same string as returned by a + // "version.bind" query. + optional bytes version = 2; + + // Extra data for this payload. + // This field can be used for adding an arbitrary byte-string annotation to + // the payload. No encoding or interpretation is applied or enforced. + optional bytes extra = 3; + + // Identifies which field below is filled in. + enum Type { + MESSAGE = 1; + } + required Type type = 15; + + // One of the following will be filled in. + optional Message message = 14; } // SocketFamily: the network protocol family of a socket. This specifies how // to interpret "network address" fields. enum SocketFamily { - INET = 1; // IPv4 (RFC 791) - INET6 = 2; // IPv6 (RFC 2460) + INET = 1; // IPv4 (RFC 791) + INET6 = 2; // IPv6 (RFC 2460) } // SocketProtocol: the protocol used to transport a DNS message. enum SocketProtocol { - UDP = 1; // DNS over UDP transport (RFC 1035 section 4.2.1) - TCP = 2; // DNS over TCP transport (RFC 1035 section 4.2.2) - DOT = 3; // DNS over TLS (RFC 7858) - DOH = 4; // DNS over HTTPS (RFC 8484) - DNSCryptUDP = 5; // DNSCrypt over UDP (https://dnscrypt.info/protocol) - DNSCryptTCP = 6; // DNSCrypt over TCP (https://dnscrypt.info/protocol) + UDP = 1; // DNS over UDP transport (RFC 1035 section 4.2.1) + TCP = 2; // DNS over TCP transport (RFC 1035 section 4.2.2) + DOT = 3; // DNS over TLS (RFC 7858) + DOH = 4; // DNS over HTTPS (RFC 8484) + DNSCryptUDP = 5; // DNSCrypt over UDP (https://dnscrypt.info/protocol) + DNSCryptTCP = 6; // DNSCrypt over TCP (https://dnscrypt.info/protocol) + DOQ = 7; // DNS over QUIC (RFC 9250) +} + +// HttpProtocol: the HTTP protocol version used to transport a DNS message over +// an HTTP-based protocol such as DNS over HTTPS. +enum HttpProtocol { + HTTP1 = 1; // HTTP/1 + HTTP2 = 2; // HTTP/2 + HTTP3 = 3; // HTTP/3 } // Policy: information about any name server operator policy // applied to the processing of a DNS message. message Policy { - // Match: what aspect of the message or message exchange - // triggered the application of the Policy. - enum Match { - QNAME = 1; // Name in question section of query - CLIENT_IP = 2; // Client IP address - RESPONSE_IP = 3; // Address in A/AAAA RRSet - NS_NAME = 4; // Authoritative name server, by name - NS_IP = 5; // Authoritative name server, by IP address - } - - // The Action taken to implement the Policy. - enum Action { - NXDOMAIN = 1; // Respond with NXDOMAIN - NODATA = 2; // Respond with empty answer section - PASS = 3; // Do not alter the response (passthrough) - DROP = 4; // Do not respond. - TRUNCATE = 5; // Truncate UDP response, forcing TCP retry - LOCAL_DATA = 6; // Respond with local data from policy - } - - // type: the type of policy applied, e.g. "RPZ" for a - // policy from a Response Policy Zone. - optional string type = 1; - - // rule: the rule matched by the message. - // - // In a RPZ context, this is the owner name of the rule in - // the Response Policy Zone in wire format. - optional bytes rule = 2; - - // action: the policy action taken in response to the - // rule match. - optional Action action = 3; - - // match: the feature of the message exchange which matched the rule. - optional Match match = 4; - - // The matched value. Format depends on the matched feature . - optional bytes value = 5; + // Match: what aspect of the message or message exchange + // triggered the application of the Policy. + enum Match { + QNAME = 1; // Name in question section of query + CLIENT_IP = 2; // Client IP address + RESPONSE_IP = 3; // Address in A/AAAA RRSet + NS_NAME = 4; // Authoritative name server, by name + NS_IP = 5; // Authoritative name server, by IP address + } + + // The Action taken to implement the Policy. + enum Action { + NXDOMAIN = 1; // Respond with NXDOMAIN + NODATA = 2; // Respond with empty answer section + PASS = 3; // Do not alter the response (passthrough) + DROP = 4; // Do not respond. + TRUNCATE = 5; // Truncate UDP response, forcing TCP retry + LOCAL_DATA = 6; // Respond with local data from policy + } + + // type: the type of policy applied, e.g. "RPZ" for a + // policy from a Response Policy Zone. + optional string type = 1; + + // rule: the rule matched by the message. + // + // In a RPZ context, this is the owner name of the rule in + // the Reponse Policy Zone in wire format. + optional bytes rule = 2; + + // action: the policy action taken in response to the + // rule match. + optional Action action = 3; + + // match: the feature of the message exchange which matched the rule. + optional Match match = 4; + + // The matched value. Format depends on the matched feature . + optional bytes value = 5; } // Message: a wire-format (RFC 1035 section 4) DNS message and associated @@ -116,174 +125,178 @@ message Policy { // certain requirements based on the MessageType, see below. message Message { - // There are eight types of "Message" defined that correspond to the - // four arrows in the following diagram, slightly modified from RFC 1035 - // section 2: - - // +---------+ +----------+ +--------+ - // | | query | | query | | - // | Stub |-SQ--------CQ->| Recursive|-RQ----AQ->| Auth. | - // | Resolver| | Server | | Name | - // | |<-SR--------CR-| |<-RR----AR-| Server | - // +---------+ response | | response | | - // +----------+ +--------+ - - // Each arrow has two Type values each, one for each "end" of each arrow, - // because these are considered to be distinct events. Each end of each - // arrow on the diagram above has been marked with a two-letter Type - // mnemonic. Clockwise from upper left, these mnemonic values are: - // - // SQ: STUB_QUERY - // CQ: CLIENT_QUERY - // RQ: RESOLVER_QUERY - // AQ: AUTH_QUERY - // AR: AUTH_RESPONSE - // RR: RESOLVER_RESPONSE - // CR: CLIENT_RESPONSE - // SR: STUB_RESPONSE - - // Two additional types of "Message" have been defined for the - // "forwarding" case where an upstream DNS server is responsible for - // further recursion. These are not shown on the diagram above, but have - // the following mnemonic values: - - // FQ: FORWARDER_QUERY - // FR: FORWARDER_RESPONSE - - // The "Message" Type values are defined below. - - enum Type { - // AUTH_QUERY is a DNS query message received from a resolver by an - // authoritative name server, from the perspective of the authoritative - // name server. - AUTH_QUERY = 1; - - // AUTH_RESPONSE is a DNS response message sent from an authoritative - // name server to a resolver, from the perspective of the authoritative - // name server. - AUTH_RESPONSE = 2; - - // RESOLVER_QUERY is a DNS query message sent from a resolver to an - // authoritative name server, from the perspective of the resolver. - // Resolvers typically clear the RD (recursion desired) bit when - // sending queries. - RESOLVER_QUERY = 3; - - // RESOLVER_RESPONSE is a DNS response message received from an - // authoritative name server by a resolver, from the perspective of - // the resolver. - RESOLVER_RESPONSE = 4; - - // CLIENT_QUERY is a DNS query message sent from a client to a DNS - // server which is expected to perform further recursion, from the - // perspective of the DNS server. The client may be a stub resolver or - // forwarder or some other type of software which typically sets the RD - // (recursion desired) bit when querying the DNS server. The DNS server - // may be a simple forwarding proxy or it may be a full recursive - // resolver. - CLIENT_QUERY = 5; - - // CLIENT_RESPONSE is a DNS response message sent from a DNS server to - // a client, from the perspective of the DNS server. The DNS server - // typically sets the RA (recursion available) bit when responding. - CLIENT_RESPONSE = 6; - - // FORWARDER_QUERY is a DNS query message sent from a downstream DNS - // server to an upstream DNS server which is expected to perform - // further recursion, from the perspective of the downstream DNS - // server. - FORWARDER_QUERY = 7; - - // FORWARDER_RESPONSE is a DNS response message sent from an upstream - // DNS server performing recursion to a downstream DNS server, from the - // perspective of the downstream DNS server. - FORWARDER_RESPONSE = 8; - - // STUB_QUERY is a DNS query message sent from a stub resolver to a DNS - // server, from the perspective of the stub resolver. - STUB_QUERY = 9; - - // STUB_RESPONSE is a DNS response message sent from a DNS server to a - // stub resolver, from the perspective of the stub resolver. - STUB_RESPONSE = 10; - - // TOOL_QUERY is a DNS query message sent from a DNS software tool to a - // DNS server, from the perspective of the tool. - TOOL_QUERY = 11; - - // TOOL_RESPONSE is a DNS response message received by a DNS software - // tool from a DNS server, from the perspective of the tool. - TOOL_RESPONSE = 12; - - // UPDATE_QUERY is a Dynamic DNS Update request (RFC 2136) received - // by an authoritative name server, from the perspective of the - // authoritative name server. - UPDATE_QUERY = 13; - - // UPDATE_RESPONSE is a Dynamic DNS Update response (RFC 2136) sent - // from an authoritative name server, from the perspective of the - // authoritative name server. - UPDATE_RESPONSE = 14; - } - - // One of the Type values described above. - required Type type = 1; - - // One of the SocketFamily values described above. - optional SocketFamily socket_family = 2; - - // One of the SocketProtocol values described above. - optional SocketProtocol socket_protocol = 3; - - // The network address of the message initiator. - // For SocketFamily INET, this field is 4 octets (IPv4 address). - // For SocketFamily INET6, this field is 16 octets (IPv6 address). - optional bytes query_address = 4; - - // The network address of the message responder. - // For SocketFamily INET, this field is 4 octets (IPv4 address). - // For SocketFamily INET6, this field is 16 octets (IPv6 address). - optional bytes response_address = 5; - - // The transport port of the message initiator. - // This is a 16-bit UDP or TCP port number, depending on SocketProtocol. - optional uint32 query_port = 6; - - // The transport port of the message responder. - // This is a 16-bit UDP or TCP port number, depending on SocketProtocol. - optional uint32 response_port = 7; - - // The time at which the DNS query message was sent or received, depending - // on whether this is an AUTH_QUERY, RESOLVER_QUERY, or CLIENT_QUERY. - // This is the number of seconds since the UNIX epoch. - optional uint64 query_time_sec = 8; - - // The time at which the DNS query message was sent or received. - // This is the seconds fraction, expressed as a count of nanoseconds. - optional fixed32 query_time_nsec = 9; - - // The initiator's original wire-format DNS query message, verbatim. - optional bytes query_message = 10; - - // The "zone" or "bailiwick" pertaining to the DNS query message. - // This is a wire-format DNS domain name. - optional bytes query_zone = 11; - - // The time at which the DNS response message was sent or received, - // depending on whether this is an AUTH_RESPONSE, RESOLVER_RESPONSE, or - // CLIENT_RESPONSE. - // This is the number of seconds since the UNIX epoch. - optional uint64 response_time_sec = 12; - - // The time at which the DNS response message was sent or received. - // This is the seconds fraction, expressed as a count of nanoseconds. - optional fixed32 response_time_nsec = 13; - - // The responder's original wire-format DNS response message, verbatim. - optional bytes response_message = 14; - - // Operator policy applied to the processing of this message, if any. - optional Policy policy = 15; + // There are eight types of "Message" defined that correspond to the + // four arrows in the following diagram, slightly modified from RFC 1035 + // section 2: + + // +---------+ +----------+ +--------+ + // | | query | | query | | + // | Stub |-SQ--------CQ->| Recursive|-RQ----AQ->| Auth. | + // | Resolver| | Server | | Name | + // | |<-SR--------CR-| |<-RR----AR-| Server | + // +---------+ response | | response | | + // +----------+ +--------+ + + // Each arrow has two Type values each, one for each "end" of each arrow, + // because these are considered to be distinct events. Each end of each + // arrow on the diagram above has been marked with a two-letter Type + // mnemonic. Clockwise from upper left, these mnemonic values are: + // + // SQ: STUB_QUERY + // CQ: CLIENT_QUERY + // RQ: RESOLVER_QUERY + // AQ: AUTH_QUERY + // AR: AUTH_RESPONSE + // RR: RESOLVER_RESPONSE + // CR: CLIENT_RESPONSE + // SR: STUB_RESPONSE + + // Two additional types of "Message" have been defined for the + // "forwarding" case where an upstream DNS server is responsible for + // further recursion. These are not shown on the diagram above, but have + // the following mnemonic values: + + // FQ: FORWARDER_QUERY + // FR: FORWARDER_RESPONSE + + // The "Message" Type values are defined below. + + enum Type { + // AUTH_QUERY is a DNS query message received from a resolver by an + // authoritative name server, from the perspective of the authoritative + // name server. + AUTH_QUERY = 1; + + // AUTH_RESPONSE is a DNS response message sent from an authoritative + // name server to a resolver, from the perspective of the authoritative + // name server. + AUTH_RESPONSE = 2; + + // RESOLVER_QUERY is a DNS query message sent from a resolver to an + // authoritative name server, from the perspective of the resolver. + // Resolvers typically clear the RD (recursion desired) bit when + // sending queries. + RESOLVER_QUERY = 3; + + // RESOLVER_RESPONSE is a DNS response message received from an + // authoritative name server by a resolver, from the perspective of + // the resolver. + RESOLVER_RESPONSE = 4; + + // CLIENT_QUERY is a DNS query message sent from a client to a DNS + // server which is expected to perform further recursion, from the + // perspective of the DNS server. The client may be a stub resolver or + // forwarder or some other type of software which typically sets the RD + // (recursion desired) bit when querying the DNS server. The DNS server + // may be a simple forwarding proxy or it may be a full recursive + // resolver. + CLIENT_QUERY = 5; + + // CLIENT_RESPONSE is a DNS response message sent from a DNS server to + // a client, from the perspective of the DNS server. The DNS server + // typically sets the RA (recursion available) bit when responding. + CLIENT_RESPONSE = 6; + + // FORWARDER_QUERY is a DNS query message sent from a downstream DNS + // server to an upstream DNS server which is expected to perform + // further recursion, from the perspective of the downstream DNS + // server. + FORWARDER_QUERY = 7; + + // FORWARDER_RESPONSE is a DNS response message sent from an upstream + // DNS server performing recursion to a downstream DNS server, from the + // perspective of the downstream DNS server. + FORWARDER_RESPONSE = 8; + + // STUB_QUERY is a DNS query message sent from a stub resolver to a DNS + // server, from the perspective of the stub resolver. + STUB_QUERY = 9; + + // STUB_RESPONSE is a DNS response message sent from a DNS server to a + // stub resolver, from the perspective of the stub resolver. + STUB_RESPONSE = 10; + + // TOOL_QUERY is a DNS query message sent from a DNS software tool to a + // DNS server, from the perspective of the tool. + TOOL_QUERY = 11; + + // TOOL_RESPONSE is a DNS response message received by a DNS software + // tool from a DNS server, from the perspective of the tool. + TOOL_RESPONSE = 12; + + // UPDATE_QUERY is a Dynamic DNS Update request (RFC 2136) received + // by an authoritative name server, from the perspective of the + // authoritative name server. + UPDATE_QUERY = 13; + + // UPDATE_RESPONSE is a Dynamic DNS Update response (RFC 2136) sent + // from an authoritative name server, from the perspective of the + // authoritative name server. + UPDATE_RESPONSE = 14; + } + + // One of the Type values described above. + required Type type = 1; + + // One of the SocketFamily values described above. + optional SocketFamily socket_family = 2; + + // One of the SocketProtocol values described above. + optional SocketProtocol socket_protocol = 3; + + // The network address of the message initiator. + // For SocketFamily INET, this field is 4 octets (IPv4 address). + // For SocketFamily INET6, this field is 16 octets (IPv6 address). + optional bytes query_address = 4; + + // The network address of the message responder. + // For SocketFamily INET, this field is 4 octets (IPv4 address). + // For SocketFamily INET6, this field is 16 octets (IPv6 address). + optional bytes response_address = 5; + + // The transport port of the message initiator. + // This is a 16-bit UDP or TCP port number, depending on SocketProtocol. + optional uint32 query_port = 6; + + // The transport port of the message responder. + // This is a 16-bit UDP or TCP port number, depending on SocketProtocol. + optional uint32 response_port = 7; + + // The time at which the DNS query message was sent or received, depending + // on whether this is an AUTH_QUERY, RESOLVER_QUERY, or CLIENT_QUERY. + // This is the number of seconds since the UNIX epoch. + optional uint64 query_time_sec = 8; + + // The time at which the DNS query message was sent or received. + // This is the seconds fraction, expressed as a count of nanoseconds. + optional fixed32 query_time_nsec = 9; + + // The initiator's original wire-format DNS query message, verbatim. + optional bytes query_message = 10; + + // The "zone" or "bailiwick" pertaining to the DNS query message. + // This is a wire-format DNS domain name. + optional bytes query_zone = 11; + + // The time at which the DNS response message was sent or received, + // depending on whether this is an AUTH_RESPONSE, RESOLVER_RESPONSE, or + // CLIENT_RESPONSE. + // This is the number of seconds since the UNIX epoch. + optional uint64 response_time_sec = 12; + + // The time at which the DNS response message was sent or received. + // This is the seconds fraction, expressed as a count of nanoseconds. + optional fixed32 response_time_nsec = 13; + + // The responder's original wire-format DNS response message, verbatim. + optional bytes response_message = 14; + + // Operator policy applied to the processing of this message, if any. + optional Policy policy = 15; + + // One of the HttpProtocol values described above. This field should only be + // set if socket_protocol is set to DOH. + optional HttpProtocol http_protocol = 16; } // All fields except for 'type' in the Message schema are optional. diff --git a/lib/dnstap-parser/src/parser.rs b/lib/dnstap-parser/src/parser.rs index 6a950c1a15..a11e18434e 100644 --- a/lib/dnstap-parser/src/parser.rs +++ b/lib/dnstap-parser/src/parser.rs @@ -7,7 +7,7 @@ use std::{ }; use vector_lib::emit; -use base64::prelude::{Engine as _, BASE64_STANDARD}; +use base64::prelude::{BASE64_STANDARD, Engine as _}; use bytes::Bytes; use chrono::{TimeZone, Utc}; use dnsmsg_parser::{dns_message_parser::DnsParserOptions, ede::EDE}; @@ -20,8 +20,8 @@ use snafu::Snafu; use vrl::{owned_value_path, path}; use vector_lib::{ - event::{LogEvent, Value}, Error, Result, + event::{LogEvent, Value}, }; #[allow(warnings, clippy::all, clippy::pedantic, clippy::nursery)] @@ -31,12 +31,12 @@ mod dnstap_proto { use crate::{internal_events::DnstapParseWarning, schema::DNSTAP_VALUE_PATHS}; use dnstap_proto::{ - message::Type as DnstapMessageType, Dnstap, Message as DnstapMessage, SocketFamily, - SocketProtocol, + Dnstap, Message as DnstapMessage, SocketFamily, SocketProtocol, + message::Type as DnstapMessageType, }; use vector_lib::config::log_schema; -use vector_lib::lookup::lookup_v2::ValuePath; use vector_lib::lookup::PathPrefix; +use vector_lib::lookup::lookup_v2::ValuePath; use dnsmsg_parser::{ dns_message::{ @@ -147,25 +147,18 @@ impl DnstapParser { dnstap_data_type.clone(), ); - if dnstap_data_type == "Message" { - if let Some(message) = proto_msg.message { - if let Err(err) = - DnstapParser::parse_dnstap_message(event, &root, message, parsing_options) - { - emit!(DnstapParseWarning { error: &err }); - need_raw_data = true; - DnstapParser::insert( - event, - &root, - &DNSTAP_VALUE_PATHS.error, - err.to_string(), - ); - } - } + if dnstap_data_type == "Message" + && let Some(message) = proto_msg.message + && let Err(err) = + DnstapParser::parse_dnstap_message(event, &root, message, parsing_options) + { + emit!(DnstapParseWarning { error: &err }); + need_raw_data = true; + DnstapParser::insert(event, &root, &DNSTAP_VALUE_PATHS.error, err.to_string()); } } else { emit!(DnstapParseWarning { - error: format!("Unknown dnstap data type: {}", dnstap_data_type_id) + error: format!("Unknown dnstap data type: {dnstap_data_type_id}") }); need_raw_data = true; } @@ -236,7 +229,7 @@ impl DnstapParser { dnstap_message_type_id, dnstap_message.query_message.as_ref(), &DNSTAP_MESSAGE_REQUEST_TYPE_IDS, - ); + )?; } if let Some(response_time_sec) = dnstap_message.response_time_sec { @@ -248,7 +241,7 @@ impl DnstapParser { dnstap_message_type_id, dnstap_message.response_message.as_ref(), &DNSTAP_MESSAGE_RESPONSE_TYPE_IDS, - ); + )?; } DnstapParser::parse_dnstap_message_type( @@ -366,19 +359,37 @@ impl DnstapParser { dnstap_message_type_id: i32, message: Option<&Vec>, type_ids: &HashSet, - ) { + ) -> Result<()> { + if time_sec > i64::MAX as u64 { + return Err(Error::from("Cannot parse timestamp")); + } + let (time_in_nanosec, query_time_nsec) = match time_nsec { - Some(nsec) => (time_sec as i64 * 1_000_000_000_i64 + nsec as i64, nsec), - None => (time_sec as i64 * 1_000_000_000_i64, 0), + Some(nsec) => { + if let Some(time_in_ns) = (time_sec as i64) + .checked_mul(1_000_000_000) + .and_then(|v| v.checked_add(nsec as i64)) + { + (time_in_ns, nsec) + } else { + return Err(Error::from("Cannot parse timestamp")); + } + } + None => { + if let Some(time_in_ns) = (time_sec as i64).checked_mul(1_000_000_000) { + (time_in_ns, 0) + } else { + return Err(Error::from("Cannot parse timestamp")); + } + } }; if type_ids.contains(&dnstap_message_type_id) { DnstapParser::log_time(event, prefix.clone(), time_in_nanosec, "ns"); - let timestamp = Utc .timestamp_opt(time_sec.try_into().unwrap(), query_time_nsec) .single() - .expect("invalid timestamp"); + .ok_or("Invalid timestamp")?; if let Some(timestamp_key) = log_schema().timestamp_key() { DnstapParser::insert(event, prefix.clone(), timestamp_key, timestamp); } @@ -392,6 +403,7 @@ impl DnstapParser { "ns", ); } + Ok(()) } fn parse_dnstap_message_socket_family<'a>( @@ -418,9 +430,15 @@ impl DnstapParser { if let Some(query_address) = dnstap_message.query_address.as_ref() { let source_address = if socket_family == 1 { + if query_address.len() < 4 { + return Err(Error::from("Cannot parse query_address")); + } let address_buffer: [u8; 4] = query_address[0..4].try_into()?; IpAddr::V4(Ipv4Addr::from(address_buffer)) } else { + if query_address.len() < 16 { + return Err(Error::from("Cannot parse query_address")); + } let address_buffer: [u8; 16] = query_address[0..16].try_into()?; IpAddr::V6(Ipv6Addr::from(address_buffer)) }; @@ -444,9 +462,15 @@ impl DnstapParser { if let Some(response_address) = dnstap_message.response_address.as_ref() { let response_addr = if socket_family == 1 { + if response_address.len() < 4 { + return Err(Error::from("Cannot parse response_address")); + } let address_buffer: [u8; 4] = response_address[0..4].try_into()?; IpAddr::V4(Ipv4Addr::from(address_buffer)) } else { + if response_address.len() < 16 { + return Err(Error::from("Cannot parse response_address")); + } let address_buffer: [u8; 16] = response_address[0..16].try_into()?; IpAddr::V6(Ipv6Addr::from(address_buffer)) }; @@ -959,8 +983,7 @@ fn to_socket_family_name(socket_family: i32) -> Result<&'static str> { Ok("INET6") } else { Err(Error::from(format!( - "Unknown socket family: {}", - socket_family + "Unknown socket family: {socket_family}" ))) } } @@ -980,8 +1003,7 @@ fn to_socket_protocol_name(socket_protocol: i32) -> Result<&'static str> { Ok("DNSCryptTCP") } else { Err(Error::from(format!( - "Unknown socket protocol: {}", - socket_protocol + "Unknown socket protocol: {socket_protocol}" ))) } } @@ -1009,7 +1031,7 @@ fn to_dnstap_message_type(type_id: i32) -> String { 12 => String::from("ToolResponse"), 13 => String::from("UpdateQuery"), 14 => String::from("UpdateResponse"), - _ => format!("Unknown dnstap message type: {}", type_id), + _ => format!("Unknown dnstap message type: {type_id}"), } } @@ -1018,7 +1040,7 @@ mod tests { use super::*; use chrono::DateTime; use dnsmsg_parser::dns_message_parser::DnsParserOptions; - use std::collections::BTreeMap; + use std::{collections::BTreeMap, vec}; #[test] fn test_parse_dnstap_data_with_query_message() { @@ -1321,6 +1343,72 @@ mod tests { assert!(e.to_string().contains("Protobuf message")); } + #[test] + fn test_parse_dnstap_data_with_invalid_timestamp() { + fn test_one_timestamp_parse(time_sec: u64, time_nsec: Option) -> Result<()> { + let mut event = LogEvent::default(); + let root = owned_value_path!(); + let type_ids = HashSet::from([1]); + DnstapParser::parse_dnstap_message_time( + &mut event, &root, time_sec, time_nsec, 1, None, &type_ids, + ) + } + // okay case + assert!(test_one_timestamp_parse(1337, Some(42)).is_ok()); + // overflow in cast (u64 -> i64) + assert!(test_one_timestamp_parse(u64::MAX, Some(42)).is_err()); + assert!(test_one_timestamp_parse(u64::MAX, None).is_err()); + // overflow in multiplication + assert!(test_one_timestamp_parse(i64::MAX as u64, Some(42)).is_err()); + assert!(test_one_timestamp_parse(i64::MAX as u64, None).is_err()); + // overflow in add + assert!( + test_one_timestamp_parse((i64::MAX / 1_000_000_000) as u64, Some(u32::MAX)).is_err() + ); + // cannot be parsed by timestamp_opt + assert!(test_one_timestamp_parse(96, Some(1616928816)).is_err()); + } + + #[test] + fn test_parse_dnstap_message_socket_family_bad_addr() { + // while parsing address is optional, but in this function assume otherwise + fn test_one_input(socket_family: i32, msg: DnstapMessage) -> Result<()> { + let mut event = LogEvent::default(); + let root = owned_value_path!(); + DnstapParser::parse_dnstap_message_socket_family(&mut event, &root, socket_family, &msg) + } + // all bad cases which can panic + { + let message = DnstapMessage { + query_address: Some(vec![]), + ..Default::default() + }; + assert!(test_one_input(1, message).is_err()); + } + { + let message = DnstapMessage { + query_address: Some(vec![]), + ..Default::default() + }; + assert!(test_one_input(2, message).is_err()); + } + + { + let message = DnstapMessage { + response_address: Some(vec![]), + ..Default::default() + }; + assert!(test_one_input(1, message).is_err()); + } + { + let message = DnstapMessage { + response_address: Some(vec![]), + ..Default::default() + }; + assert!(test_one_input(2, message).is_err()); + } + } + #[test] fn test_get_socket_family_name() { assert_eq!("INET", to_socket_family_name(1).unwrap()); diff --git a/lib/dnstap-parser/src/schema.rs b/lib/dnstap-parser/src/schema.rs index 3f90977fd5..02fb365855 100644 --- a/lib/dnstap-parser/src/schema.rs +++ b/lib/dnstap-parser/src/schema.rs @@ -1,10 +1,10 @@ use std::{collections::BTreeMap, sync::LazyLock}; -use vector_lib::lookup::{owned_value_path, OwnedValuePath}; +use vector_lib::lookup::{OwnedValuePath, owned_value_path}; use vrl::btreemap; use vrl::value::{ - kind::{Collection, Field}, Kind, + kind::{Collection, Field}, }; #[derive(Debug, Default, Clone)] diff --git a/lib/dnstap-parser/src/vrl_functions/parse_dnstap.rs b/lib/dnstap-parser/src/vrl_functions/parse_dnstap.rs index ee6c5d52bc..092ac46ff8 100644 --- a/lib/dnstap-parser/src/vrl_functions/parse_dnstap.rs +++ b/lib/dnstap-parser/src/vrl_functions/parse_dnstap.rs @@ -1,6 +1,6 @@ use crate::parser::DnstapParser; use crate::schema::DnstapEventSchema; -use base64::prelude::{Engine as _, BASE64_STANDARD}; +use base64::prelude::{BASE64_STANDARD, Engine as _}; use dnsmsg_parser::dns_message_parser::DnsParserOptions; use vector_lib::event::LogEvent; use vrl::prelude::*; @@ -120,7 +120,7 @@ impl Function for ParseDnstap { "questionTypeId": 6 } ], - "rcodeName": "BADSIG" + "rcodeName": "BADVERS" }, "responseAddress": "2001:502:7094::30", "responsePort": 53, @@ -295,7 +295,7 @@ mod tests { questionTypeId: 6, } ], - rcodeName: "BADSIG", + rcodeName: "BADVERS", }, responseAddress: "2001:502:7094::30", responsePort: 53, diff --git a/lib/docs-renderer/Cargo.toml b/lib/docs-renderer/Cargo.toml index b871ac10cb..3f25b0600c 100644 --- a/lib/docs-renderer/Cargo.toml +++ b/lib/docs-renderer/Cargo.toml @@ -2,14 +2,14 @@ name = "docs-renderer" version = "0.1.0" authors = ["Vector Contributors "] -edition = "2021" +edition = "2024" publish = false [dependencies] -anyhow = { version = "1.0.95", default-features = false, features = ["std"] } +anyhow.workspace = true serde.workspace = true serde_json.workspace = true snafu.workspace = true -tracing = { version = "0.1.34", default-features = false } +tracing.workspace = true vector-config = { path = "../vector-config" } vector-config-common = { path = "../vector-config-common" } diff --git a/lib/docs-renderer/src/main.rs b/lib/docs-renderer/src/main.rs index dfd9c829d8..ac7e813282 100644 --- a/lib/docs-renderer/src/main.rs +++ b/lib/docs-renderer/src/main.rs @@ -68,8 +68,7 @@ fn main() -> Result<()> { let component_name = component_schema.component_name().to_string(); let component_schema_renderer = SchemaRenderer::new(&querier, component_schema); let rendered_component_schema = component_schema_renderer.render().context(format!( - "Failed to render the '{}' component schema.", - component_name + "Failed to render the '{component_name}' component schema." ))?; rendered_component_schemas.insert( format!("{}s/base/{}", base_component_type.as_str(), component_name), diff --git a/lib/docs-renderer/src/renderer.rs b/lib/docs-renderer/src/renderer.rs index 49065c761a..43c00dbae4 100644 --- a/lib/docs-renderer/src/renderer.rs +++ b/lib/docs-renderer/src/renderer.rs @@ -79,7 +79,7 @@ impl RenderData { while let Some(segment) = segments.pop_front() { if destination.contains_key(segment) { match destination.get_mut(segment) { - Some(Value::Object(ref mut next)) => { + Some(Value::Object(next)) => { destination = next; continue; } @@ -91,7 +91,7 @@ impl RenderData { } else { destination.insert(segment.to_string(), Value::Object(Map::new())); match destination.get_mut(segment) { - Some(Value::Object(ref mut next)) => { + Some(Value::Object(next)) => { destination = next; } _ => panic!("New object was just inserted."), @@ -139,7 +139,7 @@ impl RenderData { let mut destination = map; while let Some(segment) = segments.pop_front() { match destination.get_mut(segment) { - Some(Value::Object(ref mut next)) => { + Some(Value::Object(next)) => { destination = next; continue; } @@ -300,7 +300,7 @@ fn render_bare_schema( // All we need to do is figure out the rendered type for the constant value, so we can // generate the right type path and stick the constant value in it. let rendered_const_type = get_rendered_value_type(&schema, const_value)?; - let const_type_path = format!("/type/{}/const", rendered_const_type); + let const_type_path = format!("/type/{rendered_const_type}/const"); data.write(const_type_path.as_str(), const_value.clone()); } SchemaType::Enum(enum_values) => { @@ -417,7 +417,7 @@ fn render_schema_description(schema: T) -> Result Ok(None), (None, Some(description)) => Ok(Some(description.trim().to_string())), (Some(title), Some(description)) => { - let concatenated = format!("{}\n\n{}", title, description); + let concatenated = format!("{title}\n\n{description}"); Ok(Some(concatenated.trim().to_string())) } } diff --git a/lib/enrichment/Cargo.toml b/lib/enrichment/Cargo.toml index 923bd375ec..9e82fd16e0 100644 --- a/lib/enrichment/Cargo.toml +++ b/lib/enrichment/Cargo.toml @@ -2,11 +2,11 @@ name = "enrichment" version = "0.1.0" authors = ["Vector Contributors "] -edition = "2021" +edition = "2024" publish = false [dependencies] arc-swap = { version = "1.7.1", default-features = false } chrono.workspace = true -dyn-clone = { version = "1.0.17", default-features = false } +dyn-clone = { version = "1.0.20", default-features = false } vrl.workspace = true diff --git a/lib/enrichment/src/find_enrichment_table_records.rs b/lib/enrichment/src/find_enrichment_table_records.rs index f2ec4313b6..18f5480fd5 100644 --- a/lib/enrichment/src/find_enrichment_table_records.rs +++ b/lib/enrichment/src/find_enrichment_table_records.rs @@ -3,8 +3,8 @@ use vrl::prelude::*; use crate::vrl_util::is_case_sensitive; use crate::{ - vrl_util::{self, add_index, evaluate_condition}, Case, Condition, IndexHandle, TableRegistry, TableSearch, + vrl_util::{self, add_index, evaluate_condition}, }; fn find_enrichment_table_records( @@ -12,6 +12,7 @@ fn find_enrichment_table_records( enrichment_tables: &TableSearch, table: &str, case_sensitive: Case, + wildcard: Option, condition: &[Condition], index: Option, ) -> Resolved { @@ -34,6 +35,7 @@ fn find_enrichment_table_records( case_sensitive, condition, select.as_ref().map(|select| select.as_ref()), + wildcard.as_ref(), index, )? .into_iter() @@ -71,6 +73,11 @@ impl Function for FindEnrichmentTableRecords { kind: kind::BOOLEAN, required: false, }, + Parameter { + keyword: "wildcard", + kind: kind::BYTES, + required: false, + }, ] } @@ -112,6 +119,7 @@ impl Function for FindEnrichmentTableRecords { let select = arguments.optional("select"); let case_sensitive = is_case_sensitive(&arguments, state)?; + let wildcard = arguments.optional("wildcard"); let index = Some( add_index(registry, &table, case_sensitive, &condition) .map_err(|err| Box::new(err) as Box<_>)?, @@ -123,6 +131,7 @@ impl Function for FindEnrichmentTableRecords { index, select, case_sensitive, + wildcard, enrichment_tables: registry.as_readonly(), } .as_expr()) @@ -136,6 +145,7 @@ pub struct FindEnrichmentTableRecordsFn { index: Option, select: Option>, case_sensitive: Case, + wildcard: Option>, enrichment_tables: TableSearch, } @@ -158,6 +168,11 @@ impl FunctionExpression for FindEnrichmentTableRecordsFn { let table = &self.table; let case_sensitive = self.case_sensitive; + let wildcard = self + .wildcard + .as_ref() + .map(|array| array.resolve(ctx)) + .transpose()?; let index = self.index; let enrichment_tables = &self.enrichment_tables; @@ -166,6 +181,7 @@ impl FunctionExpression for FindEnrichmentTableRecordsFn { enrichment_tables, table, case_sensitive, + wildcard, &condition, index, ) @@ -178,9 +194,9 @@ impl FunctionExpression for FindEnrichmentTableRecordsFn { #[cfg(test)] mod tests { - use vrl::compiler::state::RuntimeState; use vrl::compiler::TargetValue; use vrl::compiler::TimeZone; + use vrl::compiler::state::RuntimeState; use vrl::value; use vrl::value::Secrets; @@ -199,6 +215,7 @@ mod tests { index: Some(IndexHandle(999)), select: None, case_sensitive: Case::Sensitive, + wildcard: None, enrichment_tables: registry.as_readonly(), }; diff --git a/lib/enrichment/src/get_enrichment_table_record.rs b/lib/enrichment/src/get_enrichment_table_record.rs index ddd99c99c8..f605d9256d 100644 --- a/lib/enrichment/src/get_enrichment_table_record.rs +++ b/lib/enrichment/src/get_enrichment_table_record.rs @@ -3,8 +3,8 @@ use vrl::prelude::*; use crate::vrl_util::is_case_sensitive; use crate::{ - vrl_util::{self, add_index, evaluate_condition}, Case, Condition, IndexHandle, TableRegistry, TableSearch, + vrl_util::{self, add_index, evaluate_condition}, }; fn get_enrichment_table_record( @@ -12,6 +12,7 @@ fn get_enrichment_table_record( enrichment_tables: &TableSearch, table: &str, case_sensitive: Case, + wildcard: Option, condition: &[Condition], index: Option, ) -> Resolved { @@ -27,11 +28,13 @@ fn get_enrichment_table_record( }), }) .transpose()?; + let data = enrichment_tables.find_table_row( table, case_sensitive, condition, select.as_ref().map(|select| select.as_ref()), + wildcard.as_ref(), index, )?; @@ -67,6 +70,11 @@ impl Function for GetEnrichmentTableRecord { kind: kind::BOOLEAN, required: false, }, + Parameter { + keyword: "wildcard", + kind: kind::BYTES, + required: false, + }, ] } @@ -104,6 +112,7 @@ impl Function for GetEnrichmentTableRecord { let select = arguments.optional("select"); let case_sensitive = is_case_sensitive(&arguments, state)?; + let wildcard = arguments.optional("wildcard"); let index = Some( add_index(registry, &table, case_sensitive, &condition) .map_err(|err| Box::new(err) as Box<_>)?, @@ -115,6 +124,7 @@ impl Function for GetEnrichmentTableRecord { index, select, case_sensitive, + wildcard, enrichment_tables: registry.as_readonly(), } .as_expr()) @@ -127,6 +137,7 @@ pub struct GetEnrichmentTableRecordFn { condition: BTreeMap, index: Option, select: Option>, + wildcard: Option>, case_sensitive: Case, enrichment_tables: TableSearch, } @@ -150,6 +161,11 @@ impl FunctionExpression for GetEnrichmentTableRecordFn { let table = &self.table; let case_sensitive = self.case_sensitive; + let wildcard = self + .wildcard + .as_ref() + .map(|array| array.resolve(ctx)) + .transpose()?; let index = self.index; let enrichment_tables = &self.enrichment_tables; @@ -158,6 +174,7 @@ impl FunctionExpression for GetEnrichmentTableRecordFn { enrichment_tables, table, case_sensitive, + wildcard, &condition, index, ) @@ -170,9 +187,9 @@ impl FunctionExpression for GetEnrichmentTableRecordFn { #[cfg(test)] mod tests { + use vrl::compiler::TargetValue; use vrl::compiler::prelude::TimeZone; use vrl::compiler::state::RuntimeState; - use vrl::compiler::TargetValue; use vrl::value; use vrl::value::Secrets; @@ -191,6 +208,7 @@ mod tests { index: Some(IndexHandle(999)), select: None, case_sensitive: Case::Sensitive, + wildcard: None, enrichment_tables: registry.as_readonly(), }; diff --git a/lib/enrichment/src/lib.rs b/lib/enrichment/src/lib.rs index 52f2a547f5..ce02e5f8f8 100644 --- a/lib/enrichment/src/lib.rs +++ b/lib/enrichment/src/lib.rs @@ -26,6 +26,16 @@ pub enum Condition<'a> { from: chrono::DateTime, to: chrono::DateTime, }, + /// The date in the field is greater than or equal to `from`. + FromDate { + field: &'a str, + from: chrono::DateTime, + }, + /// The date in the field is less than or equal to `to`. + ToDate { + field: &'a str, + to: chrono::DateTime, + }, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -47,6 +57,7 @@ pub trait Table: DynClone { case: Case, condition: &'a [Condition<'a>], select: Option<&[String]>, + wildcard: Option<&Value>, index: Option, ) -> Result; @@ -58,6 +69,7 @@ pub trait Table: DynClone { case: Case, condition: &'a [Condition<'a>], select: Option<&[String]>, + wildcard: Option<&Value>, index: Option, ) -> Result, String>; diff --git a/lib/enrichment/src/tables.rs b/lib/enrichment/src/tables.rs index 522d77c1c2..d4ccb4b0f9 100644 --- a/lib/enrichment/src/tables.rs +++ b/lib/enrichment/src/tables.rs @@ -35,7 +35,7 @@ use std::{ }; use arc_swap::ArcSwap; -use vrl::value::ObjectMap; +use vrl::value::{ObjectMap, Value}; use super::{Condition, IndexHandle, Table}; use crate::Case; @@ -134,7 +134,7 @@ impl TableRegistry { pub fn table_ids(&self) -> Vec { let locked = self.loading.lock().unwrap(); match *locked { - Some(ref tables) => tables.iter().map(|(key, _)| key.clone()).collect(), + Some(ref tables) => tables.keys().cloned().collect(), None => Vec::new(), } } @@ -157,7 +157,7 @@ impl TableRegistry { match *locked { None => Err("finish_load has been called".to_string()), Some(ref mut tables) => match tables.get_mut(table) { - None => Err(format!("table '{}' not loaded", table)), + None => Err(format!("table '{table}' not loaded")), Some(table) => table.add_index(case, fields), }, } @@ -216,13 +216,14 @@ impl TableSearch { case: Case, condition: &'a [Condition<'a>], select: Option<&[String]>, + wildcard: Option<&Value>, index: Option, ) -> Result { let tables = self.0.load(); if let Some(ref tables) = **tables { match tables.get(table) { - None => Err(format!("table {} not loaded", table)), - Some(table) => table.find_table_row(case, condition, select, index), + None => Err(format!("table {table} not loaded")), + Some(table) => table.find_table_row(case, condition, select, wildcard, index), } } else { Err("finish_load not called".to_string()) @@ -238,13 +239,14 @@ impl TableSearch { case: Case, condition: &'a [Condition<'a>], select: Option<&[String]>, + wildcard: Option<&Value>, index: Option, ) -> Result, String> { let tables = self.0.load(); if let Some(ref tables) = **tables { match tables.get(table) { - None => Err(format!("table {} not loaded", table)), - Some(table) => table.find_table_rows(case, condition, select, index), + None => Err(format!("table {table} not loaded")), + Some(table) => table.find_table_rows(case, condition, select, wildcard, index), } } else { Err("finish_load not called".to_string()) @@ -276,9 +278,9 @@ fn fmt_enrichment_table( tables.truncate(std::cmp::max(tables.len(), 0)); tables.push(')'); - write!(f, "{} {}", name, tables) + write!(f, "{name} {tables}") } - None => write!(f, "{} loading", name), + None => write!(f, "{name} loading"), } } @@ -337,6 +339,7 @@ mod tests { value: Value::from("thang"), }], None, + None, None ) ); @@ -378,6 +381,7 @@ mod tests { value: Value::from("thang"), }], None, + None, None ) ); @@ -442,7 +446,7 @@ mod tests { tables .get("dummy1") .unwrap() - .find_table_row(Case::Sensitive, &Vec::new(), None, None) + .find_table_row(Case::Sensitive, &Vec::new(), None, None, None) .unwrap() .get("field") .cloned() @@ -455,7 +459,7 @@ mod tests { tables .get("dummy2") .unwrap() - .find_table_row(Case::Sensitive, &Vec::new(), None, None) + .find_table_row(Case::Sensitive, &Vec::new(), None, None, None) .unwrap() .get("thing") .cloned() diff --git a/lib/enrichment/src/test_util.rs b/lib/enrichment/src/test_util.rs index 1eed09e200..f56b3342dd 100644 --- a/lib/enrichment/src/test_util.rs +++ b/lib/enrichment/src/test_util.rs @@ -39,6 +39,7 @@ impl Table for DummyEnrichmentTable { _case: Case, _condition: &[Condition], _select: Option<&[String]>, + _wildcard: Option<&Value>, _index: Option, ) -> Result { Ok(self.data.clone()) @@ -49,6 +50,7 @@ impl Table for DummyEnrichmentTable { _case: Case, _condition: &[Condition], _select: Option<&[String]>, + _wildcard: Option<&Value>, _index: Option, ) -> Result, String> { Ok(vec![self.data.clone()]) diff --git a/lib/enrichment/src/vrl_util.rs b/lib/enrichment/src/vrl_util.rs index 61e7044c53..9bf924f030 100644 --- a/lib/enrichment/src/vrl_util.rs +++ b/lib/enrichment/src/vrl_util.rs @@ -38,7 +38,7 @@ impl DiagnosticMessage for Error { } /// Evaluates the condition object to search the enrichment tables with. -pub(crate) fn evaluate_condition(key: &str, value: Value) -> ExpressionResult { +pub(crate) fn evaluate_condition(key: &str, value: Value) -> ExpressionResult> { Ok(match value { Value::Object(map) if map.contains_key("from") && map.contains_key("to") => { Condition::BetweenDates { @@ -55,6 +55,22 @@ pub(crate) fn evaluate_condition(key: &str, value: Value) -> ExpressionResult Condition::FromDate { + field: key, + from: *map + .get("from") + .expect("should contain from") + .as_timestamp() + .ok_or("from in condition must be a timestamp")?, + }, + Value::Object(map) if map.contains_key("to") => Condition::ToDate { + field: key, + to: *map + .get("to") + .expect("should contain to") + .as_timestamp() + .ok_or("to in condition must be a timestamp")?, + }, _ => Condition::Equals { field: key, value }, }) } @@ -71,7 +87,12 @@ pub(crate) fn add_index( .filter_map(|(field, value)| match value { expression::Expr::Container(expression::Container { variant: expression::Variant::Object(map), - }) if map.contains_key("from") && map.contains_key("to") => None, + }) if (map.contains_key("from") && map.contains_key("to")) + || map.contains_key("from") + || map.contains_key("to") => + { + None + } _ => Some(field.as_ref()), }) .collect::>(); @@ -80,6 +101,7 @@ pub(crate) fn add_index( Ok(index) } +#[allow(clippy::result_large_err)] pub(crate) fn is_case_sensitive( arguments: &ArgumentList, state: &TypeState, diff --git a/lib/fakedata/Cargo.toml b/lib/fakedata/Cargo.toml index 8fe5f51459..708086e240 100644 --- a/lib/fakedata/Cargo.toml +++ b/lib/fakedata/Cargo.toml @@ -2,11 +2,11 @@ name = "fakedata" version = "0.1.0" authors = ["Vector Contributors "] -edition = "2021" +edition = "2024" publish = false license = "MPL-2.0" [dependencies] chrono.workspace = true fakedata_generator = "0.5.0" -rand = "0.8.5" +rand.workspace = true diff --git a/lib/fakedata/src/logs.rs b/lib/fakedata/src/logs.rs index 6956b8c379..1c3aa7683d 100644 --- a/lib/fakedata/src/logs.rs +++ b/lib/fakedata/src/logs.rs @@ -1,10 +1,10 @@ use chrono::{ + SecondsFormat, format::{DelayedFormat, StrftimeItems}, prelude::Local, - SecondsFormat, }; use fakedata_generator::{gen_domain, gen_ipv4, gen_username}; -use rand::{thread_rng, Rng}; +use rand::{Rng, rng}; static APPLICATION_NAMES: [&str; 10] = [ "auth", "data", "deploy", "etl", "scraper", "cron", "ingress", "egress", "alerter", "fwd", @@ -217,13 +217,13 @@ fn syslog_version() -> usize { // Helper functions fn random_in_range(min: usize, max: usize) -> usize { - thread_rng().gen_range(min..max) + rng().random_range(min..max) } fn random_from_array(v: &'static [&'static T]) -> &'static T { - v[thread_rng().gen_range(0..v.len())] + v[rng().random_range(0..v.len())] } fn random_from_array_copied(v: &[T]) -> T { - v[thread_rng().gen_range(0..v.len())] + v[rng().random_range(0..v.len())] } diff --git a/lib/file-source-common/.gitignore b/lib/file-source-common/.gitignore new file mode 100644 index 0000000000..eb5a316cbd --- /dev/null +++ b/lib/file-source-common/.gitignore @@ -0,0 +1 @@ +target diff --git a/lib/file-source-common/Cargo.toml b/lib/file-source-common/Cargo.toml new file mode 100644 index 0000000000..c49dbc552f --- /dev/null +++ b/lib/file-source-common/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "file-source-common" +version = "0.1.0" +authors = ["Vector Contributors ", "Mark Story "] +edition = "2024" +publish = false +license = "MIT" + +[target.'cfg(windows)'.dependencies] +libc = "0.2" +winapi = { version = "0.3", features = ["winioctl"] } + +[dependencies] +glob.workspace = true +chrono.workspace = true +tracing.workspace = true +crc = "3.3.0" +scan_fmt = "0.2.6" +serde = { version = "1.0", default-features = false, features = ["derive"] } +serde_json = { version = "1.0.143", default-features = false } +bstr = { version = "1.12", default-features = false } +bytes = { version = "1.10.1", default-features = false, features = ["serde"] } +dashmap = { version = "6.1", default-features = false } +flate2 = { version = "1.1.2", default-features = false, features = ["rust_backend"] } +vector-common = { path = "../vector-common", default-features = false } +vector-config = { path = "../vector-config", default-features = false } + +[dev-dependencies] +criterion = "0.7" +quickcheck = "1" +tempfile.workspace = true +similar-asserts = "1.7.0" diff --git a/lib/file-source-common/LICENSE b/lib/file-source-common/LICENSE new file mode 100644 index 0000000000..a7c557b1f7 --- /dev/null +++ b/lib/file-source-common/LICENSE @@ -0,0 +1,22 @@ +The MIT License + +Original work Copyright (c) 2016, Postmates, Inc. +Modified work Copyright (c) 2015, Mark Story + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/lib/file-source/src/buffer.rs b/lib/file-source-common/src/buffer.rs similarity index 85% rename from lib/file-source/src/buffer.rs rename to lib/file-source-common/src/buffer.rs index c8dbe1f400..55dd481334 100644 --- a/lib/file-source/src/buffer.rs +++ b/lib/file-source-common/src/buffer.rs @@ -1,11 +1,18 @@ -use std::io::{self, BufRead}; +use std::{ + cmp::min, + io::{self, BufRead}, +}; use bstr::Finder; use bytes::BytesMut; -use tracing::warn; use crate::FilePosition; +pub struct ReadResult { + pub successfully_read: Option, + pub discarded_for_size_and_truncated: Vec, +} + /// Read up to `max_size` bytes from `reader`, splitting by `delim` /// /// The function reads up to `max_size` bytes from `reader`, splitting the input @@ -29,17 +36,18 @@ use crate::FilePosition; /// Benchmarks indicate that this function processes in the high single-digit /// GiB/s range for buffers of length 1KiB. For buffers any smaller than this /// the overhead of setup dominates our benchmarks. -pub fn read_until_with_max_size( - reader: &mut R, - position: &mut FilePosition, - delim: &[u8], - buf: &mut BytesMut, +pub fn read_until_with_max_size<'a, R: BufRead + ?Sized>( + reader: &'a mut R, + position: &'a mut FilePosition, + delim: &'a [u8], + buf: &'a mut BytesMut, max_size: usize, -) -> io::Result> { +) -> io::Result { let mut total_read = 0; let mut discarding = false; let delim_finder = Finder::new(delim); let delim_len = delim.len(); + let mut discarded_for_size_and_truncated = Vec::new(); loop { let available: &[u8] = match reader.fill_buf() { Ok(n) => n, @@ -68,16 +76,20 @@ pub fn read_until_with_max_size( total_read += used; if !discarding && buf.len() > max_size { - warn!( - message = "Found line that exceeds max_line_bytes; discarding.", - internal_log_rate_limit = true - ); + // keep only the first <1k bytes to make sure we can actually emit a usable error + let length_to_keep = min(1000, max_size); + let mut truncated: BytesMut = BytesMut::zeroed(length_to_keep); + truncated.copy_from_slice(&buf[0..length_to_keep]); + discarded_for_size_and_truncated.push(truncated); discarding = true; } if done { if !discarding { - return Ok(Some(total_read)); + return Ok(ReadResult { + successfully_read: Some(total_read), + discarded_for_size_and_truncated, + }); } else { discarding = false; buf.clear(); @@ -87,7 +99,10 @@ pub fn read_until_with_max_size( // us to observe an incomplete write. We return None here and let the loop continue // next time the method is called. This is safe because the buffer is specific to this // FileWatcher. - return Ok(None); + return Ok(ReadResult { + successfully_read: None, + discarded_for_size_and_truncated, + }); } } } @@ -99,6 +114,8 @@ mod test { use bytes::{BufMut, BytesMut}; use quickcheck::{QuickCheck, TestResult}; + use crate::buffer::ReadResult; + use super::read_until_with_max_size; fn qc_inner(chunks: Vec>, delim: u8, max_size: NonZeroU8) -> TestResult { @@ -181,7 +198,10 @@ mod test { ) .unwrap() { - None => { + ReadResult { + successfully_read: None, + discarded_for_size_and_truncated: _, + } => { // Subject only returns None if this is the last chunk _and_ // the chunk did not contain a delimiter _or_ the delimiter // was outside the max_size range _or_ the current chunk is empty. @@ -190,7 +210,10 @@ mod test { .any(|details| ((details.chunk_index == idx) && details.within_max_size)); assert!(chunk.is_empty() || !has_valid_delimiter) } - Some(total_read) => { + ReadResult { + successfully_read: Some(total_read), + discarded_for_size_and_truncated: _, + } => { // Now that the function has returned we confirm that the // returned details match our `first_delim` and also that // the `buffer` is populated correctly. diff --git a/lib/file-source/src/checkpointer.rs b/lib/file-source-common/src/checkpointer.rs similarity index 96% rename from lib/file-source/src/checkpointer.rs rename to lib/file-source-common/src/checkpointer.rs index a7eaea307a..53bc392c67 100644 --- a/lib/file-source/src/checkpointer.rs +++ b/lib/file-source-common/src/checkpointer.rs @@ -12,8 +12,8 @@ use serde::{Deserialize, Serialize}; use tracing::{error, info, warn}; use super::{ - fingerprinter::{FileFingerprint, Fingerprinter}, FilePosition, + fingerprinter::{FileFingerprint, Fingerprinter}, }; const TMP_FILE_NAME: &str = "checkpoints.new.json"; @@ -128,10 +128,10 @@ impl CheckpointsView { match state { State::V1 { checkpoints } => { for checkpoint in checkpoints { - if let Some(ignore_before) = ignore_before { - if checkpoint.modified < ignore_before { - continue; - } + if let Some(ignore_before) = ignore_before + && checkpoint.modified < ignore_before + { + continue; } self.load(checkpoint); } @@ -179,20 +179,18 @@ impl CheckpointsView { self.update(fng, pos); } - if self.checkpoints.get(&fng).is_none() { - if let Ok(Some(fingerprint)) = + if self.checkpoints.get(&fng).is_none() + && let Ok(Some(fingerprint)) = fingerprinter.get_legacy_checksum(path, fingerprint_buffer) - { - if let Some((_, pos)) = self.checkpoints.remove(&fingerprint) { - self.update(fng, pos); - } + { + if let Some((_, pos)) = self.checkpoints.remove(&fingerprint) { + self.update(fng, pos); } if let Ok(Some(fingerprint)) = fingerprinter.get_legacy_first_lines_checksum(path, fingerprint_buffer) + && let Some((_, pos)) = self.checkpoints.remove(&fingerprint) { - if let Some((_, pos)) = self.checkpoints.remove(&fingerprint) { - self.update(fng, pos); - } + self.update(fng, pos); } } } @@ -229,10 +227,10 @@ impl Checkpointer { use FileFingerprint::*; let path = match fng { - BytesChecksum(c) => format!("g{:x}.{}", c, pos), - FirstLinesChecksum(c) => format!("h{:x}.{}", c, pos), - DevInode(dev, ino) => format!("i{:x}.{:x}.{}", dev, ino, pos), - Unknown(x) => format!("{:x}.{}", x, pos), + BytesChecksum(c) => format!("g{c:x}.{pos}"), + FirstLinesChecksum(c) => format!("h{c:x}.{pos}"), + DevInode(dev, ino) => format!("i{dev:x}.{ino:x}.{pos}"), + Unknown(x) => format!("{x:x}.{pos}"), }; self.directory.join(path) } @@ -405,15 +403,15 @@ impl Checkpointer { fn read_legacy_checkpoints(&mut self, ignore_before: Option>) { for path in glob(&self.glob_string).unwrap().flatten() { let mut mtime = None; - if let Some(ignore_before) = ignore_before { - if let Ok(Ok(modified)) = fs::metadata(&path).map(|metadata| metadata.modified()) { - let modified = DateTime::::from(modified); - if modified < ignore_before { - fs::remove_file(path).ok(); - continue; - } - mtime = Some(modified); + if let Some(ignore_before) = ignore_before + && let Ok(Ok(modified)) = fs::metadata(&path).map(|metadata| metadata.modified()) + { + let modified = DateTime::::from(modified); + if modified < ignore_before { + fs::remove_file(path).ok(); + continue; } + mtime = Some(modified); } let (fng, pos) = self.decode(&path); self.checkpoints.checkpoints.insert(fng, pos); @@ -432,7 +430,7 @@ mod test { use super::{ super::{FingerprintStrategy, Fingerprinter}, - Checkpoint, Checkpointer, FileFingerprint, FilePosition, CHECKPOINT_FILE_NAME, + CHECKPOINT_FILE_NAME, Checkpoint, Checkpointer, FileFingerprint, FilePosition, TMP_FILE_NAME, }; diff --git a/lib/file-source/src/fingerprinter.rs b/lib/file-source-common/src/fingerprinter.rs similarity index 91% rename from lib/file-source/src/fingerprinter.rs rename to lib/file-source-common/src/fingerprinter.rs index 542e67393d..89b6d89e0e 100644 --- a/lib/file-source/src/fingerprinter.rs +++ b/lib/file-source-common/src/fingerprinter.rs @@ -1,8 +1,9 @@ use std::{ - collections::HashSet, - fs::{self, metadata, File}, + collections::HashMap, + fs::{self, File, metadata}, io::{self, BufRead, BufReader, Read, Seek, SeekFrom, Write}, path::{Path, PathBuf}, + time, }; use crc::Crc; @@ -10,7 +11,7 @@ use flate2::bufread::GzDecoder; use serde::{Deserialize, Serialize}; use vector_common::constants::GZIP_MAGIC; -use crate::{metadata_ext::PortableFileExt, FileSourceInternalEvents}; +use crate::{internal_events::FileSourceInternalEvents, metadata_ext::PortableFileExt}; const FINGERPRINT_CRC: Crc = Crc::::new(&crc::CRC_64_ECMA_182); const LEGACY_FINGERPRINT_CRC: Crc = Crc::::new(&crc::CRC_64_XZ); @@ -121,9 +122,9 @@ impl UncompressedReader for UncompressedReaderImpl { fp.seek(SeekFrom::Start(0))?; let result = fp.read_exact(&mut magic); - if result.is_err() { + if let Err(e) = result { fp.seek(SeekFrom::Start(0))?; - return Err(result.unwrap_err()); + return Err(e); } if magic == magic_header_bytes { @@ -200,7 +201,7 @@ impl Fingerprinter { &self, path: &Path, buffer: &mut Vec, - known_small_files: &mut HashSet, + known_small_files: &mut HashMap, emitter: &impl FileSourceInternalEvents, ) -> Option { metadata(path) @@ -211,21 +212,30 @@ impl Fingerprinter { self.get_fingerprint_of_file(path, buffer).map(Some) } }) - .map_err(|error| match error.kind() { - io::ErrorKind::UnexpectedEof => { - if !known_small_files.contains(path) { - emitter.emit_file_checksum_failed(path); - known_small_files.insert(path.to_path_buf()); + .inspect(|_| { + // Drop the path from the small files map if we've got enough data to fingerprint it. + known_small_files.remove(&path.to_path_buf()); + }) + .map_err(|error| { + match error.kind() { + io::ErrorKind::UnexpectedEof => { + if !known_small_files.contains_key(path) { + emitter.emit_file_checksum_failed(path); + known_small_files.insert(path.to_path_buf(), time::Instant::now()); + } + return; } - } - io::ErrorKind::NotFound => { - if !self.ignore_not_found { + io::ErrorKind::NotFound => { + if !self.ignore_not_found { + emitter.emit_file_fingerprint_read_error(path, error); + } + } + _ => { emitter.emit_file_fingerprint_read_error(path, error); } - } - _ => { - emitter.emit_file_fingerprint_read_error(path, error); - } + }; + // For scenarios other than UnexpectedEOF, remove the path from the small files map. + known_small_files.remove(&path.to_path_buf()); }) .ok() .flatten() @@ -375,15 +385,16 @@ fn fingerprinter_read_until( #[cfg(test)] mod test { use std::{ - collections::HashSet, + collections::HashMap, fs, io::{Error, Read, Write}, path::Path, time::Duration, }; + use bytes::BytesMut; use flate2::write::GzEncoder; - use tempfile::{tempdir, TempDir}; + use tempfile::{TempDir, tempdir}; use super::{FileSourceInternalEvents, FingerprintStrategy, Fingerprinter}; @@ -431,15 +442,21 @@ mod test { fs::write(¬_full_line_path, not_full_line_data).unwrap(); let mut buf = Vec::new(); - assert!(fingerprinter - .get_fingerprint_of_file(&empty_path, &mut buf) - .is_err()); - assert!(fingerprinter - .get_fingerprint_of_file(&full_line_path, &mut buf) - .is_ok()); - assert!(fingerprinter - .get_fingerprint_of_file(¬_full_line_path, &mut buf) - .is_err()); + assert!( + fingerprinter + .get_fingerprint_of_file(&empty_path, &mut buf) + .is_err() + ); + assert!( + fingerprinter + .get_fingerprint_of_file(&full_line_path, &mut buf) + .is_ok() + ); + assert!( + fingerprinter + .get_fingerprint_of_file(¬_full_line_path, &mut buf) + .is_err() + ); assert_eq!( fingerprinter .get_fingerprint_of_file(&full_line_path, &mut buf) @@ -714,12 +731,16 @@ mod test { fs::write(&duplicate_path, &medium_data).unwrap(); let mut buf = Vec::new(); - assert!(fingerprinter - .get_fingerprint_of_file(&empty_path, &mut buf) - .is_ok()); - assert!(fingerprinter - .get_fingerprint_of_file(&small_path, &mut buf) - .is_ok()); + assert!( + fingerprinter + .get_fingerprint_of_file(&empty_path, &mut buf) + .is_ok() + ); + assert!( + fingerprinter + .get_fingerprint_of_file(&small_path, &mut buf) + .is_ok() + ); assert_ne!( fingerprinter .get_fingerprint_of_file(&medium_path, &mut buf) @@ -744,10 +765,17 @@ mod test { }; let mut buf = Vec::new(); - let mut small_files = HashSet::new(); - assert!(fingerprinter - .get_fingerprint_or_log_error(target_dir.path(), &mut buf, &mut small_files, &NoErrors) - .is_none()); + let mut small_files = HashMap::new(); + assert!( + fingerprinter + .get_fingerprint_or_log_error( + target_dir.path(), + &mut buf, + &mut small_files, + &NoErrors + ) + .is_none() + ); } #[test] @@ -803,5 +831,7 @@ mod test { fn emit_files_open(&self, _: usize) {} fn emit_path_globbing_failed(&self, _: &Path, _: &Error) {} + + fn emit_file_line_too_long(&self, _: &BytesMut, _: usize, _: usize) {} } } diff --git a/lib/file-source/src/internal_events.rs b/lib/file-source-common/src/internal_events.rs similarity index 83% rename from lib/file-source/src/internal_events.rs rename to lib/file-source-common/src/internal_events.rs index 9eb60e6539..3077a51d8c 100644 --- a/lib/file-source/src/internal_events.rs +++ b/lib/file-source-common/src/internal_events.rs @@ -1,5 +1,7 @@ use std::{io::Error, path::Path, time::Duration}; +use bytes::BytesMut; + /// Every internal event in this crate has a corresponding /// method in this trait which should emit the event. pub trait FileSourceInternalEvents: Send + Sync + Clone + 'static { @@ -26,4 +28,11 @@ pub trait FileSourceInternalEvents: Send + Sync + Clone + 'static { fn emit_files_open(&self, count: usize); fn emit_path_globbing_failed(&self, path: &Path, error: &Error); + + fn emit_file_line_too_long( + &self, + truncated_bytes: &BytesMut, + configured_limit: usize, + encountered_size_so_far: usize, + ); } diff --git a/lib/file-source-common/src/lib.rs b/lib/file-source-common/src/lib.rs new file mode 100644 index 0000000000..917b69e3e8 --- /dev/null +++ b/lib/file-source-common/src/lib.rs @@ -0,0 +1,51 @@ +#![deny(warnings)] +#![deny(clippy::all)] + +#[macro_use] +extern crate scan_fmt; + +pub mod buffer; +pub mod checkpointer; +mod fingerprinter; +pub mod internal_events; +mod metadata_ext; + +pub use self::{ + checkpointer::{CHECKPOINT_FILE_NAME, Checkpointer, CheckpointsView}, + fingerprinter::{FileFingerprint, FingerprintStrategy, Fingerprinter}, + internal_events::FileSourceInternalEvents, + metadata_ext::PortableFileExt, +}; + +use vector_config::configurable_component; + +pub type FilePosition = u64; + +#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] +pub enum ReadFrom { + #[default] + Beginning, + End, + Checkpoint(FilePosition), +} + +/// File position to use when reading a new file. +#[configurable_component] +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum ReadFromConfig { + /// Read from the beginning of the file. + Beginning, + + /// Start reading from the current end of the file. + End, +} + +impl From for ReadFrom { + fn from(rfc: ReadFromConfig) -> Self { + match rfc { + ReadFromConfig::Beginning => ReadFrom::Beginning, + ReadFromConfig::End => ReadFrom::End, + } + } +} diff --git a/lib/file-source/src/metadata_ext.rs b/lib/file-source-common/src/metadata_ext.rs similarity index 98% rename from lib/file-source/src/metadata_ext.rs rename to lib/file-source-common/src/metadata_ext.rs index ee3b3f79da..21717307f4 100644 --- a/lib/file-source/src/metadata_ext.rs +++ b/lib/file-source-common/src/metadata_ext.rs @@ -13,7 +13,7 @@ use std::{mem::zeroed, ptr}; use winapi::shared::minwindef::DWORD; #[cfg(windows)] use winapi::um::{ - fileapi::GetFileInformationByHandle, fileapi::BY_HANDLE_FILE_INFORMATION, + fileapi::BY_HANDLE_FILE_INFORMATION, fileapi::GetFileInformationByHandle, ioapiset::DeviceIoControl, winioctl::FSCTL_GET_REPARSE_POINT, winnt::FILE_ATTRIBUTE_REPARSE_POINT, winnt::MAXIMUM_REPARSE_DATA_BUFFER_SIZE, }; diff --git a/lib/file-source/Cargo.toml b/lib/file-source/Cargo.toml index ffc102f25f..33880443ba 100644 --- a/lib/file-source/Cargo.toml +++ b/lib/file-source/Cargo.toml @@ -2,7 +2,7 @@ name = "file-source" version = "0.1.0" authors = ["Vector Contributors ", "Mark Story "] -edition = "2021" +edition = "2024" publish = false license = "MIT" @@ -11,72 +11,22 @@ libc = "0.2" winapi = { version = "0.3", features = ["winioctl"] } [dependencies] -crc = "3.2.1" glob.workspace = true -scan_fmt = "0.2.6" +chrono.workspace = true +tokio = { workspace = true, features = ["full"] } +tracing.workspace = true +indexmap = { version = "2.11.0", default-features = false, features = ["serde"] } +bytes.workspace = true +flate2 = { version = "1.1.2", default-features = false, features = ["rust_backend"] } +futures = { version = "0.3.31", default-features = false, features = ["executor"] } vector-common = { path = "../vector-common", default-features = false } -vector-config = { path = "../vector-config", default-features = false } - -[dependencies.bstr] -version = "1.11" -default-features = false -features = [] - -[dependencies.bytes] -version = "1.9.0" -default-features = false -features = [] - -[dependencies.chrono] -version = "0.4" -default-features = false -features = ["clock", "serde"] - -[dependencies.dashmap] -version = "6.1" -default-features = false -features = [] - -[dependencies.indexmap] -version = "2.7.0" -default-features = false -features = ["serde"] - -[dependencies.flate2] -version = "1.0" -default-features = false -features = ["rust_backend"] - -[dependencies.futures] -version = "0.3" -default-features = false -features = ["executor"] - -[dependencies.serde] -version = "1.0" -default-features = false -features = ["derive"] - -[dependencies.serde_json] -version = "1.0" -default-features = false -features = [] - -[dependencies.tracing] -version = "0.1" -default-features = false -features = [] - -[dependencies.tokio] -version = "1.43.0" -default-features = false -features = ["full"] +file-source-common = { path = "../file-source-common" } [dev-dependencies] -criterion = "0.5" +criterion = "0.7" quickcheck = "1" -tempfile = "3.15.0" -similar-asserts = "1.6.0" +tempfile.workspace = true +similar-asserts = "1.7.0" [[bench]] name = "buffer" diff --git a/lib/file-source/benches/buffer.rs b/lib/file-source/benches/buffer.rs index f1a187961d..88ae4197ea 100644 --- a/lib/file-source/benches/buffer.rs +++ b/lib/file-source/benches/buffer.rs @@ -1,8 +1,8 @@ use std::{fmt, io::Cursor}; use bytes::BytesMut; -use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; -use file_source::buffer::read_until_with_max_size; +use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; +use file_source_common::buffer::read_until_with_max_size; struct Parameters { bytes: Vec, diff --git a/lib/file-source/src/file_server.rs b/lib/file-source/src/file_server.rs index 510b20694d..194ea5b7ea 100644 --- a/lib/file-source/src/file_server.rs +++ b/lib/file-source/src/file_server.rs @@ -1,6 +1,6 @@ use std::{ cmp, - collections::{BTreeMap, HashSet}, + collections::{BTreeMap, HashMap}, fs::{self, remove_file}, path::PathBuf, sync::Arc, @@ -10,19 +10,21 @@ use std::{ use bytes::Bytes; use chrono::{DateTime, Utc}; use futures::{ - future::{select, Either}, Future, Sink, SinkExt, + future::{Either, select}, }; use indexmap::IndexMap; use tokio::time::sleep; use tracing::{debug, error, info, trace}; -use crate::{ +use file_source_common::{ + FileFingerprint, FileSourceInternalEvents, Fingerprinter, ReadFrom, checkpointer::{Checkpointer, CheckpointsView}, - file_watcher::FileWatcher, - fingerprinter::{FileFingerprint, Fingerprinter}, +}; + +use crate::{ + file_watcher::{FileWatcher, RawLineResult}, paths_provider::PathsProvider, - FileSourceInternalEvents, ReadFrom, }; /// `FileServer` is a Source which cooperatively schedules reads over files, @@ -100,7 +102,7 @@ where checkpointer.read_checkpoints(self.ignore_before); - let mut known_small_files = HashSet::new(); + let mut known_small_files = HashMap::new(); let mut existing_files = Vec::new(); for path in self.paths_provider.paths().into_iter() { @@ -209,15 +211,14 @@ where if let (Ok(old_modified_time), Ok(new_modified_time)) = ( fs::metadata(old_path).and_then(|m| m.modified()), fs::metadata(new_path).and_then(|m| m.modified()), - ) { - if old_modified_time < new_modified_time { - info!( - message = "Switching to watch most recently modified file.", - new_modified_time = ?new_modified_time, - old_modified_time = ?old_modified_time, - ); - watcher.update_path(path).ok(); // ok if this fails: might fix next cycle - } + ) && old_modified_time < new_modified_time + { + info!( + message = "Switching to watch most recently modified file.", + new_modified_time = ?new_modified_time, + old_modified_time = ?old_modified_time, + ); + watcher.update_path(path).ok(); // ok if this fails: might fix next cycle } } } else { @@ -230,6 +231,29 @@ where stats.record("discovery", start.elapsed()); } + // Cleanup the known_small_files + if let Some(grace_period) = self.remove_after { + known_small_files.retain(|path, last_time_open| { + // Should the file be removed + if last_time_open.elapsed() >= grace_period { + // Try to remove + match remove_file(path) { + Ok(()) => { + self.emitter.emit_file_deleted(path); + false + } + Err(error) => { + // We will try again after some time. + self.emitter.emit_file_delete_error(path, error); + true + } + } + } else { + true + } + }); + } + // Collect lines by polling files. let mut global_bytes_read: usize = 0; let mut maxed_out_reading_single_file = false; @@ -240,7 +264,19 @@ where let start = time::Instant::now(); let mut bytes_read: usize = 0; - while let Ok(Some(line)) = watcher.read_line() { + while let Ok(RawLineResult { + raw_line: Some(line), + discarded_for_size_and_truncated, + }) = watcher.read_line() + { + discarded_for_size_and_truncated.iter().for_each(|buf| { + self.emitter.emit_file_line_too_long( + &buf.clone(), + self.max_line_bytes, + buf.len(), + ) + }); + let sz = line.bytes.len(); trace!( message = "Read bytes.", @@ -270,18 +306,18 @@ where global_bytes_read = global_bytes_read.saturating_add(bytes_read); } else { // Should the file be removed - if let Some(grace_period) = self.remove_after { - if watcher.last_read_success().elapsed() >= grace_period { - // Try to remove - match remove_file(&watcher.path) { - Ok(()) => { - self.emitter.emit_file_deleted(&watcher.path); - watcher.set_dead(); - } - Err(error) => { - // We will try again after some time. - self.emitter.emit_file_delete_error(&watcher.path, error); - } + if let Some(grace_period) = self.remove_after + && watcher.last_read_success().elapsed() >= grace_period + { + // Try to remove + match remove_file(&watcher.path) { + Ok(()) => { + self.emitter.emit_file_deleted(&watcher.path); + watcher.set_dead(); + } + Err(error) => { + // We will try again after some time. + self.emitter.emit_file_delete_error(&watcher.path, error); } } } diff --git a/lib/file-source/src/file_watcher/mod.rs b/lib/file-source/src/file_watcher/mod.rs index 80db2b9bc8..68db589c32 100644 --- a/lib/file-source/src/file_watcher/mod.rs +++ b/lib/file-source/src/file_watcher/mod.rs @@ -1,3 +1,4 @@ +#![allow(dead_code)] // FIXME use std::{ fs::{self, File}, io::{self, BufRead, Seek}, @@ -11,9 +12,11 @@ use flate2::bufread::MultiGzDecoder; use tracing::debug; use vector_common::constants::GZIP_MAGIC; -use crate::{ - buffer::read_until_with_max_size, metadata_ext::PortableFileExt, FilePosition, ReadFrom, +use file_source_common::{ + FilePosition, PortableFileExt, ReadFrom, + buffer::{ReadResult, read_until_with_max_size}, }; + #[cfg(test)] mod tests; @@ -23,11 +26,17 @@ mod tests; /// The offset field contains the byte offset of the beginning of the line within /// the file that it was read from. #[derive(Debug)] -pub(super) struct RawLine { +pub struct RawLine { pub offset: u64, pub bytes: Bytes, } +#[derive(Debug)] +pub struct RawLineResult { + pub raw_line: Option, + pub discarded_for_size_and_truncated: Vec, +} + /// The `FileWatcher` struct defines the polling based state machine which reads /// from a file path, transparently updating the underlying file descriptor when /// the file has been rolled over, as is common for logs. @@ -207,7 +216,7 @@ impl FileWatcher { /// This function will attempt to read a new line from its file, blocking, /// up to some maximum but unspecified amount of time. `read_line` will open /// a new file handler as needed, transparently to the caller. - pub(super) fn read_line(&mut self) -> io::Result> { + pub(super) fn read_line(&mut self) -> io::Result { self.track_read_attempt(); let reader = &mut self.reader; @@ -220,14 +229,23 @@ impl FileWatcher { &mut self.buf, self.max_line_bytes, ) { - Ok(Some(_)) => { + Ok(ReadResult { + successfully_read: Some(_), + discarded_for_size_and_truncated, + }) => { self.track_read_success(); - Ok(Some(RawLine { - offset: initial_position, - bytes: self.buf.split().freeze(), - })) + Ok(RawLineResult { + raw_line: Some(RawLine { + offset: initial_position, + bytes: self.buf.split().freeze(), + }), + discarded_for_size_and_truncated, + }) } - Ok(None) => { + Ok(ReadResult { + successfully_read: None, + discarded_for_size_and_truncated, + }) => { if !self.file_findable() { self.set_dead(); // File has been deleted, so return what we have in the buffer, even though it @@ -237,16 +255,25 @@ impl FileWatcher { if buf.is_empty() { // EOF self.reached_eof = true; - Ok(None) + Ok(RawLineResult { + raw_line: None, + discarded_for_size_and_truncated, + }) } else { - Ok(Some(RawLine { - offset: initial_position, - bytes: buf, - })) + Ok(RawLineResult { + raw_line: Some(RawLine { + offset: initial_position, + bytes: buf, + }), + discarded_for_size_and_truncated, + }) } } else { self.reached_eof = true; - Ok(None) + Ok(RawLineResult { + raw_line: None, + discarded_for_size_and_truncated, + }) } } Err(e) => { diff --git a/lib/file-source/src/file_watcher/tests/experiment.rs b/lib/file-source/src/file_watcher/tests/experiment.rs index decdbdab98..d5b0c128a8 100644 --- a/lib/file-source/src/file_watcher/tests/experiment.rs +++ b/lib/file-source/src/file_watcher/tests/experiment.rs @@ -7,10 +7,9 @@ use std::{fs, io::Write}; use bytes::Bytes; use quickcheck::{QuickCheck, TestResult}; -use crate::{ - file_watcher::{tests::*, FileWatcher}, - ReadFrom, -}; +use crate::file_watcher::{FileWatcher, RawLineResult, tests::*}; + +use file_source_common::ReadFrom; // Interpret all FWActions, including truncation // @@ -82,7 +81,7 @@ fn experiment(actions: Vec) { } FileWatcherAction::RotateFile => { let mut new_path = path.clone(); - new_path.set_extension(format!("log.{}", rotation_count)); + new_path.set_extension(format!("log.{rotation_count}")); rotation_count += 1; fs::rename(&path, &new_path).expect("could not rename"); fp = fs::File::create(&path).expect("could not create"); @@ -96,11 +95,14 @@ fn experiment(actions: Vec) { Err(_) => { unreachable!(); } - Ok(Some(line)) if line.bytes.is_empty() => { + Ok(RawLineResult { + raw_line: Some(line), + .. + }) if line.bytes.is_empty() => { attempts -= 1; continue; } - Ok(None) => { + Ok(RawLineResult { raw_line: None, .. }) => { attempts -= 1; continue; } @@ -129,7 +131,7 @@ fn file_watcher_with_truncation() { TestResult::passed() } QuickCheck::new() - .tests(10000) - .max_tests(100000) + .tests(5000) + .max_tests(50000) .quickcheck(inner as fn(Vec) -> TestResult); } diff --git a/lib/file-source/src/file_watcher/tests/experiment_no_truncations.rs b/lib/file-source/src/file_watcher/tests/experiment_no_truncations.rs index ee8a24a9f9..d1323cc432 100644 --- a/lib/file-source/src/file_watcher/tests/experiment_no_truncations.rs +++ b/lib/file-source/src/file_watcher/tests/experiment_no_truncations.rs @@ -1,12 +1,10 @@ use std::{fs, io::Write}; use bytes::Bytes; +use file_source_common::ReadFrom; use quickcheck::{QuickCheck, TestResult}; -use crate::{ - file_watcher::{tests::*, FileWatcher}, - ReadFrom, -}; +use crate::file_watcher::{FileWatcher, RawLineResult, tests::*}; // Interpret all FWActions, excluding truncation // @@ -49,7 +47,7 @@ fn experiment_no_truncations(actions: Vec) { } FileWatcherAction::RotateFile => { let mut new_path = path.clone(); - new_path.set_extension(format!("log.{}", rotation_count)); + new_path.set_extension(format!("log.{rotation_count}")); rotation_count += 1; fs::rename(&path, &new_path).expect("could not rename"); fp = fs::File::create(&path).expect("could not create"); @@ -63,17 +61,23 @@ fn experiment_no_truncations(actions: Vec) { Err(_) => { unreachable!(); } - Ok(Some(line)) if line.bytes.is_empty() => { + Ok(RawLineResult { + raw_line: Some(line), + .. + }) if line.bytes.is_empty() => { attempts -= 1; assert!(fwfiles[read_index].read_line().is_none()); continue; } - Ok(None) => { + Ok(RawLineResult { raw_line: None, .. }) => { attempts -= 1; assert!(fwfiles[read_index].read_line().is_none()); continue; } - Ok(Some(line)) => { + Ok(RawLineResult { + raw_line: Some(line), + .. + }) => { let exp = fwfiles[read_index].read_line().expect("could not readline"); assert_eq!(exp.into_bytes(), line.bytes); // assert_eq!(sz, buf.len() + 1); @@ -93,7 +97,7 @@ fn file_watcher_no_truncation() { TestResult::passed() } QuickCheck::new() - .tests(10000) - .max_tests(100000) + .tests(5000) + .max_tests(50000) .quickcheck(inner as fn(Vec) -> TestResult); } diff --git a/lib/file-source/src/lib.rs b/lib/file-source/src/lib.rs index ace261070f..97991f9b0a 100644 --- a/lib/file-source/src/lib.rs +++ b/lib/file-source/src/lib.rs @@ -1,53 +1,6 @@ #![deny(warnings)] #![deny(clippy::all)] -#[macro_use] -extern crate scan_fmt; - -pub mod buffer; -mod checkpointer; -mod file_server; -mod file_watcher; -mod fingerprinter; -mod internal_events; -mod metadata_ext; +pub mod file_server; +pub mod file_watcher; pub mod paths_provider; - -pub use self::{ - checkpointer::{Checkpointer, CheckpointsView, CHECKPOINT_FILE_NAME}, - file_server::{calculate_ignore_before, FileServer, Line, Shutdown as FileServerShutdown}, - fingerprinter::{FileFingerprint, FingerprintStrategy, Fingerprinter}, - internal_events::FileSourceInternalEvents, -}; -use vector_config::configurable_component; - -pub type FilePosition = u64; - -#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] -pub enum ReadFrom { - #[default] - Beginning, - End, - Checkpoint(FilePosition), -} - -/// File position to use when reading a new file. -#[configurable_component] -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum ReadFromConfig { - /// Read from the beginning of the file. - Beginning, - - /// Start reading from the current end of the file. - End, -} - -impl From for ReadFrom { - fn from(rfc: ReadFromConfig) -> Self { - match rfc { - ReadFromConfig::Beginning => ReadFrom::Beginning, - ReadFromConfig::End => ReadFrom::End, - } - } -} diff --git a/lib/file-source/src/paths_provider/glob.rs b/lib/file-source/src/paths_provider.rs similarity index 74% rename from lib/file-source/src/paths_provider/glob.rs rename to lib/file-source/src/paths_provider.rs index 26259cbcaa..c5cf5f668e 100644 --- a/lib/file-source/src/paths_provider/glob.rs +++ b/lib/file-source/src/paths_provider.rs @@ -1,12 +1,36 @@ -//! [`Glob`] paths provider. +//! [`Glob`] based paths provider implementation. use std::path::PathBuf; pub use glob::MatchOptions; use glob::Pattern; -use super::PathsProvider; -use crate::FileSourceInternalEvents; +use file_source_common::internal_events::FileSourceInternalEvents; + +/// Represents the ability to enumerate paths. +/// +/// For use in [`crate::file_server::FileServer`]. +/// +/// # Notes +/// +/// Ideally we'd use an iterator with bound lifetime here: +/// +/// ```ignore +/// type Iter<'a>: Iterator + 'a; +/// fn paths(&self) -> Self::Iter<'_>; +/// ``` +/// +/// However, that's currently unavailable at Rust. +/// See: +/// +/// We use an `IntoIter` here as a workaround. +pub trait PathsProvider { + /// Provides the iterator that returns paths. + type IntoIter: IntoIterator; + + /// Provides a set of paths. + fn paths(&self) -> Self::IntoIter; +} /// A glob-based path provider. /// diff --git a/lib/file-source/src/paths_provider/mod.rs b/lib/file-source/src/paths_provider/mod.rs deleted file mode 100644 index fa3aab9c23..0000000000 --- a/lib/file-source/src/paths_provider/mod.rs +++ /dev/null @@ -1,33 +0,0 @@ -//! Abstractions to allow configuring ways to provide the paths list for the -//! file source to watch and read. - -#![deny(missing_docs)] - -use std::path::PathBuf; - -pub mod glob; - -/// Represents the ability to enumerate paths. -/// -/// For use at [`crate::FileServer`]. -/// -/// # Notes -/// -/// Ideally we'd use an iterator with bound lifetime here: -/// -/// ```ignore -/// type Iter<'a>: Iterator + 'a; -/// fn paths(&self) -> Self::Iter<'_>; -/// ``` -/// -/// However, that's currently unavailable at Rust. -/// See: -/// -/// We use an `IntoIter` here as a workaround. -pub trait PathsProvider { - /// Provides the iterator that returns paths. - type IntoIter: IntoIterator; - - /// Provides a set of paths. - fn paths(&self) -> Self::IntoIter; -} diff --git a/lib/k8s-e2e-tests/Cargo.toml b/lib/k8s-e2e-tests/Cargo.toml index f2219b2c40..3b4adc3beb 100644 --- a/lib/k8s-e2e-tests/Cargo.toml +++ b/lib/k8s-e2e-tests/Cargo.toml @@ -2,23 +2,23 @@ name = "k8s-e2e-tests" version = "0.1.0" authors = ["Vector Contributors "] -edition = "2021" +edition = "2024" description = "End-to-end tests of Vector in the Kubernetes environment" publish = false license = "MPL-2.0" [dependencies] -futures = "0.3" +futures.workspace = true k8s-openapi = { version = "0.16.0", default-features = false, features = ["v1_19"] } k8s-test-framework = { version = "0.1", path = "../k8s-test-framework" } -regex = "1" -reqwest = { version = "0.11.26", features = ["json"] } +regex = { workspace = true, default-features = true } +reqwest.workspace = true serde_json.workspace = true -tokio = { version = "1.43.0", features = ["full"] } -indoc = "2.0.5" +tokio = { workspace = true, features = ["full"] } +indoc.workspace = true env_logger = "0.11" -tracing = { version = "0.1", features = ["log"] } -rand = "0.8" +tracing = { workspace = true, features = ["log"] } +rand.workspace = true [features] e2e-tests = [] diff --git a/lib/k8s-e2e-tests/src/lib.rs b/lib/k8s-e2e-tests/src/lib.rs index 076d2015f1..59e89501fe 100644 --- a/lib/k8s-e2e-tests/src/lib.rs +++ b/lib/k8s-e2e-tests/src/lib.rs @@ -8,8 +8,10 @@ use k8s_openapi::{ apimachinery::pkg::apis::meta::v1::{LabelSelector, ObjectMeta}, }; use k8s_test_framework::{ - test_pod, wait_for_resource::WaitFor, CommandBuilder, Framework, Interface, Manager, Reader, + CommandBuilder, Framework, Interface, Manager, Reader, test_pod, wait_for_resource::WaitFor, }; +use rand::distr::{Alphanumeric, SampleString}; +use rand::rng; use tracing::{debug, error, info}; pub mod metrics; @@ -21,28 +23,21 @@ pub fn init() { } pub fn get_namespace() -> String { - use rand::Rng; - - // Generate a random alphanumeric (lowercase) string to ensure each test is run with unique - // names. + // Generate a random alphanumeric (lowercase) string to ensure each test is run with unique names. // There is a 36 ^ 5 chance of a name collision, which is likely to be an acceptable risk. - let id: String = rand::thread_rng() - .sample_iter(&rand::distributions::Alphanumeric) - .take(5) - .map(|num| (num as char).to_ascii_lowercase()) - .collect(); + let id = Alphanumeric.sample_string(&mut rng(), 5).to_lowercase(); - format!("vector-{}", id) + format!("vector-{id}") } pub fn get_namespace_appended(namespace: &str, suffix: &str) -> String { - format!("{}-{}", namespace, suffix) + format!("{namespace}-{suffix}") } /// Gets a name we can use for roles to prevent them conflicting with other tests. /// Uses the provided namespace as the root. pub fn get_override_name(namespace: &str, suffix: &str) -> String { - format!("{}-{}", namespace, suffix) + format!("{namespace}-{suffix}") } /// Is the MULTINODE environment variable set? @@ -54,7 +49,7 @@ pub fn is_multinode() -> bool { /// to be run against the same cluster without the role names clashing. pub fn config_override_name(name: &str, cleanup: bool) -> String { let vectordir = if is_multinode() { - format!("{}-vector", name) + format!("{name}-vector") } else { "vector".to_string() }; @@ -241,9 +236,7 @@ pub async fn smoke_check_first_line(log_reader: &mut Reader) { let expected_pat = "INFO vector::app:"; assert!( first_line.contains(expected_pat), - "Expected a line ending with {:?} but got {:?}; vector might be malfunctioning", - expected_pat, - first_line + "Expected a line ending with {expected_pat:?} but got {first_line:?}; vector might be malfunctioning" ); } @@ -281,7 +274,9 @@ where // We got an EOF error, this is most likely some very long line, // we don't produce lines this bing is our test cases, so we'll // just skip the error - as if it wasn't a JSON string. - error!("The JSON line we just got was incomplete, most likely it was too long, so we're skipping it"); + error!( + "The JSON line we just got was incomplete, most likely it was too long, so we're skipping it" + ); continue; } Err(err) => return Err(err.into()), diff --git a/lib/k8s-e2e-tests/src/metrics.rs b/lib/k8s-e2e-tests/src/metrics.rs index d3fd61548e..a815acac35 100644 --- a/lib/k8s-e2e-tests/src/metrics.rs +++ b/lib/k8s-e2e-tests/src/metrics.rs @@ -64,7 +64,7 @@ pub async fn get_component_sent_events_total(url: &str) -> Result Result<(), Box> { let metrics = load(url).await?; if !extract_vector_started(&metrics) { - return Err(format!("`vector_started`-ish metric was not found:\n{}", metrics).into()); + return Err(format!("`vector_started`-ish metric was not found:\n{metrics}").into()); } Ok(()) } @@ -125,7 +125,7 @@ pub async fn assert_metrics_present( required_metrics.remove(metric_name); } if !required_metrics.is_empty() { - return Err(format!("Some host metrics were not found:\n{:?}", required_metrics).into()); + return Err(format!("Some host metrics were not found:\n{required_metrics:?}").into()); } Ok(()) } @@ -217,7 +217,7 @@ mod tests { for (input, expected_value) in cases { let input = input.join("\n"); let actual_value = extract_vector_started(&input); - assert_eq!(expected_value, actual_value, "input: {}", input); + assert_eq!(expected_value, actual_value, "input: {input}"); } } } diff --git a/lib/k8s-e2e-tests/tests/vector-agent.rs b/lib/k8s-e2e-tests/tests/vector-agent.rs index 49bd9a2439..2bf56f3233 100644 --- a/lib/k8s-e2e-tests/tests/vector-agent.rs +++ b/lib/k8s-e2e-tests/tests/vector-agent.rs @@ -1,3 +1,5 @@ +#![allow(clippy::await_holding_lock)] + use std::{ collections::{BTreeMap, HashSet}, iter::FromIterator, @@ -85,14 +87,14 @@ async fn default_agent() -> Result<(), Box> { framework .wait_for_rollout( &namespace, - &format!("daemonset/{}", override_name), + &format!("daemonset/{override_name}"), vec!["--timeout=60s"], ) .await?; let mut vector_metrics_port_forward = framework.port_forward( &namespace, - &format!("daemonset/{}", override_name), + &format!("daemonset/{override_name}"), 9090, 9090, )?; @@ -139,7 +141,7 @@ async fn default_agent() -> Result<(), Box> { .get_vector_pod_with_pod(&pod_namespace, "test-pod", &namespace, &override_name) .await?; - let mut log_reader = framework.logs(&namespace, &format!("pod/{}", vector_pod))?; + let mut log_reader = framework.logs(&namespace, &format!("pod/{vector_pod}"))?; smoke_check_first_line(&mut log_reader).await; // Read the rest of the log lines. @@ -210,14 +212,14 @@ async fn partial_merge() -> Result<(), Box> { framework .wait_for_rollout( &namespace, - &format!("daemonset/{}", override_name), + &format!("daemonset/{override_name}"), vec!["--timeout=60s"], ) .await?; let mut vector_metrics_port_forward = framework.port_forward( &namespace, - &format!("daemonset/{}", override_name), + &format!("daemonset/{override_name}"), 9090, 9090, )?; @@ -246,7 +248,7 @@ async fn partial_merge() -> Result<(), Box> { .test_pod(test_pod::Config::from_pod(&make_test_pod( &pod_namespace, "test-pod", - &format!("echo {}", test_message), + &format!("echo {test_message}"), vec![], vec![], ))?) @@ -265,7 +267,7 @@ async fn partial_merge() -> Result<(), Box> { .get_vector_pod_with_pod(&pod_namespace, "test-pod", &namespace, &override_name) .await?; - let mut log_reader = framework.logs(&namespace, &format!("pod/{}", vector_pod))?; + let mut log_reader = framework.logs(&namespace, &format!("pod/{vector_pod}"))?; smoke_check_first_line(&mut log_reader).await; // Read the rest of the log lines. @@ -362,14 +364,14 @@ async fn preexisting() -> Result<(), Box> { framework .wait_for_rollout( &namespace, - &format!("daemonset/{}", override_name), + &format!("daemonset/{override_name}"), vec!["--timeout=60s"], ) .await?; let mut vector_metrics_port_forward = framework.port_forward( &namespace, - &format!("daemonset/{}", override_name), + &format!("daemonset/{override_name}"), 9090, 9090, )?; @@ -392,7 +394,7 @@ async fn preexisting() -> Result<(), Box> { .get_vector_pod_with_pod(&pod_namespace, "test-pod", &namespace, &override_name) .await?; - let mut log_reader = framework.logs(&namespace, &format!("pod/{}", vector_pod))?; + let mut log_reader = framework.logs(&namespace, &format!("pod/{vector_pod}"))?; smoke_check_first_line(&mut log_reader).await; // Read the rest of the log lines. @@ -464,14 +466,14 @@ async fn multiple_lines() -> Result<(), Box> { framework .wait_for_rollout( &namespace, - &format!("daemonset/{}", override_name), + &format!("daemonset/{override_name}"), vec!["--timeout=60s"], ) .await?; let mut vector_metrics_port_forward = framework.port_forward( &namespace, - &format!("daemonset/{}", override_name), + &format!("daemonset/{override_name}"), 9090, 9090, )?; @@ -519,7 +521,7 @@ async fn multiple_lines() -> Result<(), Box> { .get_vector_pod_with_pod(&pod_namespace, "test-pod", &namespace, &override_name) .await?; - let mut log_reader = framework.logs(&namespace, &format!("pod/{}", vector_pod))?; + let mut log_reader = framework.logs(&namespace, &format!("pod/{vector_pod}"))?; smoke_check_first_line(&mut log_reader).await; // Read the rest of the log lines. @@ -591,14 +593,14 @@ async fn metadata_annotation() -> Result<(), Box> { framework .wait_for_rollout( &namespace, - &format!("daemonset/{}", override_name), + &format!("daemonset/{override_name}"), vec!["--timeout=60s"], ) .await?; let mut vector_metrics_port_forward = framework.port_forward( &namespace, - &format!("daemonset/{}", override_name), + &format!("daemonset/{override_name}"), 9090, 9090, )?; @@ -657,7 +659,7 @@ async fn metadata_annotation() -> Result<(), Box> { .get_vector_pod_with_pod(&pod_namespace, "test-pod", &namespace, &override_name) .await?; - let mut log_reader = framework.logs(&namespace, &format!("pod/{}", vector_pod))?; + let mut log_reader = framework.logs(&namespace, &format!("pod/{vector_pod}"))?; smoke_check_first_line(&mut log_reader).await; let k8s_version = framework.kubernetes_version().await?; @@ -665,10 +667,12 @@ async fn metadata_annotation() -> Result<(), Box> { let numeric_regex = regex::Regex::new(r#"[^\d]"#).unwrap(); let minor = k8s_version.minor(); let numeric_minor = numeric_regex.replace(&minor, ""); - let minor = u8::from_str(&numeric_minor).expect(&format!( - "Couldn't get u8 from String, received {} instead!", - k8s_version.minor() - )); + let minor = u8::from_str(&numeric_minor).unwrap_or_else(|_| { + panic!( + "Couldn't get u8 from String, received {} instead!", + k8s_version.minor() + ) + }); // Read the rest of the log lines. let mut got_marker = false; @@ -706,22 +710,28 @@ async fn metadata_annotation() -> Result<(), Box> { assert!(val["kubernetes"]["pod_ip"].is_string()); } else { assert!(val["kubernetes"]["pod_ip"].is_string()); - assert!(!val["kubernetes"]["pod_ips"] - .as_array() - .expect("Couldn't take array from expected vec") - .is_empty()); + assert!( + !val["kubernetes"]["pod_ips"] + .as_array() + .expect("Couldn't take array from expected vec") + .is_empty() + ); } // We don't have the node name to compare this to, so just assert it's // a non-empty string. - assert!(!val["kubernetes"]["pod_node_name"] - .as_str() - .unwrap() - .is_empty()); + assert!( + !val["kubernetes"]["pod_node_name"] + .as_str() + .unwrap() + .is_empty() + ); assert_eq!(val["kubernetes"]["container_name"], "test-pod"); - assert!(!val["kubernetes"]["container_id"] - .as_str() - .unwrap() - .is_empty()); + assert!( + !val["kubernetes"]["container_id"] + .as_str() + .unwrap() + .is_empty() + ); assert_eq!(val["kubernetes"]["container_image"], BUSYBOX_IMAGE); // Request to stop the flow. @@ -750,7 +760,7 @@ async fn pod_filtering() -> Result<(), Box> { let namespace = get_namespace(); let pod_namespace = get_namespace_appended(&namespace, "test-pod"); - let affinity_label = format!("{}-affinity", pod_namespace); + let affinity_label = format!("{pod_namespace}-affinity"); let framework = make_framework(); let override_name = get_override_name(&namespace, "vector-agent"); @@ -772,14 +782,14 @@ async fn pod_filtering() -> Result<(), Box> { framework .wait_for_rollout( &namespace, - &format!("daemonset/{}", override_name), + &format!("daemonset/{override_name}"), vec!["--timeout=60s"], ) .await?; let mut vector_metrics_port_forward = framework.port_forward( &namespace, - &format!("daemonset/{}", override_name), + &format!("daemonset/{override_name}"), 9090, 9090, )?; @@ -803,7 +813,7 @@ async fn pod_filtering() -> Result<(), Box> { )?) .await?; - let affinity_ns_name = format!("{}-affinity", pod_namespace); + let affinity_ns_name = format!("{pod_namespace}-affinity"); let affinity_ns = framework .namespace(namespace::Config::from_namespace( &namespace::make_namespace(affinity_ns_name.clone(), None), @@ -864,7 +874,7 @@ async fn pod_filtering() -> Result<(), Box> { ) .await?; - let mut log_reader = framework.logs(&namespace, &format!("pod/{}", vector_pod))?; + let mut log_reader = framework.logs(&namespace, &format!("pod/{vector_pod}"))?; smoke_check_first_line(&mut log_reader).await; // Read the log lines until the reasonable amount of time passes for us @@ -1012,7 +1022,7 @@ async fn custom_selectors() -> Result<(), Box> { framework .wait_for_rollout( &namespace, - &format!("daemonset/{}", override_name), + &format!("daemonset/{override_name}"), vec!["--timeout=60s"], ) .await?; @@ -1048,7 +1058,7 @@ async fn custom_selectors() -> Result<(), Box> { excluded_test_pod_names.push(name); } for name in excluded_test_pod_names { - let name = format!("pods/{}", name); + let name = format!("pods/{name}"); framework .wait( &pod_namespace, @@ -1087,7 +1097,7 @@ async fn custom_selectors() -> Result<(), Box> { ) .await?; - let mut log_reader = framework.logs(&namespace, &format!("pod/{}", vector_pod))?; + let mut log_reader = framework.logs(&namespace, &format!("pod/{vector_pod}"))?; smoke_check_first_line(&mut log_reader).await; // Read the log lines until the reasonable amount of time passes for us @@ -1210,14 +1220,14 @@ async fn container_filtering() -> Result<(), Box> { framework .wait_for_rollout( &namespace, - &format!("daemonset/{}", override_name), + &format!("daemonset/{override_name}"), vec!["--timeout=60s"], ) .await?; let mut vector_metrics_port_forward = framework.port_forward( &namespace, - &format!("daemonset/{}", override_name), + &format!("daemonset/{override_name}"), 9090, 9090, )?; @@ -1268,7 +1278,7 @@ async fn container_filtering() -> Result<(), Box> { .get_vector_pod_with_pod(&pod_namespace, "test-pod", &namespace, &override_name) .await?; - let mut log_reader = framework.logs(&namespace, &format!("pod/{}", vector_pod))?; + let mut log_reader = framework.logs(&namespace, &format!("pod/{vector_pod}"))?; smoke_check_first_line(&mut log_reader).await; // Read the log lines until the reasonable amount of time passes for us @@ -1416,7 +1426,7 @@ async fn glob_pattern_filtering() -> Result<(), Box> { framework .wait_for_rollout( &namespace, - &format!("daemonset/{}", override_name), + &format!("daemonset/{override_name}"), vec!["--timeout=60s"], ) .await?; @@ -1454,7 +1464,7 @@ async fn glob_pattern_filtering() -> Result<(), Box> { .get_vector_pod_with_pod(&pod_namespace, "test-pod", &namespace, &override_name) .await?; - let mut log_reader = framework.logs(&namespace, &format!("pod/{}", vector_pod))?; + let mut log_reader = framework.logs(&namespace, &format!("pod/{vector_pod}"))?; smoke_check_first_line(&mut log_reader).await; // Read the log lines until the reasonable amount of time passes for us @@ -1557,7 +1567,7 @@ async fn multiple_ns() -> Result<(), Box> { let namespace = get_namespace(); let pod_namespace = get_namespace_appended(&namespace, "test-pod"); - let affinity_label = format!("{}-affinity", pod_namespace); + let affinity_label = format!("{pod_namespace}-affinity"); let framework = make_framework(); let override_name = get_override_name(&namespace, "vector-agent"); @@ -1579,14 +1589,14 @@ async fn multiple_ns() -> Result<(), Box> { framework .wait_for_rollout( &namespace, - &format!("daemonset/{}", override_name), + &format!("daemonset/{override_name}"), vec!["--timeout=60s"], ) .await?; let mut vector_metrics_port_forward = framework.port_forward( &namespace, - &format!("daemonset/{}", override_name), + &format!("daemonset/{override_name}"), 9090, 9090, )?; @@ -1607,7 +1617,7 @@ async fn multiple_ns() -> Result<(), Box> { let mut test_namespaces = vec![]; let mut expected_namespaces = HashSet::new(); for i in 0..10 { - let name = format!("{}-{}", pod_namespace, i); + let name = format!("{pod_namespace}-{i}"); test_namespaces.push( framework .namespace(namespace::Config::from_namespace( @@ -1620,7 +1630,7 @@ async fn multiple_ns() -> Result<(), Box> { // Create a pod for our other pods to have an affinity to ensure they are all deployed on // the same node. - let affinity_ns_name = format!("{}-affinity", pod_namespace); + let affinity_ns_name = format!("{pod_namespace}-affinity"); let affinity_ns = framework .namespace(namespace::Config::from_namespace( &namespace::make_namespace(affinity_ns_name.clone(), None), @@ -1663,7 +1673,7 @@ async fn multiple_ns() -> Result<(), Box> { ) .await?; - let mut log_reader = framework.logs(&namespace, &format!("pod/{}", vector_pod))?; + let mut log_reader = framework.logs(&namespace, &format!("pod/{vector_pod}"))?; smoke_check_first_line(&mut log_reader).await; // Read the rest of the log lines. @@ -1741,7 +1751,7 @@ async fn existing_config_file() -> Result<(), Box> { framework .wait_for_rollout( &namespace, - &format!("daemonset/{}", override_name), + &format!("daemonset/{override_name}"), vec!["--timeout=60s"], ) .await?; @@ -1775,7 +1785,7 @@ async fn existing_config_file() -> Result<(), Box> { .get_vector_pod_with_pod(&pod_namespace, "test-pod", &namespace, &override_name) .await?; - let mut log_reader = framework.logs(&namespace, &format!("pod/{}", vector_pod))?; + let mut log_reader = framework.logs(&namespace, &format!("pod/{vector_pod}"))?; smoke_check_first_line(&mut log_reader).await; // Read the rest of the log lines. @@ -1842,14 +1852,14 @@ async fn metrics_pipeline() -> Result<(), Box> { framework .wait_for_rollout( &namespace, - &format!("daemonset/{}", override_name), + &format!("daemonset/{override_name}"), vec!["--timeout=60s"], ) .await?; let mut vector_metrics_port_forward = framework.port_forward( &namespace, - &format!("daemonset/{}", override_name), + &format!("daemonset/{override_name}"), 9090, 9090, )?; @@ -1911,7 +1921,7 @@ async fn metrics_pipeline() -> Result<(), Box> { .get_vector_pod_with_pod(&pod_namespace, "test-pod", &namespace, &override_name) .await?; - let mut log_reader = framework.logs(&namespace, &format!("pod/{}", vector_pod))?; + let mut log_reader = framework.logs(&namespace, &format!("pod/{vector_pod}"))?; smoke_check_first_line(&mut log_reader).await; // Read the rest of the log lines. @@ -1955,9 +1965,7 @@ async fn metrics_pipeline() -> Result<(), Box> { // Ensure we did get at least one event since before deployed the test pod. assert!( processed_events_after > processed_events_before, - "before: {}, after: {}", - processed_events_before, - processed_events_after + "before: {processed_events_before}, after: {processed_events_after}" ); metrics::assert_metrics_present(&vector_metrics_url, metrics::SOURCE_COMPLIANCE_METRICS) @@ -1999,14 +2007,14 @@ async fn host_metrics() -> Result<(), Box> { framework .wait_for_rollout( &namespace, - &format!("daemonset/{}", override_name), + &format!("daemonset/{override_name}"), vec!["--timeout=60s"], ) .await?; let mut vector_metrics_port_forward = framework.port_forward( &namespace, - &format!("daemonset/{}", override_name), + &format!("daemonset/{override_name}"), 9090, 9090, )?; diff --git a/lib/k8s-e2e-tests/tests/vector-aggregator.rs b/lib/k8s-e2e-tests/tests/vector-aggregator.rs index a3bb1201dd..07cb49ff4b 100644 --- a/lib/k8s-e2e-tests/tests/vector-aggregator.rs +++ b/lib/k8s-e2e-tests/tests/vector-aggregator.rs @@ -1,3 +1,5 @@ +#![allow(clippy::await_holding_lock)] + use k8s_e2e_tests::*; use k8s_test_framework::{lock, vector::Config as VectorConfig}; @@ -27,7 +29,7 @@ async fn dummy_topology() -> Result<(), Box> { framework .wait_for_rollout( &namespace, - &format!("statefulset/{}", override_name), + &format!("statefulset/{override_name}"), vec!["--timeout=60s"], ) .await?; @@ -62,14 +64,14 @@ async fn metrics_pipeline() -> Result<(), Box> { framework .wait_for_rollout( &namespace, - &format!("statefulset/{}", override_name), + &format!("statefulset/{override_name}"), vec!["--timeout=60s"], ) .await?; let mut vector_metrics_port_forward = framework.port_forward( &namespace, - &format!("statefulset/{}", override_name), + &format!("statefulset/{override_name}"), 9090, 9090, )?; diff --git a/lib/k8s-e2e-tests/tests/vector-dd-agent-aggregator.rs b/lib/k8s-e2e-tests/tests/vector-dd-agent-aggregator.rs index 55f4208d67..25c7dc4ce6 100644 --- a/lib/k8s-e2e-tests/tests/vector-dd-agent-aggregator.rs +++ b/lib/k8s-e2e-tests/tests/vector-dd-agent-aggregator.rs @@ -1,3 +1,5 @@ +#![allow(clippy::await_holding_lock)] + use indoc::indoc; use k8s_e2e_tests::*; use k8s_test_framework::{lock, namespace, test_pod, vector::Config as VectorConfig}; @@ -10,7 +12,7 @@ async fn datadog_to_vector() -> Result<(), Box> { let _guard = lock(); let namespace = get_namespace(); let override_name = get_override_name(&namespace, "vector-aggregator"); - let vector_endpoint = &format!("{}.{}.svc.cluster.local", override_name, namespace); + let vector_endpoint = &format!("{override_name}.{namespace}.svc.cluster.local"); let datadog_namespace = get_namespace_appended(&namespace, "datadog-agent"); let datadog_override_name = get_override_name(&namespace, "datadog-agent"); let pod_namespace = get_namespace_appended(&namespace, "test-pod"); @@ -66,7 +68,7 @@ async fn datadog_to_vector() -> Result<(), Box> { framework .wait_for_rollout( &namespace, - &format!("statefulset/{}", override_name), + &format!("statefulset/{override_name}"), vec!["--timeout=60s"], ) .await?; @@ -90,7 +92,7 @@ async fn datadog_to_vector() -> Result<(), Box> { framework .wait_for_rollout( &datadog_namespace, - &format!("daemonset/{}", datadog_override_name), + &format!("daemonset/{datadog_override_name}"), vec!["--timeout=60s"], ) .await?; @@ -114,7 +116,7 @@ async fn datadog_to_vector() -> Result<(), Box> { ))?) .await?; - let mut log_reader = framework.logs(&namespace, &format!("statefulset/{}", override_name))?; + let mut log_reader = framework.logs(&namespace, &format!("statefulset/{override_name}"))?; smoke_check_first_line(&mut log_reader).await; // Read the rest of the log lines. diff --git a/lib/k8s-e2e-tests/tests/vector.rs b/lib/k8s-e2e-tests/tests/vector.rs index e90e1c63e1..7d74442fb1 100644 --- a/lib/k8s-e2e-tests/tests/vector.rs +++ b/lib/k8s-e2e-tests/tests/vector.rs @@ -1,3 +1,5 @@ +#![allow(clippy::await_holding_lock)] + use indoc::{formatdoc, indoc}; use k8s_e2e_tests::*; use k8s_test_framework::{ @@ -85,7 +87,7 @@ async fn logs() -> Result<(), Box> { framework .wait_for_rollout( &namespace, - &format!("statefulset/aggregator-vector"), + "statefulset/aggregator-vector", vec!["--timeout=60s"], ) .await?; @@ -106,7 +108,7 @@ async fn logs() -> Result<(), Box> { framework .wait_for_rollout( &namespace, - &format!("daemonset/{}", agent_override_name), + &format!("daemonset/{agent_override_name}"), vec!["--timeout=60s"], ) .await?; @@ -136,7 +138,7 @@ async fn logs() -> Result<(), Box> { ) .await?; - let mut log_reader = framework.logs(&namespace, &format!("statefulset/aggregator-vector"))?; + let mut log_reader = framework.logs(&namespace, "statefulset/aggregator-vector")?; smoke_check_first_line(&mut log_reader).await; // Read the rest of the log lines. @@ -207,7 +209,7 @@ async fn haproxy() -> Result<(), Box> { framework .wait_for_rollout( &namespace, - &format!("statefulset/aggregator-vector"), + "statefulset/aggregator-vector", vec!["--timeout=60s"], ) .await?; @@ -215,7 +217,7 @@ async fn haproxy() -> Result<(), Box> { framework .wait_for_rollout( &namespace, - &format!("deployment/aggregator-vector-haproxy"), + "deployment/aggregator-vector-haproxy", vec!["--timeout=60s"], ) .await?; @@ -236,7 +238,7 @@ async fn haproxy() -> Result<(), Box> { framework .wait_for_rollout( &namespace, - &format!("daemonset/{}", agent_override_name), + &format!("daemonset/{agent_override_name}"), vec!["--timeout=60s"], ) .await?; @@ -266,7 +268,7 @@ async fn haproxy() -> Result<(), Box> { ) .await?; - let mut log_reader = framework.logs(&namespace, &format!("statefulset/aggregator-vector"))?; + let mut log_reader = framework.logs(&namespace, "statefulset/aggregator-vector")?; smoke_check_first_line(&mut log_reader).await; // Read the rest of the log lines. diff --git a/lib/k8s-test-framework/Cargo.toml b/lib/k8s-test-framework/Cargo.toml index 0253a05f23..982ccb68e0 100644 --- a/lib/k8s-test-framework/Cargo.toml +++ b/lib/k8s-test-framework/Cargo.toml @@ -2,7 +2,7 @@ name = "k8s-test-framework" version = "0.1.0" authors = ["Vector Contributors "] -edition = "2021" +edition = "2024" description = "Kubernetes Test Framework used to test Vector in Kubernetes" publish = false license = "MPL-2.0" @@ -10,6 +10,6 @@ license = "MPL-2.0" [dependencies] k8s-openapi = { version = "0.16.0", default-features = false, features = ["v1_19"] } serde_json.workspace = true -tempfile = "3" -tokio = { version = "1.43.0", features = ["full"] } +tempfile.workspace = true +tokio = { workspace = true, features = ["full"] } log = "0.4" diff --git a/lib/k8s-test-framework/src/framework.rs b/lib/k8s-test-framework/src/framework.rs index edc332fe06..fb9cf5e125 100644 --- a/lib/k8s-test-framework/src/framework.rs +++ b/lib/k8s-test-framework/src/framework.rs @@ -1,9 +1,9 @@ //! The test framework main entry point. use super::{ - exec_tail, kubernetes_version, log_lookup, namespace, pod, port_forward, restart_rollout, - test_pod, up_down, vector, wait_for_resource, wait_for_rollout, Interface, PortForwarder, - Reader, Result, + Interface, PortForwarder, Reader, Result, exec_tail, kubernetes_version, log_lookup, namespace, + pod, port_forward, restart_rollout, test_pod, up_down, vector, wait_for_resource, + wait_for_rollout, }; /// Framework wraps the interface to the system with an easy-to-use rust API diff --git a/lib/k8s-test-framework/src/helm_values_file.rs b/lib/k8s-test-framework/src/helm_values_file.rs index a539d608bf..e57e8a2395 100644 --- a/lib/k8s-test-framework/src/helm_values_file.rs +++ b/lib/k8s-test-framework/src/helm_values_file.rs @@ -9,7 +9,7 @@ pub struct HelmValuesFile(TempFile); impl HelmValuesFile { pub fn new(data: &str) -> std::io::Result { - info!("Using values \n {}", data); + info!("Using values \n {data}"); Ok(Self(TempFile::new("values.yml", data)?)) } diff --git a/lib/k8s-test-framework/src/lib.rs b/lib/k8s-test-framework/src/lib.rs index 5900d24d9f..96fcc4d893 100644 --- a/lib/k8s-test-framework/src/lib.rs +++ b/lib/k8s-test-framework/src/lib.rs @@ -45,8 +45,8 @@ pub use framework::Framework; pub use interface::Interface; pub use lock::lock; use log_lookup::log_lookup; -use port_forward::port_forward; pub use port_forward::PortForwarder; +use port_forward::port_forward; pub use reader::Reader; pub use test_pod::CommandBuilder; pub use up_down::Manager; diff --git a/lib/k8s-test-framework/src/namespace.rs b/lib/k8s-test-framework/src/namespace.rs index b1c2504340..fd2fa079bc 100644 --- a/lib/k8s-test-framework/src/namespace.rs +++ b/lib/k8s-test-framework/src/namespace.rs @@ -7,7 +7,7 @@ use std::{ use k8s_openapi::{api::core::v1::Namespace, apimachinery::pkg::apis::meta::v1::ObjectMeta}; -use super::{resource_file::ResourceFile, Result}; +use super::{Result, resource_file::ResourceFile}; use crate::up_down; /// A config that holds a test `Namespace` resource file. diff --git a/lib/k8s-test-framework/src/port_forward.rs b/lib/k8s-test-framework/src/port_forward.rs index b2acefa71f..457bce8169 100644 --- a/lib/k8s-test-framework/src/port_forward.rs +++ b/lib/k8s-test-framework/src/port_forward.rs @@ -33,7 +33,7 @@ pub fn port_forward( command.arg("port-forward"); command.arg("-n").arg(namespace); command.arg(resource); - command.arg(format!("{}:{}", local_port, resource_port)); + command.arg(format!("{local_port}:{resource_port}")); command.kill_on_drop(true); diff --git a/lib/k8s-test-framework/src/reader.rs b/lib/k8s-test-framework/src/reader.rs index 0b83d5885a..e6af49066a 100644 --- a/lib/k8s-test-framework/src/reader.rs +++ b/lib/k8s-test-framework/src/reader.rs @@ -98,7 +98,7 @@ mod tests { let mut expected_num = 0; while let Some(line) = reader.read_line().await { // Assert we're getting expected lines. - assert_eq!(line, format!("Line {}\n", expected_num)); + assert_eq!(line, format!("Line {expected_num}\n")); // On line 100 issue a `kill` to stop the infinite stream. if expected_num == 100 { diff --git a/lib/k8s-test-framework/src/temp_file.rs b/lib/k8s-test-framework/src/temp_file.rs index d1535b6b5a..567de01031 100644 --- a/lib/k8s-test-framework/src/temp_file.rs +++ b/lib/k8s-test-framework/src/temp_file.rs @@ -9,7 +9,7 @@ pub struct TempFile { impl TempFile { pub fn new(file_name: &str, data: &str) -> std::io::Result { - let dir = tempdir()?.into_path(); + let dir = tempdir()?.keep(); let path = dir.join(file_name); std::fs::write(&path, data)?; Ok(Self { path }) diff --git a/lib/k8s-test-framework/src/test_pod.rs b/lib/k8s-test-framework/src/test_pod.rs index 6ac2125eec..523b0fdd8f 100644 --- a/lib/k8s-test-framework/src/test_pod.rs +++ b/lib/k8s-test-framework/src/test_pod.rs @@ -4,7 +4,7 @@ use std::process::{Command, Stdio}; use k8s_openapi::api::core::v1::Pod; -use super::{resource_file::ResourceFile, Result}; +use super::{Result, resource_file::ResourceFile}; use crate::up_down; /// A config that holds a test `Pod` resource file. diff --git a/lib/k8s-test-framework/src/util.rs b/lib/k8s-test-framework/src/util.rs index a0499ba6fc..67e8849809 100644 --- a/lib/k8s-test-framework/src/util.rs +++ b/lib/k8s-test-framework/src/util.rs @@ -3,28 +3,28 @@ use log::info; use crate::Result; pub async fn run_command(mut command: tokio::process::Command) -> Result<()> { - info!("Running command `{:?}`", command); + info!("Running command `{command:?}`"); let exit_status = command.spawn()?.wait().await?; if !exit_status.success() { - return Err(format!("exec failed: {:?}", command).into()); + return Err(format!("exec failed: {command:?}").into()); } Ok(()) } pub fn run_command_blocking(mut command: std::process::Command) -> Result<()> { - info!("Running command blocking `{:?}`", command); + info!("Running command blocking `{command:?}`"); let exit_status = command.spawn()?.wait()?; if !exit_status.success() { - return Err(format!("exec failed: {:?}", command).into()); + return Err(format!("exec failed: {command:?}").into()); } Ok(()) } pub async fn run_command_output(mut command: tokio::process::Command) -> Result { - info!("Fetching command `{:?}`", command); + info!("Fetching command `{command:?}`"); let output = command.spawn()?.wait_with_output().await?; if !output.status.success() { - return Err(format!("exec failed: {:?}", command).into()); + return Err(format!("exec failed: {command:?}").into()); } let output = String::from_utf8(output.stdout)?; diff --git a/lib/k8s-test-framework/src/vector.rs b/lib/k8s-test-framework/src/vector.rs index 6de33f519e..8b5dc27fa3 100644 --- a/lib/k8s-test-framework/src/vector.rs +++ b/lib/k8s-test-framework/src/vector.rs @@ -2,7 +2,7 @@ use std::process::{Command, Stdio}; -use crate::{helm_values_file::HelmValuesFile, resource_file::ResourceFile, up_down, Result}; +use crate::{Result, helm_values_file::HelmValuesFile, resource_file::ResourceFile, up_down}; /// Parameters required to build `kubectl` & `helm` commands to manage charts deployments in the /// Kubernetes cluster. diff --git a/lib/k8s-test-framework/src/wait_for_resource.rs b/lib/k8s-test-framework/src/wait_for_resource.rs index d3cba0fd56..7beb0d7721 100644 --- a/lib/k8s-test-framework/src/wait_for_resource.rs +++ b/lib/k8s-test-framework/src/wait_for_resource.rs @@ -86,7 +86,7 @@ where command.arg("--for"); match wait_for { WaitFor::Delete => command.arg("delete"), - WaitFor::Condition(cond) => command.arg(format!("condition={}", cond)), + WaitFor::Condition(cond) => command.arg(format!("condition={cond}")), }; command.args(extra); diff --git a/lib/loki-logproto/Cargo.toml b/lib/loki-logproto/Cargo.toml index 934507ed3a..4547fc83eb 100644 --- a/lib/loki-logproto/Cargo.toml +++ b/lib/loki-logproto/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "loki-logproto" version = "0.1.0" -edition = "2021" +edition = "2024" license = "MPL-2.0" publish = false @@ -15,4 +15,4 @@ prost-types.workspace = true chrono.workspace = true [build-dependencies] -prost-build .workspace = true +prost-build.workspace = true diff --git a/lib/loki-logproto/src/lib.rs b/lib/loki-logproto/src/lib.rs index 81821dad7d..9aae27cf79 100644 --- a/lib/loki-logproto/src/lib.rs +++ b/lib/loki-logproto/src/lib.rs @@ -83,7 +83,7 @@ pub mod util { let mut labels: Vec = labels .iter() .filter(|(k, _)| !RESERVED_LABELS.contains(&k.as_str())) - .map(|(k, v)| format!("{}=\"{}\"", k, v)) + .map(|(k, v)| format!("{k}=\"{v}\"")) .collect(); labels.sort(); format!("{{{}}}", labels.join(", ")) diff --git a/lib/opentelemetry-proto/Cargo.toml b/lib/opentelemetry-proto/Cargo.toml index 2c17bb384e..1467a17112 100644 --- a/lib/opentelemetry-proto/Cargo.toml +++ b/lib/opentelemetry-proto/Cargo.toml @@ -2,20 +2,21 @@ name = "opentelemetry-proto" version = "0.1.0" authors = ["Vector Contributors "] -edition = "2021" +edition = "2024" publish = false [build-dependencies] prost-build.workspace = true tonic-build.workspace = true +glob.workspace = true [dependencies] -bytes = { version = "1.9.0", default-features = false, features = ["serde"] } +bytes.workspace = true chrono.workspace = true hex = { version = "0.4.3", default-features = false, features = ["std"] } lookup = { package = "vector-lookup", path = "../vector-lookup", default-features = false } -ordered-float = { version = "4.6.0", default-features = false } -prost .workspace = true +ordered-float.workspace = true +prost.workspace = true tonic.workspace = true vrl.workspace = true vector-core = { path = "../vector-core", default-features = false } diff --git a/lib/opentelemetry-proto/build.rs b/lib/opentelemetry-proto/build.rs index beaf9f4c41..7e5fcb5f40 100644 --- a/lib/opentelemetry-proto/build.rs +++ b/lib/opentelemetry-proto/build.rs @@ -1,20 +1,42 @@ -use std::io::Error; +use glob::glob; +use std::fs::{read_to_string, write}; +use std::path::Path; +use std::{io::Result, path::PathBuf}; + +fn main() -> Result<()> { + let proto_root = PathBuf::from("src/proto/opentelemetry-proto"); + let include_path = proto_root.clone(); + + let proto_paths: Vec<_> = glob(&format!("{}/**/*.proto", proto_root.display())) + .expect("Failed to read glob pattern") + .filter_map(|result| result.ok()) + .collect(); + + let out_dir = PathBuf::from(std::env::var("OUT_DIR").unwrap()); + let descriptor_path = out_dir.join("opentelemetry-proto.desc"); -fn main() -> Result<(), Error> { tonic_build::configure() .build_client(true) .build_server(true) - .compile( - &[ - "src/proto/opentelemetry-proto/opentelemetry/proto/common/v1/common.proto", - "src/proto/opentelemetry-proto/opentelemetry/proto/resource/v1/resource.proto", - "src/proto/opentelemetry-proto/opentelemetry/proto/logs/v1/logs.proto", - "src/proto/opentelemetry-proto/opentelemetry/proto/trace/v1/trace.proto", - "src/proto/opentelemetry-proto/opentelemetry/proto/collector/trace/v1/trace_service.proto", - "src/proto/opentelemetry-proto/opentelemetry/proto/collector/logs/v1/logs_service.proto", - ], - &["src/proto/opentelemetry-proto"], - )?; + .file_descriptor_set_path(&descriptor_path) + .compile(&proto_paths, &[include_path])?; + + write_static_descriptor_reference(&descriptor_path, &out_dir)?; + + Ok(()) +} + +fn write_static_descriptor_reference(descriptor_path: &Path, out_dir: &Path) -> Result<()> { + let include_line = format!( + "pub static DESCRIPTOR_BYTES: &[u8] = include_bytes!(r\"{}\");\n", + descriptor_path.display() + ); + + let include_file = out_dir.join("opentelemetry-proto.rs"); + let existing = read_to_string(&include_file).ok(); + if existing.as_deref() != Some(&include_line) { + write(&include_file, include_line)?; + } Ok(()) } diff --git a/lib/opentelemetry-proto/src/common.rs b/lib/opentelemetry-proto/src/common.rs new file mode 100644 index 0000000000..a8ac10e71d --- /dev/null +++ b/lib/opentelemetry-proto/src/common.rs @@ -0,0 +1,94 @@ +use super::proto::common::v1::{KeyValue, any_value::Value as PBValue}; +use bytes::Bytes; +use ordered_float::NotNan; +use vector_core::event::metric::TagValue; +use vrl::value::{ObjectMap, Value}; + +impl From for Value { + fn from(av: PBValue) -> Self { + match av { + PBValue::StringValue(v) => Value::Bytes(Bytes::from(v)), + PBValue::BoolValue(v) => Value::Boolean(v), + PBValue::IntValue(v) => Value::Integer(v), + PBValue::DoubleValue(v) => NotNan::new(v).map(Value::Float).unwrap_or(Value::Null), + PBValue::BytesValue(v) => Value::Bytes(Bytes::from(v)), + PBValue::ArrayValue(arr) => Value::Array( + arr.values + .into_iter() + .map(|av| av.value.map(Into::into).unwrap_or(Value::Null)) + .collect::>(), + ), + PBValue::KvlistValue(arr) => kv_list_into_value(arr.values), + } + } +} + +impl From for TagValue { + fn from(pb: PBValue) -> Self { + match pb { + PBValue::StringValue(s) => TagValue::from(s), + PBValue::BoolValue(b) => TagValue::from(b.to_string()), + PBValue::IntValue(i) => TagValue::from(i.to_string()), + PBValue::DoubleValue(f) => TagValue::from(f.to_string()), + PBValue::BytesValue(b) => TagValue::from(String::from_utf8_lossy(&b).to_string()), + _ => TagValue::from("null"), + } + } +} + +pub fn kv_list_into_value(arr: Vec) -> Value { + Value::Object( + arr.into_iter() + .filter_map(|kv| { + kv.value.map(|av| { + ( + kv.key.into(), + av.value.map(Into::into).unwrap_or(Value::Null), + ) + }) + }) + .collect::(), + ) +} + +pub fn to_hex(d: &[u8]) -> String { + if d.is_empty() { + return "".to_string(); + } + hex::encode(d) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_pb_double_value_nan_handling() { + // Test that NaN values are converted to Value::Null instead of panicking + let nan_value = PBValue::DoubleValue(f64::NAN); + let result = Value::from(nan_value); + assert_eq!(result, Value::Null); + } + + #[test] + fn test_pb_double_value_infinity() { + // Test that infinity values work correctly + let inf_value = PBValue::DoubleValue(f64::INFINITY); + let result = Value::from(inf_value); + match result { + Value::Float(f) => { + assert!(f.into_inner().is_infinite() && f.into_inner().is_sign_positive()) + } + _ => panic!("Expected Float value, got {result:?}"), + } + + let neg_inf_value = PBValue::DoubleValue(f64::NEG_INFINITY); + let result = Value::from(neg_inf_value); + match result { + Value::Float(f) => { + assert!(f.into_inner().is_infinite() && f.into_inner().is_sign_negative()) + } + _ => panic!("Expected Float value, got {result:?}"), + } + } +} diff --git a/lib/opentelemetry-proto/src/lib.rs b/lib/opentelemetry-proto/src/lib.rs index 23dff2fcdc..0a3d6b89a8 100644 --- a/lib/opentelemetry-proto/src/lib.rs +++ b/lib/opentelemetry-proto/src/lib.rs @@ -1,3 +1,6 @@ -pub mod convert; +pub mod common; +pub mod logs; +pub mod metrics; #[allow(warnings)] // Ignore some clippy warnings pub mod proto; +pub mod spans; diff --git a/lib/opentelemetry-proto/src/convert.rs b/lib/opentelemetry-proto/src/logs.rs similarity index 54% rename from lib/opentelemetry-proto/src/convert.rs rename to lib/opentelemetry-proto/src/logs.rs index 590abfebfd..9fdc134fe5 100644 --- a/lib/opentelemetry-proto/src/convert.rs +++ b/lib/opentelemetry-proto/src/logs.rs @@ -1,30 +1,19 @@ +use super::common::{kv_list_into_value, to_hex}; +use crate::proto::{ + common::v1::{InstrumentationScope, any_value::Value as PBValue}, + logs::v1::{LogRecord, ResourceLogs, SeverityNumber}, + resource::v1::Resource, +}; use bytes::Bytes; use chrono::{DateTime, TimeZone, Utc}; -use lookup::path; -use ordered_float::NotNan; -use std::collections::BTreeMap; use vector_core::{ - config::{log_schema, LegacyKey, LogNamespace}, - event::{Event, LogEvent, TraceEvent}, -}; -use vrl::value::KeyString; -use vrl::{ - event_path, - value::{ObjectMap, Value}, -}; - -use super::proto::{ - common::v1::{any_value::Value as PBValue, InstrumentationScope, KeyValue}, - logs::v1::{LogRecord, ResourceLogs, SeverityNumber}, - resource::v1::Resource, - trace::v1::{ - span::{Event as SpanEvent, Link}, - ResourceSpans, Span, Status as SpanStatus, - }, + config::{LegacyKey, LogNamespace, log_schema}, + event::{Event, LogEvent}, }; +use vrl::core::Value; +use vrl::path; const SOURCE_NAME: &str = "opentelemetry"; - pub const RESOURCE_KEY: &str = "resources"; pub const ATTRIBUTES_KEY: &str = "attributes"; pub const SCOPE_KEY: &str = "scope"; @@ -57,146 +46,12 @@ impl ResourceLogs { } } -impl ResourceSpans { - pub fn into_event_iter(self) -> impl Iterator { - let resource = self.resource; - let now = Utc::now(); - - self.scope_spans - .into_iter() - .flat_map(|instrumentation_library_spans| instrumentation_library_spans.spans) - .map(move |span| { - ResourceSpan { - resource: resource.clone(), - span, - } - .into_event(now) - }) - } -} - -impl From for Value { - fn from(av: PBValue) -> Self { - match av { - PBValue::StringValue(v) => Value::Bytes(Bytes::from(v)), - PBValue::BoolValue(v) => Value::Boolean(v), - PBValue::IntValue(v) => Value::Integer(v), - PBValue::DoubleValue(v) => Value::Float(NotNan::new(v).unwrap()), - PBValue::BytesValue(v) => Value::Bytes(Bytes::from(v)), - PBValue::ArrayValue(arr) => Value::Array( - arr.values - .into_iter() - .map(|av| av.value.map(Into::into).unwrap_or(Value::Null)) - .collect::>(), - ), - PBValue::KvlistValue(arr) => kv_list_into_value(arr.values), - } - } -} - struct ResourceLog { resource: Option, scope: Option, log_record: LogRecord, } -struct ResourceSpan { - resource: Option, - span: Span, -} - -fn kv_list_into_value(arr: Vec) -> Value { - Value::Object( - arr.into_iter() - .filter_map(|kv| { - kv.value.map(|av| { - ( - kv.key.into(), - av.value.map(Into::into).unwrap_or(Value::Null), - ) - }) - }) - .collect::(), - ) -} - -fn to_hex(d: &[u8]) -> String { - if d.is_empty() { - return "".to_string(); - } - hex::encode(d) -} - -// Unlike log events(log body + metadata), trace spans are just metadata, so we don't handle log_namespace here, -// insert all attributes into log root, just like what datadog_agent/traces does. -impl ResourceSpan { - fn into_event(self, now: DateTime) -> Event { - let mut trace = TraceEvent::default(); - let span = self.span; - trace.insert( - event_path!(TRACE_ID_KEY), - Value::from(to_hex(&span.trace_id)), - ); - trace.insert(event_path!(SPAN_ID_KEY), Value::from(to_hex(&span.span_id))); - trace.insert(event_path!("trace_state"), span.trace_state); - trace.insert( - event_path!("parent_span_id"), - Value::from(to_hex(&span.parent_span_id)), - ); - trace.insert(event_path!("name"), span.name); - trace.insert(event_path!("kind"), span.kind); - trace.insert( - event_path!("start_time_unix_nano"), - Value::from(Utc.timestamp_nanos(span.start_time_unix_nano as i64)), - ); - trace.insert( - event_path!("end_time_unix_nano"), - Value::from(Utc.timestamp_nanos(span.end_time_unix_nano as i64)), - ); - if !span.attributes.is_empty() { - trace.insert( - event_path!(ATTRIBUTES_KEY), - kv_list_into_value(span.attributes), - ); - } - trace.insert( - event_path!(DROPPED_ATTRIBUTES_COUNT_KEY), - Value::from(span.dropped_attributes_count), - ); - if !span.events.is_empty() { - trace.insert( - event_path!("events"), - Value::Array(span.events.into_iter().map(Into::into).collect()), - ); - } - trace.insert( - event_path!("dropped_events_count"), - Value::from(span.dropped_events_count), - ); - if !span.links.is_empty() { - trace.insert( - event_path!("links"), - Value::Array(span.links.into_iter().map(Into::into).collect()), - ); - } - trace.insert( - event_path!("dropped_links_count"), - Value::from(span.dropped_links_count), - ); - trace.insert(event_path!("status"), Value::from(span.status)); - if let Some(resource) = self.resource { - if !resource.attributes.is_empty() { - trace.insert( - event_path!(RESOURCE_KEY), - kv_list_into_value(resource.attributes), - ); - } - } - trace.insert(event_path!("ingest_timestamp"), Value::from(now)); - trace.into() - } -} - // https://github.com/open-telemetry/opentelemetry-specification/blob/v1.15.0/specification/logs/data-model.md impl ResourceLog { fn into_event(self, log_namespace: LogNamespace, now: DateTime) -> Event { @@ -261,16 +116,16 @@ impl ResourceLog { } // Optional fields - if let Some(resource) = self.resource { - if !resource.attributes.is_empty() { - log_namespace.insert_source_metadata( - SOURCE_NAME, - &mut log, - Some(LegacyKey::Overwrite(path!(RESOURCE_KEY))), - path!(RESOURCE_KEY), - kv_list_into_value(resource.attributes), - ); - } + if let Some(resource) = self.resource + && !resource.attributes.is_empty() + { + log_namespace.insert_source_metadata( + SOURCE_NAME, + &mut log, + Some(LegacyKey::Overwrite(path!(RESOURCE_KEY))), + path!(RESOURCE_KEY), + kv_list_into_value(resource.attributes), + ); } if !self.log_record.attributes.is_empty() { log_namespace.insert_source_metadata( @@ -381,44 +236,3 @@ impl ResourceLog { log.into() } } - -impl From for Value { - fn from(ev: SpanEvent) -> Self { - let mut obj: BTreeMap = BTreeMap::new(); - obj.insert("name".into(), ev.name.into()); - obj.insert( - "time_unix_nano".into(), - Value::Timestamp(Utc.timestamp_nanos(ev.time_unix_nano as i64)), - ); - obj.insert("attributes".into(), kv_list_into_value(ev.attributes)); - obj.insert( - "dropped_attributes_count".into(), - Value::Integer(ev.dropped_attributes_count as i64), - ); - Value::Object(obj) - } -} - -impl From for Value { - fn from(link: Link) -> Self { - let mut obj: BTreeMap = BTreeMap::new(); - obj.insert("trace_id".into(), Value::from(to_hex(&link.trace_id))); - obj.insert("span_id".into(), Value::from(to_hex(&link.span_id))); - obj.insert("trace_state".into(), link.trace_state.into()); - obj.insert("attributes".into(), kv_list_into_value(link.attributes)); - obj.insert( - "dropped_attributes_count".into(), - Value::Integer(link.dropped_attributes_count as i64), - ); - Value::Object(obj) - } -} - -impl From for Value { - fn from(status: SpanStatus) -> Self { - let mut obj: BTreeMap = BTreeMap::new(); - obj.insert("message".into(), status.message.into()); - obj.insert("code".into(), status.code.into()); - Value::Object(obj) - } -} diff --git a/lib/opentelemetry-proto/src/metrics.rs b/lib/opentelemetry-proto/src/metrics.rs new file mode 100644 index 0000000000..3fcc9a5c4a --- /dev/null +++ b/lib/opentelemetry-proto/src/metrics.rs @@ -0,0 +1,441 @@ +use super::proto::{ + common::v1::{InstrumentationScope, KeyValue}, + metrics::v1::{ + AggregationTemporality, ExponentialHistogram, ExponentialHistogramDataPoint, Gauge, + Histogram, HistogramDataPoint, NumberDataPoint, ResourceMetrics, Sum, Summary, + SummaryDataPoint, metric::Data, number_data_point::Value as NumberDataPointValue, + }, + resource::v1::Resource, +}; +use chrono::{TimeZone, Utc}; +use vector_core::event::{ + Event, Metric as MetricEvent, MetricKind, MetricTags, MetricValue, + metric::{Bucket, Quantile, TagValue}, +}; + +impl ResourceMetrics { + pub fn into_event_iter(self) -> impl Iterator { + let resource = self.resource.clone(); + + self.scope_metrics + .into_iter() + .flat_map(move |scope_metrics| { + let scope = scope_metrics.scope; + let resource = resource.clone(); + + scope_metrics.metrics.into_iter().flat_map(move |metric| { + let metric_name = metric.name.clone(); + match metric.data { + Some(Data::Gauge(g)) => { + Self::convert_gauge(g, &resource, &scope, &metric_name) + } + Some(Data::Sum(s)) => Self::convert_sum(s, &resource, &scope, &metric_name), + Some(Data::Histogram(h)) => { + Self::convert_histogram(h, &resource, &scope, &metric_name) + } + Some(Data::ExponentialHistogram(e)) => { + Self::convert_exp_histogram(e, &resource, &scope, &metric_name) + } + Some(Data::Summary(su)) => { + Self::convert_summary(su, &resource, &scope, &metric_name) + } + _ => Vec::new(), + } + }) + }) + } + + fn convert_gauge( + gauge: Gauge, + resource: &Option, + scope: &Option, + metric_name: &str, + ) -> Vec { + let resource = resource.clone(); + let scope = scope.clone(); + let metric_name = metric_name.to_string(); + + gauge + .data_points + .into_iter() + .map(move |point| { + GaugeMetric { + resource: resource.clone(), + scope: scope.clone(), + point, + } + .into_metric(metric_name.clone()) + }) + .collect() + } + + fn convert_sum( + sum: Sum, + resource: &Option, + scope: &Option, + metric_name: &str, + ) -> Vec { + let resource = resource.clone(); + let scope = scope.clone(); + let metric_name = metric_name.to_string(); + + sum.data_points + .into_iter() + .map(move |point| { + SumMetric { + aggregation_temporality: sum.aggregation_temporality, + resource: resource.clone(), + scope: scope.clone(), + is_monotonic: sum.is_monotonic, + point, + } + .into_metric(metric_name.clone()) + }) + .collect() + } + + fn convert_histogram( + histogram: Histogram, + resource: &Option, + scope: &Option, + metric_name: &str, + ) -> Vec { + let resource = resource.clone(); + let scope = scope.clone(); + let metric_name = metric_name.to_string(); + + histogram + .data_points + .into_iter() + .map(move |point| { + HistogramMetric { + aggregation_temporality: histogram.aggregation_temporality, + resource: resource.clone(), + scope: scope.clone(), + point, + } + .into_metric(metric_name.clone()) + }) + .collect() + } + + fn convert_exp_histogram( + histogram: ExponentialHistogram, + resource: &Option, + scope: &Option, + metric_name: &str, + ) -> Vec { + let resource = resource.clone(); + let scope = scope.clone(); + let metric_name = metric_name.to_string(); + + histogram + .data_points + .into_iter() + .map(move |point| { + ExpHistogramMetric { + aggregation_temporality: histogram.aggregation_temporality, + resource: resource.clone(), + scope: scope.clone(), + point, + } + .into_metric(metric_name.clone()) + }) + .collect() + } + + fn convert_summary( + summary: Summary, + resource: &Option, + scope: &Option, + metric_name: &str, + ) -> Vec { + let resource = resource.clone(); + let scope = scope.clone(); + let metric_name = metric_name.to_string(); + + summary + .data_points + .into_iter() + .map(move |point| { + SummaryMetric { + resource: resource.clone(), + scope: scope.clone(), + point, + } + .into_metric(metric_name.clone()) + }) + .collect() + } +} + +struct GaugeMetric { + resource: Option, + scope: Option, + point: NumberDataPoint, +} + +struct SumMetric { + aggregation_temporality: i32, + resource: Option, + scope: Option, + point: NumberDataPoint, + is_monotonic: bool, +} + +struct SummaryMetric { + resource: Option, + scope: Option, + point: SummaryDataPoint, +} + +struct HistogramMetric { + aggregation_temporality: i32, + resource: Option, + scope: Option, + point: HistogramDataPoint, +} + +struct ExpHistogramMetric { + aggregation_temporality: i32, + resource: Option, + scope: Option, + point: ExponentialHistogramDataPoint, +} + +pub fn build_metric_tags( + resource: Option, + scope: Option, + attributes: &[KeyValue], +) -> MetricTags { + let mut tags = MetricTags::default(); + + if let Some(res) = resource { + for attr in res.attributes { + if let Some(value) = &attr.value + && let Some(pb_value) = &value.value + { + tags.insert( + format!("resource.{}", attr.key.clone()), + TagValue::from(pb_value.clone()), + ); + } + } + } + + if let Some(scope) = scope { + if !scope.name.is_empty() { + tags.insert("scope.name".to_string(), scope.name); + } + if !scope.version.is_empty() { + tags.insert("scope.version".to_string(), scope.version); + } + for attr in scope.attributes { + if let Some(value) = &attr.value + && let Some(pb_value) = &value.value + { + tags.insert( + format!("scope.{}", attr.key.clone()), + TagValue::from(pb_value.clone()), + ); + } + } + } + + for attr in attributes { + if let Some(value) = &attr.value + && let Some(pb_value) = &value.value + { + tags.insert(attr.key.clone(), TagValue::from(pb_value.clone())); + } + } + + tags +} + +impl SumMetric { + fn into_metric(self, metric_name: String) -> Event { + let timestamp = Some(Utc.timestamp_nanos(self.point.time_unix_nano as i64)); + let value = self.point.value.to_f64().unwrap_or(0.0); + let attributes = build_metric_tags(self.resource, self.scope, &self.point.attributes); + let kind = if self.aggregation_temporality == AggregationTemporality::Delta as i32 { + MetricKind::Incremental + } else { + MetricKind::Absolute + }; + + // as per otel doc non_monotonic sum would be better transformed to gauge in time-series + let metric_value = if self.is_monotonic { + MetricValue::Counter { value } + } else { + MetricValue::Gauge { value } + }; + + MetricEvent::new(metric_name, kind, metric_value) + .with_tags(Some(attributes)) + .with_timestamp(timestamp) + .into() + } +} + +impl GaugeMetric { + fn into_metric(self, metric_name: String) -> Event { + let timestamp = Some(Utc.timestamp_nanos(self.point.time_unix_nano as i64)); + let value = self.point.value.to_f64().unwrap_or(0.0); + let attributes = build_metric_tags(self.resource, self.scope, &self.point.attributes); + + MetricEvent::new( + metric_name, + MetricKind::Absolute, + MetricValue::Gauge { value }, + ) + .with_timestamp(timestamp) + .with_tags(Some(attributes)) + .into() + } +} + +impl HistogramMetric { + fn into_metric(self, metric_name: String) -> Event { + let timestamp = Some(Utc.timestamp_nanos(self.point.time_unix_nano as i64)); + let attributes = build_metric_tags(self.resource, self.scope, &self.point.attributes); + let buckets = match self.point.bucket_counts.len() { + 0 => Vec::new(), + n => { + let mut buckets = Vec::with_capacity(n); + + for (i, &count) in self.point.bucket_counts.iter().enumerate() { + // there are n+1 buckets, since we have -Inf, +Inf on the sides + let upper_limit = self + .point + .explicit_bounds + .get(i) + .copied() + .unwrap_or(f64::INFINITY); + buckets.push(Bucket { count, upper_limit }); + } + + buckets + } + }; + + let kind = if self.aggregation_temporality == AggregationTemporality::Delta as i32 { + MetricKind::Incremental + } else { + MetricKind::Absolute + }; + + MetricEvent::new( + metric_name, + kind, + MetricValue::AggregatedHistogram { + buckets, + count: self.point.count, + sum: self.point.sum.unwrap_or(0.0), + }, + ) + .with_timestamp(timestamp) + .with_tags(Some(attributes)) + .into() + } +} + +impl ExpHistogramMetric { + fn into_metric(self, metric_name: String) -> Event { + // we have to convert Exponential Histogram to agg histogram using scale and base + let timestamp = Some(Utc.timestamp_nanos(self.point.time_unix_nano as i64)); + let attributes = build_metric_tags(self.resource, self.scope, &self.point.attributes); + + let scale = self.point.scale; + // from Opentelemetry docs: base = 2**(2**(-scale)) + let base = 2f64.powf(2f64.powi(-scale)); + + let mut buckets = Vec::new(); + + if let Some(negative_buckets) = self.point.negative { + for (i, &count) in negative_buckets.bucket_counts.iter().enumerate() { + let index = negative_buckets.offset + i as i32; + let upper_limit = -base.powi(index); + buckets.push(Bucket { count, upper_limit }); + } + } + + if self.point.zero_count > 0 { + buckets.push(Bucket { + count: self.point.zero_count, + upper_limit: 0.0, + }); + } + + if let Some(positive_buckets) = self.point.positive { + for (i, &count) in positive_buckets.bucket_counts.iter().enumerate() { + let index = positive_buckets.offset + i as i32; + let upper_limit = base.powi(index + 1); + buckets.push(Bucket { count, upper_limit }); + } + } + + let kind = if self.aggregation_temporality == AggregationTemporality::Delta as i32 { + MetricKind::Incremental + } else { + MetricKind::Absolute + }; + + MetricEvent::new( + metric_name, + kind, + MetricValue::AggregatedHistogram { + buckets, + count: self.point.count, + sum: self.point.sum.unwrap_or(0.0), + }, + ) + .with_timestamp(timestamp) + .with_tags(Some(attributes)) + .into() + } +} + +impl SummaryMetric { + fn into_metric(self, metric_name: String) -> Event { + let timestamp = Some(Utc.timestamp_nanos(self.point.time_unix_nano as i64)); + let attributes = build_metric_tags(self.resource, self.scope, &self.point.attributes); + + let quantiles: Vec = self + .point + .quantile_values + .iter() + .map(|q| Quantile { + quantile: q.quantile, + value: q.value, + }) + .collect(); + + MetricEvent::new( + metric_name, + MetricKind::Absolute, + MetricValue::AggregatedSummary { + quantiles, + count: self.point.count, + sum: self.point.sum, + }, + ) + .with_timestamp(timestamp) + .with_tags(Some(attributes)) + .into() + } +} + +pub trait ToF64 { + fn to_f64(self) -> Option; +} + +impl ToF64 for Option { + fn to_f64(self) -> Option { + match self { + Some(NumberDataPointValue::AsDouble(f)) => Some(f), + Some(NumberDataPointValue::AsInt(i)) => Some(i as f64), + None => None, + } + } +} diff --git a/lib/opentelemetry-proto/src/proto.rs b/lib/opentelemetry-proto/src/proto.rs index e77748c1c1..5559113bd1 100644 --- a/lib/opentelemetry-proto/src/proto.rs +++ b/lib/opentelemetry-proto/src/proto.rs @@ -10,6 +10,11 @@ pub mod collector { tonic::include_proto!("opentelemetry.proto.collector.logs.v1"); } } + pub mod metrics { + pub mod v1 { + tonic::include_proto!("opentelemetry.proto.collector.metrics.v1"); + } + } } /// Common types used across all event types. @@ -26,6 +31,13 @@ pub mod logs { } } +/// Generated types used for metrics. +pub mod metrics { + pub mod v1 { + tonic::include_proto!("opentelemetry.proto.metrics.v1"); + } +} + /// Generated types used for trace. pub mod trace { pub mod v1 { @@ -39,3 +51,6 @@ pub mod resource { tonic::include_proto!("opentelemetry.proto.resource.v1"); } } + +/// The raw descriptor bytes for all the above. +include!(concat!(env!("OUT_DIR"), "/opentelemetry-proto.rs")); diff --git a/lib/opentelemetry-proto/src/spans.rs b/lib/opentelemetry-proto/src/spans.rs new file mode 100644 index 0000000000..8cacd2d9d1 --- /dev/null +++ b/lib/opentelemetry-proto/src/spans.rs @@ -0,0 +1,155 @@ +use super::common::{kv_list_into_value, to_hex}; +use super::proto::{ + resource::v1::Resource, + trace::v1::{ + ResourceSpans, Span, Status as SpanStatus, + span::{Event as SpanEvent, Link}, + }, +}; +use chrono::{DateTime, TimeZone, Utc}; +use std::collections::BTreeMap; +use vector_core::event::{Event, TraceEvent}; +use vrl::{ + event_path, + value::{KeyString, Value}, +}; + +pub const TRACE_ID_KEY: &str = "trace_id"; +pub const SPAN_ID_KEY: &str = "span_id"; +pub const DROPPED_ATTRIBUTES_COUNT_KEY: &str = "dropped_attributes_count"; +pub const RESOURCE_KEY: &str = "resources"; +pub const ATTRIBUTES_KEY: &str = "attributes"; + +impl ResourceSpans { + pub fn into_event_iter(self) -> impl Iterator { + let resource = self.resource; + let now = Utc::now(); + + self.scope_spans + .into_iter() + .flat_map(|instrumentation_library_spans| instrumentation_library_spans.spans) + .map(move |span| { + ResourceSpan { + resource: resource.clone(), + span, + } + .into_event(now) + }) + } +} + +struct ResourceSpan { + resource: Option, + span: Span, +} + +// Unlike log events(log body + metadata), trace spans are just metadata, so we don't handle log_namespace here, +// insert all attributes into log root, just like what datadog_agent/traces does. +impl ResourceSpan { + fn into_event(self, now: DateTime) -> Event { + let mut trace = TraceEvent::default(); + let span = self.span; + trace.insert( + event_path!(TRACE_ID_KEY), + Value::from(to_hex(&span.trace_id)), + ); + trace.insert(event_path!(SPAN_ID_KEY), Value::from(to_hex(&span.span_id))); + trace.insert(event_path!("trace_state"), span.trace_state); + trace.insert( + event_path!("parent_span_id"), + Value::from(to_hex(&span.parent_span_id)), + ); + trace.insert(event_path!("name"), span.name); + trace.insert(event_path!("kind"), span.kind); + trace.insert( + event_path!("start_time_unix_nano"), + Value::from(Utc.timestamp_nanos(span.start_time_unix_nano as i64)), + ); + trace.insert( + event_path!("end_time_unix_nano"), + Value::from(Utc.timestamp_nanos(span.end_time_unix_nano as i64)), + ); + if !span.attributes.is_empty() { + trace.insert( + event_path!(ATTRIBUTES_KEY), + kv_list_into_value(span.attributes), + ); + } + trace.insert( + event_path!(DROPPED_ATTRIBUTES_COUNT_KEY), + Value::from(span.dropped_attributes_count), + ); + if !span.events.is_empty() { + trace.insert( + event_path!("events"), + Value::Array(span.events.into_iter().map(Into::into).collect()), + ); + } + trace.insert( + event_path!("dropped_events_count"), + Value::from(span.dropped_events_count), + ); + if !span.links.is_empty() { + trace.insert( + event_path!("links"), + Value::Array(span.links.into_iter().map(Into::into).collect()), + ); + } + trace.insert( + event_path!("dropped_links_count"), + Value::from(span.dropped_links_count), + ); + trace.insert(event_path!("status"), Value::from(span.status)); + if let Some(resource) = self.resource + && !resource.attributes.is_empty() + { + trace.insert( + event_path!(RESOURCE_KEY), + kv_list_into_value(resource.attributes), + ); + } + trace.insert(event_path!("ingest_timestamp"), Value::from(now)); + trace.into() + } +} + +impl From for Value { + fn from(ev: SpanEvent) -> Self { + let mut obj: BTreeMap = BTreeMap::new(); + obj.insert("name".into(), ev.name.into()); + obj.insert( + "time_unix_nano".into(), + Value::Timestamp(Utc.timestamp_nanos(ev.time_unix_nano as i64)), + ); + obj.insert("attributes".into(), kv_list_into_value(ev.attributes)); + obj.insert( + "dropped_attributes_count".into(), + Value::Integer(ev.dropped_attributes_count as i64), + ); + Value::Object(obj) + } +} + +impl From for Value { + fn from(link: Link) -> Self { + let mut obj: BTreeMap = BTreeMap::new(); + obj.insert("trace_id".into(), Value::from(to_hex(&link.trace_id))); + obj.insert("span_id".into(), Value::from(to_hex(&link.span_id))); + obj.insert("trace_state".into(), link.trace_state.into()); + obj.insert("attributes".into(), kv_list_into_value(link.attributes)); + obj.insert( + "dropped_attributes_count".into(), + Value::Integer(link.dropped_attributes_count as i64), + ); + Value::Object(obj) + } +} + +impl From for Value { + fn from(status: SpanStatus) -> Self { + let mut obj: BTreeMap = BTreeMap::new(); + obj.insert("message".into(), status.message.into()); + obj.insert("code".into(), status.code.into()); + Value::Object(obj) + } +} diff --git a/lib/portpicker/Cargo.toml b/lib/portpicker/Cargo.toml index b7447ff944..0b296fdce8 100644 --- a/lib/portpicker/Cargo.toml +++ b/lib/portpicker/Cargo.toml @@ -2,9 +2,9 @@ name = "portpicker" version = "1.0.0" authors = ["Vector Contributors ", "Hannes Karppila "] -edition = "2021" +edition = "2024" publish = false license = "Unlicense" [dependencies] -rand = "0.8.5" +rand.workspace = true diff --git a/lib/portpicker/src/lib.rs b/lib/portpicker/src/lib.rs index 84cbb4dd13..8c83f6e4d1 100644 --- a/lib/portpicker/src/lib.rs +++ b/lib/portpicker/src/lib.rs @@ -30,7 +30,7 @@ use std::net::{IpAddr, SocketAddr, TcpListener, ToSocketAddrs, UdpSocket}; -use rand::{thread_rng, Rng}; +use rand::{Rng, rng}; pub type Port = u16; @@ -71,12 +71,12 @@ fn ask_free_tcp_port(ip: IpAddr) -> Option { /// let port: u16 = pick_unused_port(IpAddr::V4(Ipv4Addr::LOCALHOST)); /// ``` pub fn pick_unused_port(ip: IpAddr) -> Port { - let mut rng = thread_rng(); + let mut rng = rng(); loop { // Try random port first for _ in 0..10 { - let port = rng.gen_range(15000..25000); + let port = rng.random_range(15000..25000); if is_free(ip, port) { return port; } @@ -94,6 +94,28 @@ pub fn pick_unused_port(ip: IpAddr) -> Port { } } +pub fn bind_unused_udp(ip: IpAddr) -> UdpSocket { + let mut rng = rng(); + + loop { + // Try random port first + for _ in 0..10 { + let port = rng.random_range(15000..25000); + + if let Ok(socket) = UdpSocket::bind(SocketAddr::new(ip, port)) { + return socket; + } + } + + // Ask the OS for a port + for _ in 0..10 { + if let Ok(socket) = UdpSocket::bind(SocketAddr::new(ip, 0)) { + return socket; + } + } + } +} + #[cfg(test)] mod tests { use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; diff --git a/lib/prometheus-parser/Cargo.toml b/lib/prometheus-parser/Cargo.toml index 4e1c261b09..3947771f85 100644 --- a/lib/prometheus-parser/Cargo.toml +++ b/lib/prometheus-parser/Cargo.toml @@ -2,7 +2,7 @@ name = "prometheus-parser" version = "0.1.0" authors = ["Vector Contributors ", "Duy Do "] -edition = "2021" +edition = "2024" publish = false license = "MPL-2.0" @@ -10,7 +10,7 @@ license = "MPL-2.0" [dependencies] indexmap.workspace = true -nom = "7.1.3" +nom.workspace = true prost.workspace = true prost-types.workspace = true snafu.workspace = true diff --git a/lib/prometheus-parser/src/lib.rs b/lib/prometheus-parser/src/lib.rs index 986bf37157..31dfbe07b9 100644 --- a/lib/prometheus-parser/src/lib.rs +++ b/lib/prometheus-parser/src/lib.rs @@ -155,9 +155,7 @@ impl GroupKind { let value = metric.value; match self { - Self::Counter(ref mut metrics) - | Self::Gauge(ref mut metrics) - | Self::Untyped(ref mut metrics) => { + Self::Counter(metrics) | Self::Gauge(metrics) | Self::Untyped(metrics) => { if !suffix.is_empty() { return Ok(Some(Metric { name: metric.name, @@ -168,7 +166,7 @@ impl GroupKind { } metrics.insert(key, SimpleMetric { value }); } - Self::Histogram(ref mut metrics) => match suffix { + Self::Histogram(metrics) => match suffix { "_bucket" => { let bucket = key.labels.remove("le").ok_or(ParserError::ExpectedLeTag)?; let (_, bucket) = line::Metric::parse_value(&bucket) @@ -193,10 +191,10 @@ impl GroupKind { timestamp: key.timestamp, labels: key.labels, value, - })) + })); } }, - Self::Summary(ref mut metrics) => match suffix { + Self::Summary(metrics) => match suffix { "" => { let quantile = key .labels @@ -224,7 +222,7 @@ impl GroupKind { timestamp: key.timestamp, labels: key.labels, value, - })) + })); } }, } diff --git a/lib/prometheus-parser/src/line.rs b/lib/prometheus-parser/src/line.rs index aeac1e8110..a3a39a3d8f 100644 --- a/lib/prometheus-parser/src/line.rs +++ b/lib/prometheus-parser/src/line.rs @@ -3,14 +3,15 @@ use std::collections::BTreeMap; use nom::{ + Parser, branch::alt, bytes::complete::{is_not, tag, take_while, take_while1}, character::complete::{char, digit1}, - combinator::{map, opt, recognize, value}, + combinator::{map, map_res, opt, recognize, value}, error::ParseError, multi::fold_many0, number::complete::double, - sequence::{delimited, pair, preceded, tuple}, + sequence::{delimited, pair, preceded}, }; /// We try to catch all nom's `ErrorKind` with our own `ErrorKind`, @@ -112,7 +113,7 @@ impl Metric { /// ``` /// /// We don't parse timestamp. - fn parse(input: &str) -> IResult { + fn parse(input: &str) -> IResult<'_, Self> { let input = trim_space(input); let (input, name) = parse_name(input)?; let (input, labels) = Self::parse_labels(input)?; @@ -130,7 +131,7 @@ impl Metric { } /// Float value, and +Inf, -Int, Nan. - pub(crate) fn parse_value(input: &str) -> IResult { + pub(crate) fn parse_value(input: &str) -> IResult<'_, f64> { let input = trim_space(input); alt(( value(f64::INFINITY, tag("+Inf")), @@ -140,7 +141,8 @@ impl Metric { // This shouldn't be necessary if that issue is remedied. value(f64::NAN, tag("NaN")), double, - ))(input) + )) + .parse(input) .map_err(|_: NomError| { ErrorKind::ParseFloatError { input: input.to_owned(), @@ -149,25 +151,34 @@ impl Metric { }) } - fn parse_timestamp(input: &str) -> IResult> { + fn parse_timestamp(input: &str) -> IResult<'_, Option> { let input = trim_space(input); - opt(map(recognize(pair(opt(char('-')), digit1)), |s: &str| { - s.parse().unwrap() - }))(input) + opt(map_res( + recognize(pair(opt(char('-')), digit1)), + |s: &str| s.parse(), + )) + .parse(input) + .map_err(|_: NomError| { + ErrorKind::ParseTimestampError { + input: input.to_owned(), + } + .into() + }) } - fn parse_name_value(input: &str) -> IResult<(String, String)> { + fn parse_name_value(input: &str) -> IResult<'_, (String, String)> { map( - tuple((parse_name, match_char('='), Self::parse_escaped_string)), + (parse_name, match_char('='), Self::parse_escaped_string), |(name, _, value)| (name, value), - )(input) + ) + .parse(input) } // Return: // - Some((name, value)) => success // - None => list is properly ended with "}" // - Error => errors of parse_name_value - fn element_parser(input: &str) -> IResult> { + fn element_parser(input: &str) -> IResult<'_, Option<(String, String)>> { match Self::parse_name_value(input) { Ok((input, result)) => Ok((input, Some(result))), Err(nom::Err::Error(parse_name_value_error)) => match match_char('}')(input) { @@ -179,7 +190,7 @@ impl Metric { } } - fn parse_labels_inner(mut input: &str) -> IResult> { + fn parse_labels_inner(mut input: &str) -> IResult<'_, BTreeMap> { let sep = match_char(','); let mut result = BTreeMap::new(); @@ -214,10 +225,10 @@ impl Metric { } /// Parse `{label_name="value",...}` - fn parse_labels(input: &str) -> IResult> { + fn parse_labels(input: &str) -> IResult<'_, BTreeMap> { let input = trim_space(input); - match opt(char('{'))(input) { + match opt(char('{')).parse(input) { Ok((input, None)) => Ok((input, BTreeMap::new())), Ok((input, Some(_))) => Self::parse_labels_inner(input), Err(failure) => Err(failure), @@ -227,7 +238,7 @@ impl Metric { /// Parse `'"' string_content '"'`. `string_content` can contain any unicode characters, /// backslash (`\`), double-quote (`"`), and line feed (`\n`) characters have to be /// escaped as `\\`, `\"`, and `\n`, respectively. - fn parse_escaped_string(input: &str) -> IResult { + fn parse_escaped_string(input: &str) -> IResult<'_, String> { #[derive(Debug)] enum StringFragment<'a> { Literal(&'a str), @@ -263,7 +274,7 @@ impl Metric { }, ); - fn match_quote(input: &str) -> IResult { + fn match_quote(input: &str) -> IResult<'_, char> { char('"')(input).map_err(|_: NomError| { ErrorKind::ExpectedChar { expected: '"', @@ -273,12 +284,12 @@ impl Metric { }) } - delimited(match_quote, build_string, match_quote)(input) + delimited(match_quote, build_string, match_quote).parse(input) } } impl Header { - fn space1(input: &str) -> IResult<()> { + fn space1(input: &str) -> IResult<'_, ()> { take_while1(|c| c == ' ' || c == '\t')(input) .map_err(|_: NomError| { ErrorKind::ExpectedSpace { @@ -290,7 +301,7 @@ impl Header { } /// `# TYPE ` - fn parse(input: &str) -> IResult { + fn parse(input: &str) -> IResult<'_, Self> { let input = trim_space(input); let (input, _) = char('#')(input).map_err(|_: NomError| ErrorKind::ExpectedChar { expected: '#', @@ -310,7 +321,8 @@ impl Header { value(MetricKind::Summary, tag("summary")), value(MetricKind::Histogram, tag("histogram")), value(MetricKind::Untyped, tag("untyped")), - ))(input) + )) + .parse(input) .map_err(|_: NomError| ErrorKind::InvalidMetricKind { input: input.to_owned(), })?; @@ -349,7 +361,7 @@ impl Line { }; if let Ok((input, _)) = char::<_, NomErrorType>('#')(input) { - if tuple::<_, _, NomErrorType, _>((sp, tag("TYPE")))(input).is_ok() { + if (sp, tag::<_, _, NomErrorType>("TYPE")).parse(input).is_ok() { return Err(header_error); } Ok(None) @@ -360,12 +372,13 @@ impl Line { } /// Name matches the regex `[a-zA-Z_][a-zA-Z0-9_]*`. -fn parse_name(input: &str) -> IResult { +fn parse_name(input: &str) -> IResult<'_, String> { let input = trim_space(input); let (input, (a, b)) = pair( take_while1(|c: char| c.is_alphabetic() || c == '_'), take_while(|c: char| c.is_alphanumeric() || c == '_' || c == ':'), - )(input) + ) + .parse(input) .map_err(|_: NomError| ErrorKind::ParseNameError { input: input.to_owned(), })?; @@ -382,7 +395,7 @@ fn sp<'a, E: ParseError<&'a str>>(i: &'a str) -> nom::IResult<&'a str, &'a str, fn match_char(c: char) -> impl Fn(&str) -> IResult { move |input| { - preceded(sp, char(c))(input).map_err(|_: NomError| { + preceded(sp, char(c)).parse(input).map_err(|_: NomError| { ErrorKind::ExpectedChar { expected: c, input: input.to_owned(), @@ -401,7 +414,7 @@ mod test { #[test] fn test_parse_escaped_string() { fn wrap(s: &str) -> String { - format!(" \t \"{}\" .", s) + format!(" \t \"{s}\" .") } // parser should not consume more that it needed @@ -441,7 +454,7 @@ mod test { #[test] fn test_parse_name() { fn wrap(s: &str) -> String { - format!(" \t {} .", s) + format!(" \t {s} .") } let tail = " ."; @@ -467,7 +480,7 @@ mod test { #[test] fn test_parse_header() { fn wrap(s: &str) -> String { - format!(" \t {} .", s) + format!(" \t {s} .") } let tail = " ."; @@ -541,7 +554,7 @@ mod test { #[test] fn test_parse_value() { fn wrap(s: &str) -> String { - format!(" \t {} .", s) + format!(" \t {s} .") } let tail = " ."; @@ -609,7 +622,7 @@ mod test { #[test] fn test_parse_labels() { fn wrap(s: &str) -> String { - format!(" \t {} .", s) + format!(" \t {s} .") } let tail = " ."; @@ -667,6 +680,24 @@ mod test { assert_eq!(Metric::parse_timestamp(""), Ok(("", None))); assert_eq!(Metric::parse_timestamp("123"), Ok(("", Some(123)))); assert_eq!(Metric::parse_timestamp(" -23"), Ok(("", Some(-23)))); + // Edge cases + assert_eq!( + Metric::parse_timestamp("9223372036854775807"), + Ok(("", Some(9223372036854775807i64))) + ); + assert_eq!( + Metric::parse_timestamp("-9223372036854775808"), + Ok(("", Some(-9223372036854775808i64))) + ); + // overflow + assert_eq!( + Metric::parse_timestamp("9223372036854775809"), + Ok(("9223372036854775809", None)) + ); + assert_eq!( + Metric::parse_timestamp("-9223372036854775809"), + Ok(("-9223372036854775809", None)) + ); } #[test] diff --git a/lib/tracing-limit/Cargo.toml b/lib/tracing-limit/Cargo.toml index 8fb0fb6209..e6252ea3e6 100644 --- a/lib/tracing-limit/Cargo.toml +++ b/lib/tracing-limit/Cargo.toml @@ -2,20 +2,20 @@ name = "tracing-limit" version = "0.1.0" authors = ["Vector Contributors "] -edition = "2021" +edition = "2024" publish = false license = "MPL-2.0" [dependencies] tracing-core = { version = "0.1", default-features = false } -tracing-subscriber = { version = "0.3", default-features = false, features = ["registry", "std"] } +tracing-subscriber = { workspace = true, features = ["registry", "std"] } dashmap = { version = "6.1.0", default-features = false } [dev-dependencies] -criterion = "0.5" +criterion = "0.7" tracing = "0.1.34" -mock_instant = { version = "0.5" } -tracing-subscriber = { version = "0.3.19", default-features = false, features = ["env-filter", "fmt"] } +mock_instant = { version = "0.6" } +tracing-subscriber = { workspace = true, features = ["env-filter", "fmt"] } [[bench]] name = "limit" diff --git a/lib/tracing-limit/benches/limit.rs b/lib/tracing-limit/benches/limit.rs index f7214525e9..4188d2e987 100644 --- a/lib/tracing-limit/benches/limit.rs +++ b/lib/tracing-limit/benches/limit.rs @@ -1,16 +1,15 @@ -#[macro_use] -extern crate tracing; - #[macro_use] extern crate criterion; +#[macro_use] +extern crate tracing; +use criterion::{BenchmarkId, Criterion}; +use std::hint::black_box; use std::{ fmt, sync::{Mutex, MutexGuard}, }; - -use criterion::{black_box, BenchmarkId, Criterion}; -use tracing::{field, span, subscriber::Interest, Event, Metadata, Subscriber}; +use tracing::{Event, Metadata, Subscriber, field, span, subscriber::Interest}; use tracing_limit::RateLimitedLayer; use tracing_subscriber::layer::{Context, Layer, SubscriberExt}; @@ -97,7 +96,7 @@ struct Visitor<'a>(MutexGuard<'a, String>); impl field::Visit for Visitor<'_> { fn record_debug(&mut self, _field: &field::Field, value: &dyn fmt::Debug) { use std::fmt::Write; - _ = write!(&mut *self.0, "{:?}", value); + _ = write!(&mut *self.0, "{value:?}"); } } diff --git a/lib/tracing-limit/examples/basic.rs b/lib/tracing-limit/examples/basic.rs index 3b4428d540..d0affb3e47 100644 --- a/lib/tracing-limit/examples/basic.rs +++ b/lib/tracing-limit/examples/basic.rs @@ -1,4 +1,4 @@ -use tracing::{info, trace, Dispatch}; +use tracing::{Dispatch, info, trace}; use tracing_limit::RateLimitedLayer; use tracing_subscriber::layer::SubscriberExt; diff --git a/lib/tracing-limit/examples/by_span.rs b/lib/tracing-limit/examples/by_span.rs index 63b8775d99..72e021b4da 100644 --- a/lib/tracing-limit/examples/by_span.rs +++ b/lib/tracing-limit/examples/by_span.rs @@ -1,4 +1,4 @@ -use tracing::{info, info_span, trace, Dispatch}; +use tracing::{Dispatch, info, info_span, trace}; use tracing_limit::RateLimitedLayer; use tracing_subscriber::layer::SubscriberExt; diff --git a/lib/tracing-limit/src/lib.rs b/lib/tracing-limit/src/lib.rs index bd5952bf39..81bea3dc33 100644 --- a/lib/tracing-limit/src/lib.rs +++ b/lib/tracing-limit/src/lib.rs @@ -4,11 +4,11 @@ use std::fmt; use dashmap::DashMap; use tracing_core::{ + Event, Metadata, Subscriber, callsite::Identifier, - field::{display, Field, Value, Visit}, + field::{Field, Value, Visit, display}, span, subscriber::Interest, - Event, Metadata, Subscriber, }; use tracing_subscriber::layer::{Context, Layer}; @@ -389,7 +389,7 @@ impl Visit for RateLimitedSpanKeys { } fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) { - self.record(field, format!("{:?}", value).into()); + self.record(field, format!("{value:?}").into()); } } @@ -437,7 +437,7 @@ impl Visit for MessageVisitor { fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) { if self.message.is_none() && field.name() == MESSAGE_FIELD { - self.message = Some(format!("{:?}", value)); + self.message = Some(format!("{value:?}")); } } } @@ -445,13 +445,15 @@ impl Visit for MessageVisitor { #[cfg(test)] mod test { use std::{ - sync::{Arc, Mutex}, + sync::{Arc, LazyLock, Mutex}, time::Duration, }; use mock_instant::global::MockClock; use tracing_subscriber::layer::SubscriberExt; + static TRACING_DEFAULT_LOCK: LazyLock> = LazyLock::new(|| Mutex::new(())); + use super::*; #[derive(Default)] @@ -494,6 +496,8 @@ mod test { #[test] fn rate_limits() { + let _guard = TRACING_DEFAULT_LOCK.lock().unwrap(); + let events: Arc>> = Default::default(); let recorder = RecordingLayer::new(Arc::clone(&events)); @@ -527,6 +531,8 @@ mod test { #[test] fn override_rate_limit_at_callsite() { + let _guard = TRACING_DEFAULT_LOCK.lock().unwrap(); + let events: Arc>> = Default::default(); let recorder = RecordingLayer::new(Arc::clone(&events)); @@ -564,6 +570,8 @@ mod test { #[test] fn rate_limit_by_span_key() { + let _guard = TRACING_DEFAULT_LOCK.lock().unwrap(); + let events: Arc>> = Default::default(); let recorder = RecordingLayer::new(Arc::clone(&events)); @@ -577,8 +585,7 @@ mod test { info_span!("span", component_id = &key, vrl_position = &line_number); let _enter = span.enter(); info!( - message = - format!("Hello {} on line_number {}!", key, line_number).as_str(), + message = format!("Hello {key} on line_number {line_number}!").as_str(), internal_log_rate_limit = true ); } @@ -629,6 +636,8 @@ mod test { #[test] fn rate_limit_by_event_key() { + let _guard = TRACING_DEFAULT_LOCK.lock().unwrap(); + let events: Arc>> = Default::default(); let recorder = RecordingLayer::new(Arc::clone(&events)); @@ -639,8 +648,7 @@ mod test { for key in &["foo", "bar"] { for line_number in &[1, 2] { info!( - message = - format!("Hello {} on line_number {}!", key, line_number).as_str(), + message = format!("Hello {key} on line_number {line_number}!").as_str(), internal_log_rate_limit = true, component_id = &key, vrl_position = &line_number diff --git a/lib/vector-api-client/Cargo.toml b/lib/vector-api-client/Cargo.toml index 3a4a07d143..5b3b329088 100644 --- a/lib/vector-api-client/Cargo.toml +++ b/lib/vector-api-client/Cargo.toml @@ -2,7 +2,7 @@ name = "vector-api-client" version = "0.1.2" authors = ["Vector Contributors "] -edition = "2021" +edition = "2024" publish = false license = "MPL-2.0" @@ -13,11 +13,11 @@ serde.workspace = true serde_json.workspace = true # Error handling -anyhow = { version = "1.0.95", default-features = false, features = ["std"] } +anyhow.workspace = true # Tokio / Futures futures.workspace = true -tokio = { version = "1.43.0", default-features = false, features = ["macros", "rt", "sync"] } +tokio = { workspace = true, features = ["macros", "rt", "sync"] } tokio-stream = { version = "0.1.17", default-features = false, features = ["sync"] } # GraphQL diff --git a/lib/vector-api-client/src/gql/components.rs b/lib/vector-api-client/src/gql/components.rs index a2bfde4cdf..d278b30c31 100644 --- a/lib/vector-api-client/src/gql/components.rs +++ b/lib/vector-api-client/src/gql/components.rs @@ -181,7 +181,7 @@ impl fmt::Display for components_query::ComponentsQueryComponentsEdgesNodeOn { components_query::ComponentsQueryComponentsEdgesNodeOn::Sink(_) => "sink", }; - write!(f, "{}", res) + write!(f, "{res}") } } @@ -199,7 +199,7 @@ impl fmt::Display for component_added_subscription::ComponentAddedSubscriptionCo } }; - write!(f, "{}", res) + write!(f, "{res}") } } @@ -219,6 +219,6 @@ impl fmt::Display } }; - write!(f, "{}", res) + write!(f, "{res}") } } diff --git a/lib/vector-api-client/src/subscription.rs b/lib/vector-api-client/src/subscription.rs index ca83f074ae..0a4308c6ed 100644 --- a/lib/vector-api-client/src/subscription.rs +++ b/lib/vector-api-client/src/subscription.rs @@ -12,7 +12,7 @@ use tokio::sync::{ broadcast::{self, Sender}, mpsc, oneshot, }; -use tokio_stream::{wrappers::BroadcastStream, Stream, StreamExt}; +use tokio_stream::{Stream, StreamExt, wrappers::BroadcastStream}; use tokio_tungstenite::{connect_async, tungstenite::Message}; use url::Url; use uuid::Uuid; @@ -39,15 +39,6 @@ pub struct Payload { } impl Payload { - /// Returns an "init" payload to confirm the connection to the server. - pub fn init(id: Uuid) -> Self { - Self { - id, - payload_type: "connection_init".to_owned(), - payload: json!({}), - } - } - /// Returns a "start" payload necessary for starting a new subscription. pub fn start( id: Uuid, @@ -158,8 +149,7 @@ impl SubscriptionClient { self.subscriptions.lock().unwrap().insert(id, tx); - // Initialize the connection with the relevant control messages. - _ = self.tx.send(Payload::init(id)); + // Send start subscription command with the relevant control messages. _ = self.tx.send(Payload::start::(id, request_body)); Box::pin( @@ -182,6 +172,11 @@ pub async fn connect_subscription_client( let (send_tx, mut send_rx) = mpsc::unbounded_channel::(); let (recv_tx, recv_rx) = mpsc::unbounded_channel::(); + // Initialize the connection + _ = ws_tx + .send(Message::Text(r#"{"type":"connection_init"}"#.to_string())) + .await; + // Forwarded received messages back upstream to the GraphQL server tokio::spawn(async move { while let Some(p) = send_rx.recv().await { diff --git a/lib/vector-buffers/Cargo.toml b/lib/vector-buffers/Cargo.toml index b43f922148..0c67c95665 100644 --- a/lib/vector-buffers/Cargo.toml +++ b/lib/vector-buffers/Cargo.toml @@ -2,7 +2,7 @@ name = "vector-buffers" version = "0.1.0" authors = ["Vector Contributors "] -edition = "2021" +edition = "2024" publish = false [lints.rust] @@ -11,43 +11,45 @@ unexpected_cfgs = { level = "warn", check-cfg = ['cfg(tokio_unstable)'] } [dependencies] async-recursion = "1.1.1" async-stream = "0.3.6" -async-trait = { version = "0.1", default-features = false } +async-trait.workspace = true bytecheck = { version = "0.6.9", default-features = false, features = ["std"] } -bytes = { version = "1.9.0", default-features = false } -crc32fast = { version = "1.4.2", default-features = false } +bytes.workspace = true +crc32fast = { version = "1.5.0", default-features = false } crossbeam-queue = { version = "0.3.12", default-features = false, features = ["std"] } -crossbeam-utils = { version = "0.8.21", default-features = false } -derivative = { version = "2.2.0", default-features = false } +crossbeam-utils.workspace = true +derivative.workspace = true fslock = { version = "0.2.1", default-features = false, features = ["std"] } futures.workspace = true -memmap2 = { version = "0.9.5", default-features = false } +memmap2 = { version = "0.9.8", default-features = false } metrics.workspace = true num-traits = { version = "0.2.19", default-features = false } -paste = "1.0.15" +paste.workspace = true rkyv = { version = "0.7.45", default-features = false, features = ["size_32", "std", "strict", "validation"] } serde.workspace = true snafu.workspace = true tokio-util = { version = "0.7.0", default-features = false } -tokio = { version = "1.43.0", default-features = false, features = ["rt", "macros", "rt-multi-thread", "sync", "fs", "io-util", "time"] } -tracing = { version = "0.1.34", default-features = false, features = ["attributes"] } +tokio = { workspace = true, features = ["rt", "macros", "rt-multi-thread", "sync", "fs", "io-util", "time"] } +tracing = { workspace = true, features = ["attributes"] } vector-config = { path = "../vector-config", default-features = false } vector-common = { path = "../vector-common", default-features = false, features = ["byte_size_of"] } +dashmap.workspace = true +ordered-float.workspace = true [dev-dependencies] clap.workspace = true -criterion = { version = "0.5", features = ["html_reports", "async_tokio"] } +criterion = { version = "0.7", features = ["html_reports", "async_tokio"] } crossbeam-queue = "0.3.12" hdrhistogram = "7.5.4" metrics-tracing-context.workspace = true metrics-util = { workspace = true, features = ["debugging"] } -proptest = "1.5" +proptest = "1.7" quickcheck = "1.0" -rand = "0.8.5" -serde_yaml = { version = "0.9", default-features = false } -temp-dir = "0.1.14" +rand.workspace = true +serde_yaml.workspace = true +temp-dir = "0.1.16" tokio-test = "0.4.4" tracing-fluent-assertions = { version = "0.3" } -tracing-subscriber = { version = "0.3.19", default-features = false, features = ["env-filter", "fmt", "registry", "std", "ansi"] } +tracing-subscriber = { workspace = true, features = ["env-filter", "fmt", "registry", "std", "ansi"] } [[bench]] name = "sized_records" diff --git a/lib/vector-buffers/benches/common.rs b/lib/vector-buffers/benches/common.rs index cb2f1ec402..7547924bdb 100644 --- a/lib/vector-buffers/benches/common.rs +++ b/lib/vector-buffers/benches/common.rs @@ -7,19 +7,22 @@ use metrics_util::layers::Layer; use tracing::Span; use tracing_subscriber::prelude::__tracing_subscriber_SubscriberExt; use vector_buffers::{ + BufferType, EventCount, encoding::FixedEncodable, topology::{ builder::TopologyBuilder, channel::{BufferReceiver, BufferSender}, }, - BufferType, EventCount, }; use vector_common::byte_size_of::ByteSizeOf; use vector_common::finalization::{AddBatchNotifier, BatchNotifier, EventFinalizers, Finalizable}; -#[derive(Clone, Copy, Debug)] +#[derive(Clone, Debug)] pub struct Message { id: u64, + // Purpose of `_heap_allocated` is to simulate memory pressure in the buffer benchmarks when the + // max_size option is selected. + _heap_allocated: Box<[u64; N]>, _padding: [u64; N], } @@ -27,6 +30,7 @@ impl Message { fn new(id: u64) -> Self { Message { id, + _heap_allocated: Box::new([0; N]), _padding: [0; N], } } @@ -40,7 +44,7 @@ impl AddBatchNotifier for Message { impl ByteSizeOf for Message { fn allocated_bytes(&self) -> usize { - 0 + N * std::mem::size_of::() } } @@ -61,7 +65,7 @@ pub struct EncodeError; impl fmt::Display for EncodeError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{:?}", self) + write!(f, "{self:?}") } } @@ -72,7 +76,7 @@ pub struct DecodeError; impl fmt::Display for DecodeError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{:?}", self) + write!(f, "{self:?}") } } @@ -88,8 +92,8 @@ impl FixedEncodable for Message { Self: Sized, { buffer.put_u64(self.id); - for _ in 0..N { - // this covers self._padding + for _ in 0..(N * 2) { + // this covers self._padding and self.heap_allocated buffer.put_u64(0); } Ok(()) @@ -101,8 +105,8 @@ impl FixedEncodable for Message { Self: Sized, { let id = buffer.get_u64(); - for _ in 0..N { - // this covers self._padding + for _ in 0..(N * 2) { + // this covers self._padding and self.heap_allocated _ = buffer.get_u64(); } Ok(Message::new(id)) diff --git a/lib/vector-buffers/benches/sized_records.rs b/lib/vector-buffers/benches/sized_records.rs index 80ffd4275d..91d04b29f3 100644 --- a/lib/vector-buffers/benches/sized_records.rs +++ b/lib/vector-buffers/benches/sized_records.rs @@ -6,11 +6,11 @@ use std::{ }; use criterion::{ - criterion_group, criterion_main, measurement::WallTime, BatchSize, BenchmarkGroup, BenchmarkId, - Criterion, SamplingMode, Throughput, + BatchSize, BenchmarkGroup, BenchmarkId, Criterion, SamplingMode, Throughput, criterion_group, + criterion_main, measurement::WallTime, }; use tokio::runtime::{Handle, Runtime}; -use vector_buffers::{BufferType, WhenFull}; +use vector_buffers::{BufferType, MemoryBufferSize, WhenFull}; use crate::common::{init_instrumentation, war_measurement, wtr_measurement}; @@ -37,6 +37,11 @@ struct PathGuard { inner: PathBuf, } +enum BoundBy { + Bytes, + NumberEvents, +} + impl DataDir { fn new(name: &str) -> Self { let mut base_dir = PathBuf::new(); @@ -79,9 +84,15 @@ fn create_disk_v2_variant(_max_events: usize, max_size: u64) -> BufferType { } } -fn create_in_memory_variant(max_events: usize, _max_size: u64) -> BufferType { +fn create_in_memory_variant(bound_by: BoundBy, max_events: usize, max_size: u64) -> BufferType { + let size = match bound_by { + BoundBy::Bytes => MemoryBufferSize::MaxSize(NonZeroUsize::new(max_size as usize).unwrap()), + BoundBy::NumberEvents => { + MemoryBufferSize::MaxEvents(NonZeroUsize::new(max_events).unwrap()) + } + }; BufferType::Memory { - max_events: NonZeroUsize::new(max_events).unwrap(), + size, when_full: WhenFull::DropNewest, } } @@ -146,13 +157,24 @@ fn write_then_read(c: &mut Criterion) { create_disk_v2_variant ); + let f = |a, b| create_in_memory_variant(BoundBy::NumberEvents, a, b); experiment!( c, [32, 64, 128, 256, 512, 1024], "buffer-in-memory-v2", "write-then-read", wtr_measurement, - create_in_memory_variant + f + ); + + let f = |a, b| create_in_memory_variant(BoundBy::Bytes, a, b); + experiment!( + c, + [32, 64, 128, 256, 512, 1024], + "buffer-in-memory-bytes-v2", + "write-then-read", + wtr_measurement, + f ); } @@ -167,13 +189,24 @@ fn write_and_read(c: &mut Criterion) { create_disk_v2_variant ); + let f = |a, b| create_in_memory_variant(BoundBy::NumberEvents, a, b); experiment!( c, [32, 64, 128, 256, 512, 1024], "buffer-in-memory-v2", "write-and-read", war_measurement, - create_in_memory_variant + f + ); + + let f = |a, b| create_in_memory_variant(BoundBy::Bytes, a, b); + experiment!( + c, + [32, 64, 128, 256, 512, 1024], + "buffer-in-memory-bytes-v2", + "write-and-read", + war_measurement, + f ); } diff --git a/lib/vector-buffers/examples/buffer_perf.rs b/lib/vector-buffers/examples/buffer_perf.rs index c571c8b74e..816380d303 100644 --- a/lib/vector-buffers/examples/buffer_perf.rs +++ b/lib/vector-buffers/examples/buffer_perf.rs @@ -2,8 +2,8 @@ use std::{ cmp, error, fmt, path::PathBuf, sync::{ - atomic::{AtomicUsize, Ordering}, Arc, + atomic::{AtomicUsize, Ordering}, }, time::{Duration, Instant}, }; @@ -13,15 +13,15 @@ use clap::{Arg, Command}; use hdrhistogram::Histogram; use rand::Rng; use tokio::{select, sync::oneshot, task, time}; -use tracing::{debug, info, Span}; +use tracing::{Span, debug, info}; use tracing_subscriber::EnvFilter; use vector_buffers::{ + BufferType, Bufferable, EventCount, MemoryBufferSize, WhenFull, encoding::FixedEncodable, topology::{ builder::TopologyBuilder, channel::{BufferReceiver, BufferSender}, }, - BufferType, Bufferable, EventCount, WhenFull, }; use vector_common::byte_size_of::ByteSizeOf; use vector_common::finalization::{ @@ -113,7 +113,7 @@ pub struct EncodeError; impl fmt::Display for EncodeError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{:?}", self) + write!(f, "{self:?}") } } @@ -124,7 +124,7 @@ pub struct DecodeError; impl fmt::Display for DecodeError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{:?}", self) + write!(f, "{self:?}") } } @@ -231,11 +231,11 @@ impl Configuration { } fn generate_record_cache(min: usize, max: usize) -> Vec { - let mut rng = rand::thread_rng(); + let mut rng = rand::rng(); let mut records = Vec::new(); for i in 1..=200_000 { - let payload_size = rng.gen_range(min..max); - let payload = (0..payload_size).map(|_| rng.gen::()).collect(); + let payload_size = rng.random_range(min..max); + let payload = (0..payload_size).map(|_| rng.random::()).collect(); let message = VariableMessage::new(i, payload); records.push(message); } @@ -247,7 +247,7 @@ where T: Bufferable + Clone + Finalizable, { let data_dir = PathBuf::from("/tmp/vector"); - let id = format!("{}-buffer-perf-testing", buffer_type); + let id = format!("{buffer_type}-buffer-perf-testing"); let max_size_events = std::num::NonZeroUsize::new(500).unwrap(); let max_size_bytes = std::num::NonZeroU64::new(32 * 1024 * 1024 * 1024).unwrap(); let when_full = WhenFull::Block; @@ -261,7 +261,7 @@ where max_size_events ); BufferType::Memory { - max_events: max_size_events, + size: MemoryBufferSize::MaxEvents(max_size_events), when_full, } } @@ -276,8 +276,7 @@ where } } s => panic!( - "unknown buffer type '{}' requested; valid types are in-memory, disk-v1, and disk-v2", - s + "unknown buffer type '{s}' requested; valid types are in-memory, disk-v1, and disk-v2" ), }; diff --git a/lib/vector-buffers/src/buffer_usage_data.rs b/lib/vector-buffers/src/buffer_usage_data.rs index ac06d2757d..61f6f5094c 100644 --- a/lib/vector-buffers/src/buffer_usage_data.rs +++ b/lib/vector-buffers/src/buffer_usage_data.rs @@ -1,7 +1,7 @@ use std::{ sync::{ - atomic::{AtomicU64, Ordering}, Arc, + atomic::{AtomicU64, Ordering}, }, time::Duration, }; @@ -15,6 +15,38 @@ use crate::{ spawn_named, }; +/// Since none of the values used with atomic operations are used to protect other values, we can +/// always used a "relaxed" ordering when updating them. +const ORDERING: Ordering = Ordering::Relaxed; + +fn increment_counter(counter: &AtomicU64, delta: u64) { + counter + .fetch_update(ORDERING, ORDERING, |current| { + Some(current.checked_add(delta).unwrap_or_else(|| { + warn!( + current, + delta, "Buffer counter overflowed. Clamping value to `u64::MAX`." + ); + u64::MAX + })) + }) + .ok(); +} + +fn decrement_counter(counter: &AtomicU64, delta: u64) { + counter + .fetch_update(ORDERING, ORDERING, |current| { + Some(current.checked_sub(delta).unwrap_or_else(|| { + warn!( + current, + delta, "Buffer counter underflowed. Clamping value to `0`." + ); + 0 + })) + }) + .ok(); +} + /// Snapshot of category metrics. struct CategorySnapshot { event_count: u64, @@ -43,25 +75,29 @@ struct CategoryMetrics { impl CategoryMetrics { /// Increments the event count and byte size by the given amounts. fn increment(&self, event_count: u64, event_byte_size: u64) { - self.event_count.fetch_add(event_count, Ordering::Relaxed); - self.event_byte_size - .fetch_add(event_byte_size, Ordering::Relaxed); + increment_counter(&self.event_count, event_count); + increment_counter(&self.event_byte_size, event_byte_size); + } + + /// Decrements the event count and byte size by the given amounts. + fn decrement(&self, event_count: u64, event_byte_size: u64) { + decrement_counter(&self.event_count, event_count); + decrement_counter(&self.event_byte_size, event_byte_size); } /// Sets the event count and event byte size to the given amount. /// /// Most updates are meant to be incremental, so this should be used sparingly. fn set(&self, event_count: u64, event_byte_size: u64) { - self.event_count.store(event_count, Ordering::Release); - self.event_byte_size - .store(event_byte_size, Ordering::Release); + self.event_count.store(event_count, ORDERING); + self.event_byte_size.store(event_byte_size, ORDERING); } /// Gets a snapshot of the event count and event byte size. fn get(&self) -> CategorySnapshot { CategorySnapshot { - event_count: self.event_count.load(Ordering::Acquire), - event_byte_size: self.event_byte_size.load(Ordering::Acquire), + event_count: self.event_count.load(ORDERING), + event_byte_size: self.event_byte_size.load(ORDERING), } } @@ -73,8 +109,8 @@ impl CategoryMetrics { /// track the last seen values. fn consume(&self) -> CategorySnapshot { CategorySnapshot { - event_count: self.event_count.swap(0, Ordering::AcqRel), - event_byte_size: self.event_byte_size.swap(0, Ordering::AcqRel), + event_count: self.event_count.swap(0, ORDERING), + event_byte_size: self.event_byte_size.swap(0, ORDERING), } } } @@ -117,14 +153,20 @@ impl BufferUsageHandle { /// /// This represents the events being sent into the buffer. pub fn increment_received_event_count_and_byte_size(&self, count: u64, byte_size: u64) { - self.state.received.increment(count, byte_size); + if count > 0 || byte_size > 0 { + self.state.received.increment(count, byte_size); + self.state.current.increment(count, byte_size); + } } /// Increments the number of events (and their total size) sent by this buffer component. /// /// This represents the events being read out of the buffer. pub fn increment_sent_event_count_and_byte_size(&self, count: u64, byte_size: u64) { - self.state.sent.increment(count, byte_size); + if count > 0 || byte_size > 0 { + self.state.sent.increment(count, byte_size); + self.state.current.decrement(count, byte_size); + } } /// Increment the number of dropped events (and their total size) for this buffer component. @@ -134,10 +176,13 @@ impl BufferUsageHandle { byte_size: u64, intentional: bool, ) { - if intentional { - self.state.dropped_intentional.increment(count, byte_size); - } else { - self.state.dropped.increment(count, byte_size); + if count > 0 || byte_size > 0 { + if intentional { + self.state.dropped_intentional.increment(count, byte_size); + } else { + self.state.dropped.increment(count, byte_size); + } + self.state.current.decrement(count, byte_size); } } } @@ -150,6 +195,7 @@ struct BufferUsageData { dropped: CategoryMetrics, dropped_intentional: CategoryMetrics, max_size: CategoryMetrics, + current: CategoryMetrics, } impl BufferUsageData { @@ -242,8 +288,10 @@ impl BufferUsage { /// The `buffer_id` should be a unique name -- ideally the `component_id` of the sink using this buffer -- but is /// not used for anything other than reporting, and so has no _requirement_ to be unique. pub fn install(self, buffer_id: &str) { + let buffer_id = buffer_id.to_string(); let span = self.span; let stages = self.stages; + let task_name = format!("buffer usage reporter ({buffer_id})"); let task = async move { let mut interval = interval(Duration::from_secs(2)); @@ -253,6 +301,7 @@ impl BufferUsage { for stage in &stages { let max_size = stage.max_size.get(); emit(BufferCreated { + buffer_id: buffer_id.clone(), idx: stage.idx, max_size_bytes: max_size.event_byte_size, max_size_events: max_size @@ -261,50 +310,129 @@ impl BufferUsage { .expect("should never be bigger than `usize`"), }); + let current = stage.current.get(); let received = stage.received.consume(); if received.has_updates() { emit(BufferEventsReceived { + buffer_id: buffer_id.clone(), idx: stage.idx, count: received.event_count, byte_size: received.event_byte_size, + total_count: current.event_count, + total_byte_size: current.event_byte_size, }); } let sent = stage.sent.consume(); if sent.has_updates() { emit(BufferEventsSent { + buffer_id: buffer_id.clone(), idx: stage.idx, count: sent.event_count, byte_size: sent.event_byte_size, + total_count: current.event_count, + total_byte_size: current.event_byte_size, }); } let dropped = stage.dropped.consume(); if dropped.has_updates() { emit(BufferEventsDropped { + buffer_id: buffer_id.clone(), idx: stage.idx, intentional: false, reason: "corrupted_events", count: dropped.event_count, byte_size: dropped.event_byte_size, + total_count: current.event_count, + total_byte_size: current.event_byte_size, }); } let dropped_intentional = stage.dropped_intentional.consume(); if dropped_intentional.has_updates() { emit(BufferEventsDropped { + buffer_id: buffer_id.clone(), idx: stage.idx, intentional: true, reason: "drop_newest", count: dropped_intentional.event_count, byte_size: dropped_intentional.event_byte_size, + total_count: current.event_count, + total_byte_size: current.event_byte_size, }); } } } }; - let task_name = format!("buffer usage reporter ({buffer_id})"); spawn_named(task.instrument(span.or_current()), task_name.as_str()); } } + +#[cfg(test)] +mod tests { + use super::*; + use std::thread; + + #[test] + fn test_multithreaded_updates_are_correct() { + const NUM_THREADS: u64 = 16; + const INCREMENTS_PER_THREAD: u64 = 10_000; + + let counter = Arc::new(AtomicU64::new(0)); + + let mut handles = vec![]; + + for _ in 0..NUM_THREADS { + let counter = Arc::clone(&counter); + let handle = thread::spawn(move || { + for _ in 0..INCREMENTS_PER_THREAD { + increment_counter(&counter, 1); + decrement_counter(&counter, 1); + } + }); + handles.push(handle); + } + + for handle in handles { + handle.join().unwrap(); + } + + assert_eq!(counter.load(ORDERING), 0); + } + + #[test] + fn test_decrement_counter_prevents_negatives() { + let counter = AtomicU64::new(100); + + decrement_counter(&counter, 50); + assert_eq!(counter.load(ORDERING), 50); + + decrement_counter(&counter, 100); + assert_eq!(counter.load(ORDERING), 0); + + decrement_counter(&counter, 50); + assert_eq!(counter.load(ORDERING), 0); + + decrement_counter(&counter, u64::MAX); + assert_eq!(counter.load(ORDERING), 0); + } + + #[test] + fn test_increment_counter_prevents_overflow() { + let counter = AtomicU64::new(u64::MAX - 2); + + increment_counter(&counter, 1); + assert_eq!(counter.load(ORDERING), u64::MAX - 1); + + increment_counter(&counter, 1); + assert_eq!(counter.load(ORDERING), u64::MAX); + + increment_counter(&counter, 1); + assert_eq!(counter.load(ORDERING), u64::MAX); + + increment_counter(&counter, u64::MAX); + assert_eq!(counter.load(ORDERING), u64::MAX); + } +} diff --git a/lib/vector-buffers/src/config.rs b/lib/vector-buffers/src/config.rs index 14826b6c20..ea51bfce3c 100644 --- a/lib/vector-buffers/src/config.rs +++ b/lib/vector-buffers/src/config.rs @@ -5,19 +5,19 @@ use std::{ slice, }; -use serde::{de, Deserialize, Deserializer, Serialize}; +use serde::{Deserialize, Deserializer, Serialize, de}; use snafu::{ResultExt, Snafu}; use tracing::Span; use vector_common::{config::ComponentKey, finalization::Finalizable}; use vector_config::configurable_component; use crate::{ + Bufferable, WhenFull, topology::{ builder::{TopologyBuilder, TopologyError}, channel::{BufferReceiver, BufferSender}, }, variants::{DiskV2Buffer, MemoryBuffer}, - Bufferable, WhenFull, }; #[derive(Debug, Snafu)] @@ -86,16 +86,32 @@ impl BufferTypeVisitor { let when_full = when_full.unwrap_or_default(); match kind { BufferTypeKind::Memory => { - if max_size.is_some() { - return Err(de::Error::unknown_field( - "max_size", - &["type", "max_events", "when_full"], - )); - } - Ok(BufferType::Memory { - max_events: max_events.unwrap_or_else(memory_buffer_default_max_events), - when_full, - }) + let size = match (max_events, max_size) { + (Some(_), Some(_)) => { + return Err(de::Error::unknown_field( + "max_events", + &["type", "max_size", "when_full"], + )); + } + (_, Some(max_size)) => { + if let Ok(bounded_max_bytes) = usize::try_from(max_size.get()) { + MemoryBufferSize::MaxSize(NonZeroUsize::new(bounded_max_bytes).unwrap()) + } else { + return Err(de::Error::invalid_value( + de::Unexpected::Unsigned(max_size.into()), + &format!( + "Value for max_bytes must be a positive integer <= {}", + usize::MAX + ) + .as_str(), + )); + } + } + _ => MemoryBufferSize::MaxEvents( + max_events.unwrap_or_else(memory_buffer_default_max_events), + ), + }; + Ok(BufferType::Memory { size, when_full }) } BufferTypeKind::DiskV2 => { if max_events.is_some() { @@ -175,6 +191,23 @@ impl DiskUsage { } } +/// Enumeration to define exactly what terms the bounds of the buffer is expressed in: length, or +/// `byte_size`. +#[configurable_component(no_deser)] +#[serde(rename_all = "snake_case")] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum MemoryBufferSize { + /// The maximum number of events allowed in the buffer. + MaxEvents(#[serde(default = "memory_buffer_default_max_events")] NonZeroUsize), + + // Doc string is duplicated here as a workaround due to a name collision with the `max_size` + // field with the DiskV2 variant of `BufferType`. + /// The maximum allowed amount of allocated memory the buffer can hold. + /// + /// If `type = "disk"` then must be at least ~256 megabytes (268435488 bytes). + MaxSize(#[configurable(metadata(docs::type_unit = "bytes"))] NonZeroUsize), +} + /// A specific type of buffer stage. #[configurable_component(no_deser)] #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -186,11 +219,10 @@ pub enum BufferType { /// This is more performant, but less durable. Data will be lost if Vector is restarted /// forcefully or crashes. #[configurable(title = "Events are buffered in memory.")] - #[serde(rename = "memory")] Memory { - /// The maximum number of events allowed in the buffer. - #[serde(default = "memory_buffer_default_max_events")] - max_events: NonZeroUsize, + /// The terms around how to express buffering limits, can be in size or `bytes_size`. + #[serde(flatten)] + size: MemoryBufferSize, #[configurable(derived)] #[serde(default)] @@ -277,11 +309,8 @@ impl BufferType { T: Bufferable + Clone + Finalizable, { match *self { - BufferType::Memory { - when_full, - max_events, - } => { - builder.stage(MemoryBuffer::new(max_events), when_full); + BufferType::Memory { size, when_full } => { + builder.stage(MemoryBuffer::new(size), when_full); } BufferType::DiskV2 { when_full, @@ -290,7 +319,7 @@ impl BufferType { let data_dir = data_dir.ok_or(BufferBuildError::RequiresDataDir)?; builder.stage(DiskV2Buffer::new(id, data_dir, max_size), when_full); } - }; + } Ok(()) } @@ -322,7 +351,7 @@ impl BufferType { description = r#"More information about the individual buffer types, and buffer behavior, can be found in the [Buffering Model][buffering_model] section. -[buffering_model]: /docs/about/under-the-hood/architecture/buffering-model/"# +[buffering_model]: /docs/architecture/buffering-model/"# )] pub enum BufferConfig { /// A single stage buffer topology. @@ -335,7 +364,7 @@ pub enum BufferConfig { impl Default for BufferConfig { fn default() -> Self { Self::Single(BufferType::Memory { - max_events: memory_buffer_default_max_events(), + size: MemoryBufferSize::MaxEvents(memory_buffer_default_max_events()), when_full: WhenFull::default(), }) } @@ -389,7 +418,7 @@ impl BufferConfig { mod test { use std::num::{NonZeroU64, NonZeroUsize}; - use crate::{BufferConfig, BufferType, WhenFull}; + use crate::{BufferConfig, BufferType, MemoryBufferSize, WhenFull}; fn check_single_stage(source: &str, expected: BufferType) { let config: BufferConfig = serde_yaml::from_str(source).unwrap(); @@ -439,7 +468,20 @@ max_events: 42 max_events: 100 ", BufferType::Memory { - max_events: NonZeroUsize::new(100).unwrap(), + size: MemoryBufferSize::MaxEvents(NonZeroUsize::new(100).unwrap()), + when_full: WhenFull::Block, + }, + ); + } + + #[test] + fn parse_memory_with_byte_size_option() { + check_single_stage( + r" + max_size: 4096 + ", + BufferType::Memory { + size: MemoryBufferSize::MaxSize(NonZeroUsize::new(4096).unwrap()), when_full: WhenFull::Block, }, ); @@ -455,11 +497,11 @@ max_events: 42 ", &[ BufferType::Memory { - max_events: NonZeroUsize::new(42).unwrap(), + size: MemoryBufferSize::MaxEvents(NonZeroUsize::new(42).unwrap()), when_full: WhenFull::Block, }, BufferType::Memory { - max_events: NonZeroUsize::new(100).unwrap(), + size: MemoryBufferSize::MaxEvents(NonZeroUsize::new(100).unwrap()), when_full: WhenFull::DropNewest, }, ], @@ -473,7 +515,7 @@ max_events: 42 type: memory ", BufferType::Memory { - max_events: NonZeroUsize::new(500).unwrap(), + size: MemoryBufferSize::MaxEvents(NonZeroUsize::new(500).unwrap()), when_full: WhenFull::Block, }, ); @@ -484,7 +526,7 @@ max_events: 42 max_events: 100 ", BufferType::Memory { - max_events: NonZeroUsize::new(100).unwrap(), + size: MemoryBufferSize::MaxEvents(NonZeroUsize::new(100).unwrap()), when_full: WhenFull::Block, }, ); @@ -495,7 +537,7 @@ max_events: 42 when_full: drop_newest ", BufferType::Memory { - max_events: NonZeroUsize::new(500).unwrap(), + size: MemoryBufferSize::MaxEvents(NonZeroUsize::new(500).unwrap()), when_full: WhenFull::DropNewest, }, ); @@ -506,7 +548,7 @@ max_events: 42 when_full: overflow ", BufferType::Memory { - max_events: NonZeroUsize::new(500).unwrap(), + size: MemoryBufferSize::MaxEvents(NonZeroUsize::new(500).unwrap()), when_full: WhenFull::Overflow, }, ); diff --git a/lib/vector-buffers/src/internal_events.rs b/lib/vector-buffers/src/internal_events.rs index 0cda5beec4..2b70980790 100644 --- a/lib/vector-buffers/src/internal_events.rs +++ b/lib/vector-buffers/src/internal_events.rs @@ -1,104 +1,182 @@ use std::time::Duration; -use metrics::{counter, gauge, histogram, Histogram}; +use metrics::{Histogram, counter, gauge, histogram}; use vector_common::{ - internal_event::{error_type, InternalEvent}, + internal_event::{InternalEvent, error_type}, registered_event, }; pub struct BufferCreated { + pub buffer_id: String, pub idx: usize, pub max_size_events: usize, pub max_size_bytes: u64, } impl InternalEvent for BufferCreated { - #[allow(clippy::cast_precision_loss)] + #[expect(clippy::cast_precision_loss)] fn emit(self) { if self.max_size_events != 0 { - gauge!("buffer_max_event_size", "stage" => self.idx.to_string()) - .set(self.max_size_events as f64); + gauge!( + "buffer_max_event_size", + "buffer_id" => self.buffer_id.clone(), + "stage" => self.idx.to_string(), + ) + .set(self.max_size_events as f64); } if self.max_size_bytes != 0 { - gauge!("buffer_max_byte_size", "stage" => self.idx.to_string()) - .set(self.max_size_bytes as f64); + gauge!( + "buffer_max_byte_size", + "buffer_id" => self.buffer_id, + "stage" => self.idx.to_string(), + ) + .set(self.max_size_bytes as f64); } } } pub struct BufferEventsReceived { + pub buffer_id: String, pub idx: usize, pub count: u64, pub byte_size: u64, + pub total_count: u64, + pub total_byte_size: u64, } impl InternalEvent for BufferEventsReceived { - #[allow(clippy::cast_precision_loss)] + #[expect(clippy::cast_precision_loss)] fn emit(self) { - counter!("buffer_received_events_total", "stage" => self.idx.to_string()) - .increment(self.count); - counter!("buffer_received_bytes_total", "stage" => self.idx.to_string()) - .increment(self.byte_size); - gauge!("buffer_events", "stage" => self.idx.to_string()).increment(self.count as f64); - gauge!("buffer_byte_size", "stage" => self.idx.to_string()) - .increment(self.byte_size as f64); + counter!( + "buffer_received_events_total", + "buffer_id" => self.buffer_id.clone(), + "stage" => self.idx.to_string() + ) + .increment(self.count); + + counter!( + "buffer_received_bytes_total", + "buffer_id" => self.buffer_id.clone(), + "stage" => self.idx.to_string() + ) + .increment(self.byte_size); + gauge!( + "buffer_events", + "buffer_id" => self.buffer_id.clone(), + "stage" => self.idx.to_string() + ) + .set(self.total_count as f64); + gauge!( + "buffer_byte_size", + "buffer_id" => self.buffer_id, + "stage" => self.idx.to_string() + ) + .set(self.total_byte_size as f64); } } pub struct BufferEventsSent { + pub buffer_id: String, pub idx: usize, pub count: u64, pub byte_size: u64, + pub total_count: u64, + pub total_byte_size: u64, } impl InternalEvent for BufferEventsSent { - #[allow(clippy::cast_precision_loss)] + #[expect(clippy::cast_precision_loss)] fn emit(self) { - counter!("buffer_sent_events_total", "stage" => self.idx.to_string()).increment(self.count); - counter!("buffer_sent_bytes_total", "stage" => self.idx.to_string()) - .increment(self.byte_size); - gauge!("buffer_events", "stage" => self.idx.to_string()).decrement(self.count as f64); - gauge!("buffer_byte_size", "stage" => self.idx.to_string()) - .decrement(self.byte_size as f64); + counter!( + "buffer_sent_events_total", + "buffer_id" => self.buffer_id.clone(), + "stage" => self.idx.to_string() + ) + .increment(self.count); + counter!( + "buffer_sent_bytes_total", + "buffer_id" => self.buffer_id.clone(), + "stage" => self.idx.to_string() + ) + .increment(self.byte_size); + gauge!( + "buffer_events", + "buffer_id" => self.buffer_id.clone(), + "stage" => self.idx.to_string() + ) + .set(self.total_count as f64); + gauge!( + "buffer_byte_size", + "buffer_id" => self.buffer_id, + "stage" => self.idx.to_string() + ) + .set(self.total_byte_size as f64); } } pub struct BufferEventsDropped { + pub buffer_id: String, pub idx: usize, pub count: u64, pub byte_size: u64, + pub total_count: u64, + pub total_byte_size: u64, pub intentional: bool, pub reason: &'static str, } impl InternalEvent for BufferEventsDropped { - #[allow(clippy::cast_precision_loss)] + #[expect(clippy::cast_precision_loss)] fn emit(self) { let intentional_str = if self.intentional { "true" } else { "false" }; if self.intentional { debug!( message = "Events dropped.", count = %self.count, + byte_size = %self.byte_size, intentional = %intentional_str, reason = %self.reason, + buffer_id = %self.buffer_id, stage = %self.idx, ); } else { error!( message = "Events dropped.", count = %self.count, + byte_size = %self.byte_size, intentional = %intentional_str, reason = %self.reason, + buffer_id = %self.buffer_id, stage = %self.idx, ); } + counter!( - "buffer_discarded_events_total", "intentional" => intentional_str, + "buffer_discarded_events_total", + "buffer_id" => self.buffer_id.clone(), + "stage" => self.idx.to_string(), + "intentional" => intentional_str, ) .increment(self.count); - gauge!("buffer_events", "stage" => self.idx.to_string()).decrement(self.count as f64); - gauge!("buffer_byte_size", "stage" => self.idx.to_string()) - .decrement(self.byte_size as f64); + counter!( + "buffer_discarded_bytes_total", + "buffer_id" => self.buffer_id.clone(), + "stage" => self.idx.to_string(), + "intentional" => intentional_str, + ) + .increment(self.byte_size); + gauge!( + "buffer_events", + "buffer_id" => self.buffer_id.clone(), + "stage" => self.idx.to_string() + ) + .set(self.total_count as f64); + gauge!( + "buffer_byte_size", + "buffer_id" => self.buffer_id, + "stage" => self.idx.to_string() + ) + .set(self.total_byte_size as f64); } } diff --git a/lib/vector-buffers/src/lib.rs b/lib/vector-buffers/src/lib.rs index 84e88b6591..795c8bb0ee 100644 --- a/lib/vector-buffers/src/lib.rs +++ b/lib/vector-buffers/src/lib.rs @@ -17,7 +17,7 @@ extern crate tracing; mod buffer_usage_data; pub mod config; -pub use config::{BufferConfig, BufferType}; +pub use config::{BufferConfig, BufferType, MemoryBufferSize}; use encoding::Encodable; use vector_config::configurable_component; diff --git a/lib/vector-buffers/src/test/action.rs b/lib/vector-buffers/src/test/action.rs index d33362f987..14e1a2e655 100644 --- a/lib/vector-buffers/src/test/action.rs +++ b/lib/vector-buffers/src/test/action.rs @@ -1,4 +1,4 @@ -use quickcheck::{single_shrinker, Arbitrary, Gen}; +use quickcheck::{Arbitrary, Gen, single_shrinker}; use crate::test::Message; diff --git a/lib/vector-buffers/src/test/helpers.rs b/lib/vector-buffers/src/test/helpers.rs index 5978a72c6f..af7c067119 100644 --- a/lib/vector-buffers/src/test/helpers.rs +++ b/lib/vector-buffers/src/test/helpers.rs @@ -2,7 +2,7 @@ use std::{future::Future, path::Path, str::FromStr, sync::LazyLock}; use temp_dir::TempDir; use tracing_fluent_assertions::{AssertionRegistry, AssertionsLayer}; -use tracing_subscriber::{filter::LevelFilter, layer::SubscriberExt, Layer, Registry}; +use tracing_subscriber::{Layer, Registry, filter::LevelFilter, layer::SubscriberExt}; use vector_common::finalization::{EventStatus, Finalizable}; #[macro_export] diff --git a/lib/vector-buffers/src/test/messages.rs b/lib/vector-buffers/src/test/messages.rs index 6d73d02093..628831fb5e 100644 --- a/lib/vector-buffers/src/test/messages.rs +++ b/lib/vector-buffers/src/test/messages.rs @@ -7,7 +7,7 @@ use vector_common::finalization::{ AddBatchNotifier, BatchNotifier, EventFinalizer, EventFinalizers, Finalizable, }; -use crate::{encoding::FixedEncodable, EventCount}; +use crate::{EventCount, encoding::FixedEncodable}; macro_rules! message_wrapper { ($id:ident: $ty:ty, $event_count:expr) => { @@ -160,14 +160,11 @@ impl FixedEncodable for SizedRecord { { let minimum_len = self.encoded_len(); if buffer.remaining_mut() < minimum_len { - return Err(io::Error::new( - io::ErrorKind::Other, - format!( - "not enough capacity to encode record: need {}, only have {}", - minimum_len, - buffer.remaining_mut() - ), - )); + return Err(io::Error::other(format!( + "not enough capacity to encode record: need {}, only have {}", + minimum_len, + buffer.remaining_mut() + ))); } buffer.put_u32(self.0); @@ -219,10 +216,7 @@ impl FixedEncodable for UndecodableRecord { B: BufMut, { if buffer.remaining_mut() < 4 { - return Err(io::Error::new( - io::ErrorKind::Other, - "not enough capacity to encode record", - )); + return Err(io::Error::other("not enough capacity to encode record")); } buffer.put_u32(42); @@ -233,7 +227,7 @@ impl FixedEncodable for UndecodableRecord { where B: Buf, { - Err(io::Error::new(io::ErrorKind::Other, "failed to decode")) + Err(io::Error::other("failed to decode")) } } @@ -254,10 +248,7 @@ impl FixedEncodable for MultiEventRecord { B: BufMut, { if buffer.remaining_mut() < self.encoded_size() { - return Err(io::Error::new( - io::ErrorKind::Other, - "not enough capacity to encode record", - )); + return Err(io::Error::other("not enough capacity to encode record")); } buffer.put_u32(self.0); @@ -274,45 +265,3 @@ impl FixedEncodable for MultiEventRecord { Ok(MultiEventRecord::new(event_count)) } } - -message_wrapper!(PoisonPillMultiEventRecord: u32, |m: &Self| m.0); - -impl PoisonPillMultiEventRecord { - pub fn encoded_size(&self) -> usize { - usize::try_from(self.0).unwrap_or(usize::MAX) + std::mem::size_of::() - } -} - -impl FixedEncodable for PoisonPillMultiEventRecord { - type EncodeError = io::Error; - type DecodeError = io::Error; - - fn encode(self, buffer: &mut B) -> Result<(), Self::EncodeError> - where - B: BufMut, - { - if buffer.remaining_mut() < self.encoded_size() { - return Err(io::Error::new( - io::ErrorKind::Other, - "not enough capacity to encode record", - )); - } - - buffer.put_u32(self.0); - buffer.put_bytes(0x42, usize::try_from(self.0).unwrap_or(usize::MAX)); - Ok(()) - } - - fn decode(mut buffer: B) -> Result - where - B: Buf, - { - let event_count = buffer.get_u32(); - if event_count == 42 { - return Err(io::Error::new(io::ErrorKind::Other, "failed to decode")); - } - - buffer.advance(usize::try_from(event_count).unwrap_or(usize::MAX)); - Ok(PoisonPillMultiEventRecord::new(event_count)) - } -} diff --git a/lib/vector-buffers/src/test/variant.rs b/lib/vector-buffers/src/test/variant.rs index a14995d86b..f8d27432eb 100644 --- a/lib/vector-buffers/src/test/variant.rs +++ b/lib/vector-buffers/src/test/variant.rs @@ -1,5 +1,5 @@ use std::{ - num::{NonZeroU16, NonZeroU64, NonZeroUsize}, + num::{NonZeroU16, NonZeroU64}, path::PathBuf, }; @@ -9,12 +9,12 @@ use tracing::Span; use vector_common::finalization::Finalizable; use crate::{ + Bufferable, MemoryBufferSize, WhenFull, topology::{ builder::TopologyBuilder, channel::{BufferReceiver, BufferSender}, }, variants::{DiskV2Buffer, MemoryBuffer}, - Bufferable, WhenFull, }; #[cfg(test)] @@ -31,7 +31,7 @@ const ALPHABET: [&str; 27] = [ #[derive(Debug, Clone)] pub enum Variant { Memory { - max_events: NonZeroUsize, + size: MemoryBufferSize, when_full: WhenFull, }, DiskV2 { @@ -50,11 +50,9 @@ impl Variant { let mut builder = TopologyBuilder::default(); match self { Variant::Memory { - max_events, - when_full, - .. + size, when_full, .. } => { - builder.stage(MemoryBuffer::new(*max_events), *when_full); + builder.stage(MemoryBuffer::new(*size), *when_full); } Variant::DiskV2 { max_size, @@ -67,7 +65,7 @@ impl Variant { *when_full, ); } - }; + } let (sender, receiver) = builder .build(String::from("benches"), Span::none()) @@ -105,14 +103,12 @@ impl Arbitrary for Variant { // Using a u16 ensures we avoid any allocation errors for our holding buffers, etc. let max_events = NonZeroU16::arbitrary(g).into(); let max_size = NonZeroU16::arbitrary(g).into(); + let size = MemoryBufferSize::MaxEvents(max_events); let when_full = WhenFull::arbitrary(g); if use_memory_buffer { - Variant::Memory { - max_events, - when_full, - } + Variant::Memory { size, when_full } } else { Variant::DiskV2 { max_size, @@ -126,15 +122,23 @@ impl Arbitrary for Variant { fn shrink(&self) -> Box> { match self { Variant::Memory { - max_events, - when_full, - .. + size, when_full, .. } => { let when_full = *when_full; - Box::new(max_events.shrink().map(move |me| Variant::Memory { - max_events: me, - when_full, - })) + match size { + MemoryBufferSize::MaxEvents(max_events) => { + Box::new(max_events.shrink().map(move |me| Variant::Memory { + size: MemoryBufferSize::MaxEvents(me), + when_full, + })) + } + MemoryBufferSize::MaxSize(max_size) => { + Box::new(max_size.shrink().map(move |me| Variant::Memory { + size: MemoryBufferSize::MaxSize(me), + when_full, + })) + } + } } Variant::DiskV2 { max_size, diff --git a/lib/vector-buffers/src/topology/builder.rs b/lib/vector-buffers/src/topology/builder.rs index 51f52900ec..0d909a7eb1 100644 --- a/lib/vector-buffers/src/topology/builder.rs +++ b/lib/vector-buffers/src/topology/builder.rs @@ -6,10 +6,10 @@ use tracing::Span; use super::channel::{ReceiverAdapter, SenderAdapter}; use crate::{ + Bufferable, WhenFull, buffer_usage_data::{BufferUsage, BufferUsageHandle}, topology::channel::{BufferReceiver, BufferSender}, variants::MemoryBuffer, - Bufferable, WhenFull, }; /// Value that can be used as a stage in a buffer topology. @@ -129,7 +129,7 @@ impl TopologyBuilder { return Err(TopologyError::NextStageNotUsed { stage_idx }); } } - }; + } // Create the buffer usage handle for this stage and initialize it as we create the // sender/receiver. This is slightly awkward since we just end up actually giving @@ -193,7 +193,7 @@ impl TopologyBuilder { ) -> (BufferSender, BufferReceiver) { let usage_handle = BufferUsageHandle::noop(); - let memory_buffer = Box::new(MemoryBuffer::new(max_events)); + let memory_buffer = Box::new(MemoryBuffer::with_max_events(max_events)); let (sender, receiver) = memory_buffer .into_buffer_parts(usage_handle.clone()) .await @@ -229,7 +229,7 @@ impl TopologyBuilder { when_full: WhenFull, usage_handle: BufferUsageHandle, ) -> (BufferSender, BufferReceiver) { - let memory_buffer = Box::new(MemoryBuffer::new(max_events)); + let memory_buffer = Box::new(MemoryBuffer::with_max_events(max_events)); let (sender, receiver) = memory_buffer .into_buffer_parts(usage_handle.clone()) .await @@ -263,17 +263,19 @@ mod tests { use super::TopologyBuilder; use crate::{ - topology::builder::TopologyError, - topology::test_util::{assert_current_send_capacity, Sample}, - variants::MemoryBuffer, WhenFull, + topology::{ + builder::TopologyError, + test_util::{Sample, assert_current_send_capacity}, + }, + variants::MemoryBuffer, }; #[tokio::test] async fn single_stage_topology_block() { let mut builder = TopologyBuilder::::default(); builder.stage( - MemoryBuffer::new(NonZeroUsize::new(1).unwrap()), + MemoryBuffer::with_max_events(NonZeroUsize::new(1).unwrap()), WhenFull::Block, ); let result = builder.build(String::from("test"), Span::none()).await; @@ -287,7 +289,7 @@ mod tests { async fn single_stage_topology_drop_newest() { let mut builder = TopologyBuilder::::default(); builder.stage( - MemoryBuffer::new(NonZeroUsize::new(1).unwrap()), + MemoryBuffer::with_max_events(NonZeroUsize::new(1).unwrap()), WhenFull::DropNewest, ); let result = builder.build(String::from("test"), Span::none()).await; @@ -301,7 +303,7 @@ mod tests { async fn single_stage_topology_overflow() { let mut builder = TopologyBuilder::::default(); builder.stage( - MemoryBuffer::new(NonZeroUsize::new(1).unwrap()), + MemoryBuffer::with_max_events(NonZeroUsize::new(1).unwrap()), WhenFull::Overflow, ); let result = builder.build(String::from("test"), Span::none()).await; @@ -315,11 +317,11 @@ mod tests { async fn two_stage_topology_block() { let mut builder = TopologyBuilder::::default(); builder.stage( - MemoryBuffer::new(NonZeroUsize::new(1).unwrap()), + MemoryBuffer::with_max_events(NonZeroUsize::new(1).unwrap()), WhenFull::Block, ); builder.stage( - MemoryBuffer::new(NonZeroUsize::new(1).unwrap()), + MemoryBuffer::with_max_events(NonZeroUsize::new(1).unwrap()), WhenFull::Block, ); let result = builder.build(String::from("test"), Span::none()).await; @@ -333,11 +335,11 @@ mod tests { async fn two_stage_topology_drop_newest() { let mut builder = TopologyBuilder::::default(); builder.stage( - MemoryBuffer::new(NonZeroUsize::new(1).unwrap()), + MemoryBuffer::with_max_events(NonZeroUsize::new(1).unwrap()), WhenFull::DropNewest, ); builder.stage( - MemoryBuffer::new(NonZeroUsize::new(1).unwrap()), + MemoryBuffer::with_max_events(NonZeroUsize::new(1).unwrap()), WhenFull::Block, ); let result = builder.build(String::from("test"), Span::none()).await; @@ -351,11 +353,11 @@ mod tests { async fn two_stage_topology_overflow() { let mut builder = TopologyBuilder::::default(); builder.stage( - MemoryBuffer::new(NonZeroUsize::new(1).unwrap()), + MemoryBuffer::with_max_events(NonZeroUsize::new(1).unwrap()), WhenFull::Overflow, ); builder.stage( - MemoryBuffer::new(NonZeroUsize::new(1).unwrap()), + MemoryBuffer::with_max_events(NonZeroUsize::new(1).unwrap()), WhenFull::Block, ); diff --git a/lib/vector-buffers/src/topology/channel/limited_queue.rs b/lib/vector-buffers/src/topology/channel/limited_queue.rs index 887ce2639a..4320936cec 100644 --- a/lib/vector-buffers/src/topology/channel/limited_queue.rs +++ b/lib/vector-buffers/src/topology/channel/limited_queue.rs @@ -1,18 +1,19 @@ use std::{ cmp, fmt, + fmt::Debug, pin::Pin, sync::{ - atomic::{AtomicUsize, Ordering}, Arc, + atomic::{AtomicUsize, Ordering}, }, }; use async_stream::stream; -use crossbeam_queue::ArrayQueue; +use crossbeam_queue::{ArrayQueue, SegQueue}; use futures::Stream; use tokio::sync::{Notify, OwnedSemaphorePermit, Semaphore, TryAcquireError}; -use crate::InMemoryBufferable; +use crate::{InMemoryBufferable, config::MemoryBufferSize}; /// Error returned by `LimitedSender::send` when the receiver has disconnected. #[derive(Debug, PartialEq, Eq)] @@ -54,10 +55,43 @@ impl fmt::Display for TrySendError { impl std::error::Error for TrySendError {} +// Trait over common queue operations so implementation can be chosen at initialization phase +trait QueueImpl: Send + Sync + fmt::Debug { + fn push(&self, item: T); + fn pop(&self) -> Option; +} + +impl QueueImpl for ArrayQueue +where + T: Send + Sync + fmt::Debug, +{ + fn push(&self, item: T) { + self.push(item) + .unwrap_or_else(|_| unreachable!("acquired permits but channel reported being full.")); + } + + fn pop(&self) -> Option { + self.pop() + } +} + +impl QueueImpl for SegQueue +where + T: Send + Sync + fmt::Debug, +{ + fn push(&self, item: T) { + self.push(item); + } + + fn pop(&self) -> Option { + self.pop() + } +} + #[derive(Debug)] struct Inner { - data: Arc>, - limit: usize, + data: Arc>, + limit: MemoryBufferSize, limiter: Arc, read_waker: Arc, } @@ -85,7 +119,11 @@ impl LimitedSender { // We have to limit the number of permits we ask for to the overall limit since we're always // willing to store more items than the limit if the queue is entirely empty, because // otherwise we might deadlock ourselves by not being able to send a single item. - cmp::min(self.inner.limit, item.event_count()) as u32 + let (limit, value) = match self.inner.limit { + MemoryBufferSize::MaxSize(max_size) => (max_size, item.allocated_bytes()), + MemoryBufferSize::MaxEvents(max_events) => (max_events, item.event_count()), + }; + cmp::min(limit.get(), value) as u32 } /// Gets the number of items that this channel could accept. @@ -112,10 +150,7 @@ impl LimitedSender { return Err(SendError(item)); }; - self.inner - .data - .push((permits, item)) - .unwrap_or_else(|_| unreachable!("acquired permits but channel reported being full")); + self.inner.data.push((permits, item)); self.inner.read_waker.notify_one(); trace!("Sent item."); @@ -149,14 +184,11 @@ impl LimitedSender { return match ae { TryAcquireError::NoPermits => Err(TrySendError::InsufficientCapacity(item)), TryAcquireError::Closed => Err(TrySendError::Disconnected(item)), - } + }; } }; - self.inner - .data - .push((permits, item)) - .unwrap_or_else(|_| unreachable!("acquired permits but channel reported being full")); + self.inner.data.push((permits, item)); self.inner.read_waker.notify_one(); trace!("Attempt to send item succeeded."); @@ -235,12 +267,22 @@ impl Drop for LimitedReceiver { } } -pub fn limited(limit: usize) -> (LimitedSender, LimitedReceiver) { - let inner = Inner { - data: Arc::new(ArrayQueue::new(limit)), - limit, - limiter: Arc::new(Semaphore::new(limit)), - read_waker: Arc::new(Notify::new()), +pub fn limited( + limit: MemoryBufferSize, +) -> (LimitedSender, LimitedReceiver) { + let inner = match limit { + MemoryBufferSize::MaxEvents(max_events) => Inner { + data: Arc::new(ArrayQueue::new(max_events.get())), + limit, + limiter: Arc::new(Semaphore::new(max_events.get())), + read_waker: Arc::new(Notify::new()), + }, + MemoryBufferSize::MaxSize(max_size) => Inner { + data: Arc::new(SegQueue::new()), + limit, + limiter: Arc::new(Semaphore::new(max_size.get())), + read_waker: Arc::new(Notify::new()), + }, }; let sender = LimitedSender { @@ -254,21 +296,25 @@ pub fn limited(limit: usize) -> (LimitedSender, LimitedReceiver) { #[cfg(test)] mod tests { + use std::num::NonZeroUsize; + use tokio_test::{assert_pending, assert_ready, task::spawn}; + use vector_common::byte_size_of::ByteSizeOf; use super::limited; use crate::{ - test::MultiEventRecord, topology::channel::limited_queue::SendError, - topology::test_util::Sample, + MemoryBufferSize, + test::MultiEventRecord, + topology::{channel::limited_queue::SendError, test_util::Sample}, }; #[tokio::test] async fn send_receive() { - let (mut tx, mut rx) = limited(2); + let (mut tx, mut rx) = limited(MemoryBufferSize::MaxEvents(NonZeroUsize::new(2).unwrap())); assert_eq!(2, tx.available_capacity()); - let msg = Sample(42); + let msg = Sample::new(42); // Create our send and receive futures. let mut send = spawn(async { tx.send(msg).await }); @@ -287,17 +333,58 @@ mod tests { // Now our receive should have been woken up, and should immediately be ready. assert!(recv.is_woken()); - assert_eq!(Some(msg), assert_ready!(recv.poll())); + assert_eq!(Some(Sample::new(42)), assert_ready!(recv.poll())); + } + + #[test] + fn test_limiting_by_byte_size() { + let max_elements = 10; + let msg = Sample::new_with_heap_allocated_values(50); + let msg_size = msg.allocated_bytes(); + let max_allowed_bytes = msg_size * max_elements; + + // With this configuration a maximum of exactly 10 messages can fit in the channel + let (mut tx, mut rx) = limited(MemoryBufferSize::MaxSize( + NonZeroUsize::new(max_allowed_bytes).unwrap(), + )); + + assert_eq!(max_allowed_bytes, tx.available_capacity()); + + // Send 10 messages into the channel, filling it + for _ in 0..10 { + let msg_clone = msg.clone(); + let mut f = spawn(async { tx.send(msg_clone).await }); + assert_eq!(Ok(()), assert_ready!(f.poll())); + } + // With the 10th message in the channel no space should be left + assert_eq!(0, tx.available_capacity()); + + // Attemting to produce one more then the max capacity should block + let mut send_final = spawn({ + let msg_clone = msg.clone(); + async { tx.send(msg_clone).await } + }); + assert_pending!(send_final.poll()); + + // Read all data from the channel, assert final states are as expected + for _ in 0..10 { + let mut f = spawn(async { rx.next().await }); + let value = assert_ready!(f.poll()); + assert_eq!(value.allocated_bytes(), msg_size); + } + // Channel should have no more data + let mut recv = spawn(async { rx.next().await }); + assert_pending!(recv.poll()); } #[test] fn sender_waits_for_more_capacity_when_none_available() { - let (mut tx, mut rx) = limited(1); + let (mut tx, mut rx) = limited(MemoryBufferSize::MaxEvents(NonZeroUsize::new(1).unwrap())); assert_eq!(1, tx.available_capacity()); - let msg1 = Sample(42); - let msg2 = Sample(43); + let msg1 = Sample::new(42); + let msg2 = Sample::new(43); // Create our send and receive futures. let mut send1 = spawn(async { tx.send(msg1).await }); @@ -328,7 +415,7 @@ mod tests { assert_pending!(send2.poll()); // Now if we receive the item, our second send should be woken up and be able to send in. - assert_eq!(Some(msg1), assert_ready!(recv1.poll())); + assert_eq!(Some(Sample::new(42)), assert_ready!(recv1.poll())); drop(recv1); // Since the second send was already waiting for permits, the semaphore returns them @@ -346,14 +433,14 @@ mod tests { // And the final receive to get our second send: assert!(recv2.is_woken()); - assert_eq!(Some(msg2), assert_ready!(recv2.poll())); + assert_eq!(Some(Sample::new(43)), assert_ready!(recv2.poll())); assert_eq!(1, tx.available_capacity()); } #[test] fn sender_waits_for_more_capacity_when_partial_available() { - let (mut tx, mut rx) = limited(7); + let (mut tx, mut rx) = limited(MemoryBufferSize::MaxEvents(NonZeroUsize::new(7).unwrap())); assert_eq!(7, tx.available_capacity()); @@ -441,12 +528,12 @@ mod tests { #[test] fn empty_receiver_returns_none_when_last_sender_drops() { - let (mut tx, mut rx) = limited(1); + let (mut tx, mut rx) = limited(MemoryBufferSize::MaxEvents(NonZeroUsize::new(1).unwrap())); assert_eq!(1, tx.available_capacity()); let tx2 = tx.clone(); - let msg = Sample(42); + let msg = Sample::new(42); // Create our send and receive futures. let mut send = spawn(async { tx.send(msg).await }); @@ -473,7 +560,7 @@ mod tests { drop(tx); assert!(recv.is_woken()); - assert_eq!(Some(msg), assert_ready!(recv.poll())); + assert_eq!(Some(Sample::new(42)), assert_ready!(recv.poll())); drop(recv); let mut recv2 = spawn(async { rx.next().await }); @@ -483,7 +570,8 @@ mod tests { #[test] fn receiver_returns_none_once_empty_when_last_sender_drops() { - let (tx, mut rx) = limited::(1); + let (tx, mut rx) = + limited::(MemoryBufferSize::MaxEvents(NonZeroUsize::new(1).unwrap())); assert_eq!(1, tx.available_capacity()); @@ -512,7 +600,7 @@ mod tests { #[test] fn oversized_send_allowed_when_empty() { - let (mut tx, mut rx) = limited(1); + let (mut tx, mut rx) = limited(MemoryBufferSize::MaxEvents(NonZeroUsize::new(1).unwrap())); assert_eq!(1, tx.available_capacity()); @@ -544,7 +632,7 @@ mod tests { #[test] fn oversized_send_allowed_when_partial_capacity() { - let (mut tx, mut rx) = limited(2); + let (mut tx, mut rx) = limited(MemoryBufferSize::MaxEvents(NonZeroUsize::new(2).unwrap())); assert_eq!(2, tx.available_capacity()); diff --git a/lib/vector-buffers/src/topology/channel/mod.rs b/lib/vector-buffers/src/topology/channel/mod.rs index d9a1d06926..b6bb0bb966 100644 --- a/lib/vector-buffers/src/topology/channel/mod.rs +++ b/lib/vector-buffers/src/topology/channel/mod.rs @@ -2,7 +2,7 @@ mod limited_queue; mod receiver; mod sender; -pub use limited_queue::{limited, LimitedReceiver, LimitedSender, SendError}; +pub use limited_queue::{LimitedReceiver, LimitedSender, SendError, limited}; pub use receiver::*; pub use sender::*; diff --git a/lib/vector-buffers/src/topology/channel/receiver.rs b/lib/vector-buffers/src/topology/channel/receiver.rs index d21aa1ed67..c5bb870f1c 100644 --- a/lib/vector-buffers/src/topology/channel/receiver.rs +++ b/lib/vector-buffers/src/topology/channel/receiver.rs @@ -1,7 +1,7 @@ use std::{ mem, pin::Pin, - task::{ready, Context, Poll}, + task::{Context, Poll, ready}, }; use async_recursion::async_recursion; @@ -12,12 +12,13 @@ use vector_common::internal_event::emit; use super::limited_queue::LimitedReceiver; use crate::{ + Bufferable, buffer_usage_data::BufferUsageHandle, variants::disk_v2::{self, ProductionFilesystem}, - Bufferable, }; /// Adapter for papering over various receiver backends. +#[allow(clippy::large_enum_variant)] #[derive(Debug)] pub enum ReceiverAdapter { /// The in-memory channel buffer. @@ -54,7 +55,6 @@ where // If we've hit a recoverable error, we'll emit an event to indicate as much but we'll still // keep trying to read the next available record. emit(re); - continue; } None => panic!("Reader encountered unrecoverable error: {e:?}"), }, @@ -140,13 +140,13 @@ impl BufferReceiver { // If instrumentation is enabled, and we got the item from the base receiver, then and only // then do we track sending the event out. - if let Some(handle) = self.instrumentation.as_ref() { - if from_base { - handle.increment_sent_event_count_and_byte_size( - item.event_count() as u64, - item.size_of() as u64, - ); - } + if let Some(handle) = self.instrumentation.as_ref() + && from_base + { + handle.increment_sent_event_count_and_byte_size( + item.event_count() as u64, + item.size_of() as u64, + ); } Some(item) @@ -157,6 +157,7 @@ impl BufferReceiver { } } +#[allow(clippy::large_enum_variant)] enum StreamState { Idle(BufferReceiver), Polling, diff --git a/lib/vector-buffers/src/topology/channel/sender.rs b/lib/vector-buffers/src/topology/channel/sender.rs index 15191ec02e..cdfc41912f 100644 --- a/lib/vector-buffers/src/topology/channel/sender.rs +++ b/lib/vector-buffers/src/topology/channel/sender.rs @@ -4,14 +4,14 @@ use async_recursion::async_recursion; use derivative::Derivative; use tokio::sync::Mutex; use tracing::Span; -use vector_common::internal_event::{register, InternalEventHandle, Registered}; +use vector_common::internal_event::{InternalEventHandle, Registered, register}; use super::limited_queue::LimitedSender; use crate::{ + Bufferable, WhenFull, buffer_usage_data::BufferUsageHandle, internal_events::BufferSendDuration, variants::disk_v2::{self, ProductionFilesystem}, - Bufferable, WhenFull, }; /// Adapter for papering over various sender backends. @@ -221,32 +221,31 @@ impl BufferSender { .await?; } } - }; + } - if sent_to_base || was_dropped { - if let (Some(send_duration), Some(send_reference)) = + if (sent_to_base || was_dropped) + && let (Some(send_duration), Some(send_reference)) = (self.send_duration.as_ref(), send_reference) - { - send_duration.emit(send_reference.elapsed()); - } + { + send_duration.emit(send_reference.elapsed()); } - if let Some(instrumentation) = self.instrumentation.as_ref() { - if let Some((item_count, item_size)) = item_sizing { - if sent_to_base { - instrumentation.increment_received_event_count_and_byte_size( - item_count as u64, - item_size as u64, - ); - } + if let Some(instrumentation) = self.instrumentation.as_ref() + && let Some((item_count, item_size)) = item_sizing + { + if sent_to_base { + instrumentation.increment_received_event_count_and_byte_size( + item_count as u64, + item_size as u64, + ); + } - if was_dropped { - instrumentation.increment_dropped_event_count_and_byte_size( - item_count as u64, - item_size as u64, - true, - ); - } + if was_dropped { + instrumentation.increment_dropped_event_count_and_byte_size( + item_count as u64, + item_size as u64, + true, + ); } } diff --git a/lib/vector-buffers/src/topology/channel/tests.rs b/lib/vector-buffers/src/topology/channel/tests.rs index a1c68c8071..3ebfdda32a 100644 --- a/lib/vector-buffers/src/topology/channel/tests.rs +++ b/lib/vector-buffers/src/topology/channel/tests.rs @@ -6,11 +6,11 @@ use std::{ use tokio::{pin, sync::Barrier, time::sleep}; use crate::{ + Bufferable, WhenFull, topology::{ channel::{BufferReceiver, BufferSender}, test_util::{assert_current_send_capacity, build_buffer}, }, - Bufferable, WhenFull, }; async fn assert_send_ok_with_capacities( diff --git a/lib/vector-buffers/src/topology/test_util.rs b/lib/vector-buffers/src/topology/test_util.rs index ec0acac741..bc8af153e8 100644 --- a/lib/vector-buffers/src/topology/test_util.rs +++ b/lib/vector-buffers/src/topology/test_util.rs @@ -6,24 +6,45 @@ use vector_common::finalization::{AddBatchNotifier, BatchNotifier}; use super::builder::TopologyBuilder; use crate::{ + Bufferable, EventCount, WhenFull, buffer_usage_data::BufferUsageHandle, encoding::FixedEncodable, topology::channel::{BufferReceiver, BufferSender}, - Bufferable, EventCount, WhenFull, }; -#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] -pub(crate) struct Sample(pub u64); +const SINGLE_VALUE_FLAG: u8 = 0; +const HEAP_ALLOCATED_VALUES_FLAG: u8 = 1; + +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub(crate) enum Sample { + SingleValue(u64), + HeapAllocatedValues(Vec), +} impl From for Sample { fn from(v: u64) -> Self { - Self(v) + Self::SingleValue(v) } } impl From for u64 { - fn from(s: Sample) -> Self { - s.0 + fn from(v: Sample) -> Self { + match v { + Sample::SingleValue(sv) => sv, + Sample::HeapAllocatedValues(_) => { + panic!("Cannot use this API with other enum states of this type.") + } + } + } +} + +impl Sample { + pub fn new(value: u64) -> Self { + Self::SingleValue(value) + } + + pub fn new_with_heap_allocated_values(n: usize) -> Self { + Self::HeapAllocatedValues(vec![0; n]) } } @@ -35,7 +56,10 @@ impl AddBatchNotifier for Sample { impl ByteSizeOf for Sample { fn allocated_bytes(&self) -> usize { - 0 + match self { + Self::SingleValue(_) => 0, + Self::HeapAllocatedValues(uints) => uints.len() * 8, + } } } @@ -44,12 +68,29 @@ impl FixedEncodable for Sample { type EncodeError = BasicError; type DecodeError = BasicError; + // Serialization format: + // - Encode type flag + // - if single flag encode int value + // - otherwise encode array length and encode array contents fn encode(self, buffer: &mut B) -> Result<(), Self::EncodeError> where B: BufMut, Self: Sized, { - buffer.put_u64(self.0); + match self { + Self::SingleValue(uint) => { + buffer.put_u8(SINGLE_VALUE_FLAG); + buffer.put_u64(uint); + } + Self::HeapAllocatedValues(uints) => { + buffer.put_u8(HEAP_ALLOCATED_VALUES_FLAG); + // Prepend with array size + buffer.put_u32(u32::try_from(uints.len()).unwrap()); + for v in uints { + buffer.put_u64(v); + } + } + } Ok(()) } @@ -57,10 +98,16 @@ impl FixedEncodable for Sample { where B: Buf, { - if buffer.remaining() >= 8 { - Ok(Self(buffer.get_u64())) - } else { - Err(BasicError("need 8 bytes minimum".to_string())) + match buffer.get_u8() { + SINGLE_VALUE_FLAG => Ok(Self::SingleValue(buffer.get_u64())), + HEAP_ALLOCATED_VALUES_FLAG => { + let length = buffer.get_u32(); + let values = (0..length).map(|_| buffer.get_u64()).collect(); + Ok(Self::HeapAllocatedValues(values)) + } + _ => Err(BasicError( + "Unknown serialization flag observed".to_string(), + )), } } } diff --git a/lib/vector-buffers/src/variants/disk_v2/backed_archive.rs b/lib/vector-buffers/src/variants/disk_v2/backed_archive.rs index baf5323872..264d868d11 100644 --- a/lib/vector-buffers/src/variants/disk_v2/backed_archive.rs +++ b/lib/vector-buffers/src/variants/disk_v2/backed_archive.rs @@ -4,10 +4,9 @@ use std::pin::Pin; use bytecheck::CheckBytes; use rkyv::{ - archived_root, check_archived_root, - ser::{serializers::AllocSerializer, Serializer}, + Archive, Serialize, archived_root, check_archived_root, + ser::{Serializer, serializers::AllocSerializer}, validation::validators::DefaultValidator, - Archive, Serialize, }; use super::ser::{DeserializeError, SerializeError}; diff --git a/lib/vector-buffers/src/variants/disk_v2/common.rs b/lib/vector-buffers/src/variants/disk_v2/common.rs index e3426937c3..de38eb1cc1 100644 --- a/lib/vector-buffers/src/variants/disk_v2/common.rs +++ b/lib/vector-buffers/src/variants/disk_v2/common.rs @@ -424,7 +424,7 @@ mod tests { use crate::variants::disk_v2::common::MAX_ALIGNABLE_AMOUNT; use super::{ - align16, BuildError, DiskBufferConfigBuilder, MINIMUM_MAX_RECORD_SIZE, SERIALIZER_ALIGNMENT, + BuildError, DiskBufferConfigBuilder, MINIMUM_MAX_RECORD_SIZE, SERIALIZER_ALIGNMENT, align16, }; #[test] diff --git a/lib/vector-buffers/src/variants/disk_v2/ledger.rs b/lib/vector-buffers/src/variants/disk_v2/ledger.rs index 7f56f30b34..9ad8065536 100644 --- a/lib/vector-buffers/src/variants/disk_v2/ledger.rs +++ b/lib/vector-buffers/src/variants/disk_v2/ledger.rs @@ -1,8 +1,8 @@ use std::{ fmt, io, mem, path::PathBuf, - sync::atomic::{AtomicBool, AtomicU16, AtomicU64, Ordering}, sync::Arc, + sync::atomic::{AtomicBool, AtomicU16, AtomicU64, Ordering}, time::Instant, }; @@ -11,17 +11,17 @@ use bytes::BytesMut; use crossbeam_utils::atomic::AtomicCell; use fslock::LockFile; use futures::StreamExt; -use rkyv::{with::Atomic, Archive, Serialize}; +use rkyv::{Archive, Serialize, with::Atomic}; use snafu::{ResultExt, Snafu}; use tokio::{fs, io::AsyncWriteExt, sync::Notify}; use vector_common::finalizer::OrderedFinalizer; use super::{ + Filesystem, backed_archive::BackedArchive, - common::{align16, DiskBufferConfig, MAX_FILE_ID}, + common::{DiskBufferConfig, MAX_FILE_ID, align16}, io::{AsyncFile, WritableMemoryMap}, ser::SerializeError, - Filesystem, }; use crate::buffer_usage_data::BufferUsageHandle; @@ -465,11 +465,7 @@ where Ordering::Release, Ordering::Relaxed, |n| { - if n == 0 { - None - } else { - Some(n - 1) - } + if n == 0 { None } else { Some(n - 1) } }, ); @@ -603,7 +599,7 @@ where break; } Err(SerializeError::FailedToSerialize(reason)) => { - return Err(LedgerLoadCreateError::FailedToSerialize { reason }) + return Err(LedgerLoadCreateError::FailedToSerialize { reason }); } // Our buffer wasn't big enough, but that's OK! Resize it and try again. Err(SerializeError::BackingStoreTooSmall(_, min_len)) => buf.resize(min_len, 0), @@ -628,7 +624,7 @@ where Err(e) => { return Err(LedgerLoadCreateError::FailedToDeserialize { reason: e.into_inner(), - }) + }); } }; diff --git a/lib/vector-buffers/src/variants/disk_v2/mod.rs b/lib/vector-buffers/src/variants/disk_v2/mod.rs index e93738b178..a0b303581e 100644 --- a/lib/vector-buffers/src/variants/disk_v2/mod.rs +++ b/lib/vector-buffers/src/variants/disk_v2/mod.rs @@ -203,12 +203,12 @@ pub use self::{ writer::{BufferWriter, WriterError}, }; use crate::{ + Bufferable, buffer_usage_data::BufferUsageHandle, topology::{ builder::IntoBuffer, channel::{ReceiverAdapter, SenderAdapter}, }, - Bufferable, }; /// Error that occurred when creating/loading a disk buffer. diff --git a/lib/vector-buffers/src/variants/disk_v2/reader.rs b/lib/vector-buffers/src/variants/disk_v2/reader.rs index 8aea89645c..76fd01055c 100644 --- a/lib/vector-buffers/src/variants/disk_v2/reader.rs +++ b/lib/vector-buffers/src/variants/disk_v2/reader.rs @@ -8,23 +8,23 @@ use std::{ }; use crc32fast::Hasher; -use rkyv::{archived_root, AlignedVec}; +use rkyv::{AlignedVec, archived_root}; use snafu::{ResultExt, Snafu}; use tokio::io::{AsyncBufReadExt, AsyncRead, BufReader}; use vector_common::{finalization::BatchNotifier, finalizer::OrderedFinalizer}; use super::{ + Filesystem, common::create_crc32c_hasher, ledger::Ledger, - record::{validate_record_archive, ArchivedRecord, Record, RecordStatus}, - Filesystem, + record::{ArchivedRecord, Record, RecordStatus, validate_record_archive}, }; use crate::{ + Bufferable, encoding::{AsMetadata, Encodable}, internal_events::BufferReadError, topology::acks::{EligibleMarker, EligibleMarkerLength, MarkerError, OrderedAcknowledgements}, variants::disk_v2::{io::AsyncFile, record::try_as_record_archive}, - Bufferable, }; pub(super) struct ReadToken { @@ -1035,7 +1035,7 @@ where return Err(e); } - }; + } // Fundamentally, when `try_read_record` returns `None`, there's three possible // scenarios: diff --git a/lib/vector-buffers/src/variants/disk_v2/record.rs b/lib/vector-buffers/src/variants/disk_v2/record.rs index 3cf55458c1..986beb807f 100644 --- a/lib/vector-buffers/src/variants/disk_v2/record.rs +++ b/lib/vector-buffers/src/variants/disk_v2/record.rs @@ -3,14 +3,14 @@ use std::{mem, ptr::addr_of}; use bytecheck::{CheckBytes, ErrorBox, StructCheckError}; use crc32fast::Hasher; use rkyv::{ + Archive, Archived, Serialize, boxed::ArchivedBox, with::{CopyOptimize, RefAsBox}, - Archive, Archived, Serialize, }; use super::{ common::align16, - ser::{try_as_archive, DeserializeError}, + ser::{DeserializeError, try_as_archive}, }; pub const RECORD_HEADER_LEN: usize = align16(mem::size_of::>() + 8); @@ -86,31 +86,33 @@ where value: *const Self, context: &mut C, ) -> Result<&'b Self, Self::Error> { - Archived::::check_bytes(addr_of!((*value).checksum), context).map_err(|e| { - StructCheckError { - field_name: "checksum", - inner: ErrorBox::new(e), - } - })?; - Archived::::check_bytes(addr_of!((*value).id), context).map_err(|e| { - StructCheckError { - field_name: "id", - inner: ErrorBox::new(e), - } - })?; - Archived::::check_bytes(addr_of!((*value).metadata), context).map_err(|e| { - StructCheckError { - field_name: "schema_metadata", - inner: ErrorBox::new(e), - } - })?; - ArchivedBox::<[u8]>::check_bytes(addr_of!((*value).payload), context).map_err(|e| { - StructCheckError { - field_name: "payload", - inner: ErrorBox::new(e), - } - })?; - Ok(&*value) + unsafe { + Archived::::check_bytes(addr_of!((*value).checksum), context).map_err(|e| { + StructCheckError { + field_name: "checksum", + inner: ErrorBox::new(e), + } + })?; + Archived::::check_bytes(addr_of!((*value).id), context).map_err(|e| { + StructCheckError { + field_name: "id", + inner: ErrorBox::new(e), + } + })?; + Archived::::check_bytes(addr_of!((*value).metadata), context).map_err(|e| { + StructCheckError { + field_name: "schema_metadata", + inner: ErrorBox::new(e), + } + })?; + ArchivedBox::<[u8]>::check_bytes(addr_of!((*value).payload), context).map_err(|e| { + StructCheckError { + field_name: "payload", + inner: ErrorBox::new(e), + } + })?; + Ok(&*value) + } } } diff --git a/lib/vector-buffers/src/variants/disk_v2/ser.rs b/lib/vector-buffers/src/variants/disk_v2/ser.rs index b751c2cfa4..9f7fa08769 100644 --- a/lib/vector-buffers/src/variants/disk_v2/ser.rs +++ b/lib/vector-buffers/src/variants/disk_v2/ser.rs @@ -2,9 +2,8 @@ use std::fmt; use bytecheck::CheckBytes; use rkyv::{ - check_archived_root, - validation::{validators::DefaultValidator, CheckArchiveError}, - Archive, + Archive, check_archived_root, + validation::{CheckArchiveError, validators::DefaultValidator}, }; /// Error that occurred during serialization. diff --git a/lib/vector-buffers/src/variants/disk_v2/tests/acknowledgements.rs b/lib/vector-buffers/src/variants/disk_v2/tests/acknowledgements.rs index 437bf1c58a..a63ed70635 100644 --- a/lib/vector-buffers/src/variants/disk_v2/tests/acknowledgements.rs +++ b/lib/vector-buffers/src/variants/disk_v2/tests/acknowledgements.rs @@ -6,7 +6,7 @@ use vector_common::finalization::{BatchNotifier, EventFinalizer, EventStatus}; use crate::{ buffer_usage_data::BufferUsageHandle, test::with_temp_dir, - variants::disk_v2::{ledger::Ledger, DiskBufferConfigBuilder}, + variants::disk_v2::{DiskBufferConfigBuilder, ledger::Ledger}, }; pub(crate) async fn acknowledge(batch: BatchNotifier) { diff --git a/lib/vector-buffers/src/variants/disk_v2/tests/basic.rs b/lib/vector-buffers/src/variants/disk_v2/tests/basic.rs index 781e30ad5b..30cb47d3c2 100644 --- a/lib/vector-buffers/src/variants/disk_v2/tests/basic.rs +++ b/lib/vector-buffers/src/variants/disk_v2/tests/basic.rs @@ -1,16 +1,16 @@ -use std::io::Cursor; +use std::{io::Cursor, time::Duration}; -use futures::{stream, StreamExt}; -use tokio_test::{assert_pending, assert_ready, task::spawn}; +use futures::{StreamExt, stream}; +use tokio::{select, time::sleep}; +use tokio_test::{assert_pending, task::spawn}; use tracing::Instrument; use vector_common::finalization::Finalizable; use super::{create_default_buffer_v2, read_next, read_next_some}; use crate::{ - assert_buffer_is_empty, assert_buffer_records, - test::{acknowledge, install_tracing_helpers, with_temp_dir, MultiEventRecord, SizedRecord}, + EventCount, assert_buffer_is_empty, assert_buffer_records, + test::{MultiEventRecord, SizedRecord, acknowledge, install_tracing_helpers, with_temp_dir}, variants::disk_v2::{tests::create_default_buffer_v2_with_usage, writer::RecordWriter}, - EventCount, }; #[tokio::test] @@ -66,6 +66,7 @@ async fn basic_read_write_loop() { .await; } +#[ignore = "flaky. See https://github.com/vectordotdev/vector/issues/23456"] #[tokio::test] async fn reader_exits_cleanly_when_writer_done_and_in_flight_acks() { let assertion_registry = install_tracing_helpers(); @@ -127,7 +128,15 @@ async fn reader_exits_cleanly_when_writer_done_and_in_flight_acks() { // albeit with a return value of `None`... because the writer is closed, and we read all // the records, so nothing is left. :) assert!(blocked_read.is_woken()); - let second_read = assert_ready!(blocked_read.poll()); + + let second_read = select! { + // if the reader task finishes in time, extract its output + res = blocked_read => res, + // otherwise panics after 1s + () = sleep(Duration::from_secs(1)) => { + panic!("Reader not ready after 1s"); + } + }; assert_eq!(second_read.expect("read should not fail"), None); // All records should be consumed at this point. diff --git a/lib/vector-buffers/src/variants/disk_v2/tests/initialization.rs b/lib/vector-buffers/src/variants/disk_v2/tests/initialization.rs index 217057d3e0..d0a51f1f05 100644 --- a/lib/vector-buffers/src/variants/disk_v2/tests/initialization.rs +++ b/lib/vector-buffers/src/variants/disk_v2/tests/initialization.rs @@ -4,7 +4,7 @@ use tokio::time::timeout; use tracing::Instrument; use crate::{ - test::{acknowledge, install_tracing_helpers, with_temp_dir, SizedRecord}, + test::{SizedRecord, acknowledge, install_tracing_helpers, with_temp_dir}, variants::disk_v2::tests::{create_default_buffer_v2, set_file_length}, }; diff --git a/lib/vector-buffers/src/variants/disk_v2/tests/invariants.rs b/lib/vector-buffers/src/variants/disk_v2/tests/invariants.rs index 29adb49f17..a37075fce4 100644 --- a/lib/vector-buffers/src/variants/disk_v2/tests/invariants.rs +++ b/lib/vector-buffers/src/variants/disk_v2/tests/invariants.rs @@ -4,10 +4,10 @@ use tracing::Instrument; use super::{create_buffer_v2_with_max_data_file_size, read_next, read_next_some}; use crate::{ - assert_buffer_is_empty, assert_buffer_records, assert_buffer_size, assert_enough_bytes_written, - assert_reader_last_writer_next_positions, assert_reader_writer_v2_file_positions, - await_timeout, set_data_file_length, - test::{acknowledge, install_tracing_helpers, with_temp_dir, MultiEventRecord, SizedRecord}, + EventCount, assert_buffer_is_empty, assert_buffer_records, assert_buffer_size, + assert_enough_bytes_written, assert_reader_last_writer_next_positions, + assert_reader_writer_v2_file_positions, await_timeout, set_data_file_length, + test::{MultiEventRecord, SizedRecord, acknowledge, install_tracing_helpers, with_temp_dir}, variants::disk_v2::{ common::{DEFAULT_FLUSH_INTERVAL, MAX_FILE_ID}, tests::{ @@ -15,7 +15,6 @@ use crate::{ get_corrected_max_record_size, get_minimum_data_file_size_for_record_payload, }, }, - EventCount, }; #[tokio::test] diff --git a/lib/vector-buffers/src/variants/disk_v2/tests/known_errors.rs b/lib/vector-buffers/src/variants/disk_v2/tests/known_errors.rs index f613a14373..ba858428f1 100644 --- a/lib/vector-buffers/src/variants/disk_v2/tests/known_errors.rs +++ b/lib/vector-buffers/src/variants/disk_v2/tests/known_errors.rs @@ -15,12 +15,11 @@ use vector_common::finalization::{AddBatchNotifier, BatchNotifier}; use super::{create_buffer_v2_with_max_data_file_size, create_default_buffer_v2}; use crate::{ - assert_buffer_size, assert_enough_bytes_written, assert_file_does_not_exist_async, + EventCount, assert_buffer_size, assert_enough_bytes_written, assert_file_does_not_exist_async, assert_file_exists_async, assert_reader_writer_v2_file_positions, await_timeout, encoding::{AsMetadata, Encodable}, - test::{acknowledge, install_tracing_helpers, with_temp_dir, SizedRecord, UndecodableRecord}, - variants::disk_v2::{backed_archive::BackedArchive, record::Record, ReaderError}, - EventCount, + test::{SizedRecord, UndecodableRecord, acknowledge, install_tracing_helpers, with_temp_dir}, + variants::disk_v2::{ReaderError, backed_archive::BackedArchive, record::Record}, }; impl AsMetadata for u32 { @@ -29,11 +28,7 @@ impl AsMetadata for u32 { } fn from_u32(value: u32) -> Option { - if value < 32 { - Some(value) - } else { - None - } + if value < 32 { Some(value) } else { None } } } diff --git a/lib/vector-buffers/src/variants/disk_v2/tests/mod.rs b/lib/vector-buffers/src/variants/disk_v2/tests/mod.rs index 2319fbf97d..fa8ac8b11a 100644 --- a/lib/vector-buffers/src/variants/disk_v2/tests/mod.rs +++ b/lib/vector-buffers/src/variants/disk_v2/tests/mod.rs @@ -10,14 +10,14 @@ use tokio::{ }; use super::{ + Buffer, BufferReader, BufferWriter, DiskBufferConfigBuilder, Filesystem, Ledger, io::{AsyncFile, Metadata, ProductionFilesystem, ReadableMemoryMap, WritableMemoryMap}, ledger::LEDGER_LEN, record::RECORD_HEADER_LEN, - Buffer, BufferReader, BufferWriter, DiskBufferConfigBuilder, Filesystem, Ledger, }; use crate::{ - buffer_usage_data::BufferUsageHandle, encoding::FixedEncodable, - variants::disk_v2::common::align16, Bufferable, + Bufferable, buffer_usage_data::BufferUsageHandle, encoding::FixedEncodable, + variants::disk_v2::common::align16, }; type FilesystemUnderTest = ProductionFilesystem; diff --git a/lib/vector-buffers/src/variants/disk_v2/tests/model/action.rs b/lib/vector-buffers/src/variants/disk_v2/tests/model/action.rs index a5e3898959..36c5ea21d6 100644 --- a/lib/vector-buffers/src/variants/disk_v2/tests/model/action.rs +++ b/lib/vector-buffers/src/variants/disk_v2/tests/model/action.rs @@ -1,6 +1,6 @@ use proptest::{ arbitrary::any, - collection::{vec as arb_vec, SizeRange}, + collection::{SizeRange, vec as arb_vec}, prop_compose, prop_oneof, strategy::Just, strategy::Strategy, diff --git a/lib/vector-buffers/src/variants/disk_v2/tests/model/filesystem.rs b/lib/vector-buffers/src/variants/disk_v2/tests/model/filesystem.rs index f6d5d11d5d..a88204d7f7 100644 --- a/lib/vector-buffers/src/variants/disk_v2/tests/model/filesystem.rs +++ b/lib/vector-buffers/src/variants/disk_v2/tests/model/filesystem.rs @@ -11,8 +11,8 @@ use std::{ use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; use crate::variants::disk_v2::{ - io::{AsyncFile, Metadata, ReadableMemoryMap, WritableMemoryMap}, Filesystem, + io::{AsyncFile, Metadata, ReadableMemoryMap, WritableMemoryMap}, }; fn io_err_already_exists() -> io::Error { diff --git a/lib/vector-buffers/src/variants/disk_v2/tests/model/mod.rs b/lib/vector-buffers/src/variants/disk_v2/tests/model/mod.rs index d723af0fa6..ebab8bd145 100644 --- a/lib/vector-buffers/src/variants/disk_v2/tests/model/mod.rs +++ b/lib/vector-buffers/src/variants/disk_v2/tests/model/mod.rs @@ -2,8 +2,8 @@ use std::{ collections::{HashMap, VecDeque}, io::Cursor, sync::{ - atomic::{AtomicBool, AtomicU16, AtomicU64, Ordering}, Arc, Mutex, + atomic::{AtomicBool, AtomicU16, AtomicU64, Ordering}, }, task::Poll, }; @@ -14,24 +14,24 @@ use temp_dir::TempDir; use tokio::runtime::Builder; use crate::{ + EventCount, buffer_usage_data::BufferUsageHandle, encoding::FixedEncodable, test::install_tracing_helpers, variants::disk_v2::{ - common::MAX_FILE_ID, record::RECORD_HEADER_LEN, writer::RecordWriter, Buffer, - DiskBufferConfig, WriterError, + Buffer, DiskBufferConfig, WriterError, common::MAX_FILE_ID, record::RECORD_HEADER_LEN, + writer::RecordWriter, }, - EventCount, }; mod action; use self::{ - action::{arb_actions, Action}, + action::{Action, arb_actions}, record::EncodeError, }; mod common; -use self::common::{arb_buffer_config, Progress}; +use self::common::{Progress, arb_buffer_config}; mod filesystem; use self::filesystem::TestFilesystem; diff --git a/lib/vector-buffers/src/variants/disk_v2/tests/model/record.rs b/lib/vector-buffers/src/variants/disk_v2/tests/model/record.rs index 3b13e35ca4..9ad9e5ebda 100644 --- a/lib/vector-buffers/src/variants/disk_v2/tests/model/record.rs +++ b/lib/vector-buffers/src/variants/disk_v2/tests/model/record.rs @@ -7,9 +7,9 @@ use vector_common::finalization::{ }; use crate::{ + EventCount, encoding::FixedEncodable, variants::disk_v2::{record::RECORD_HEADER_LEN, tests::align16}, - EventCount, }; #[derive(Debug)] diff --git a/lib/vector-buffers/src/variants/disk_v2/tests/model/sequencer.rs b/lib/vector-buffers/src/variants/disk_v2/tests/model/sequencer.rs index 270cf7c84e..0e77c78b66 100644 --- a/lib/vector-buffers/src/variants/disk_v2/tests/model/sequencer.rs +++ b/lib/vector-buffers/src/variants/disk_v2/tests/model/sequencer.rs @@ -1,7 +1,7 @@ use std::{collections::VecDeque, io, mem, task::Poll}; -use futures::{future::BoxFuture, Future, FutureExt}; -use tokio_test::task::{spawn, Spawn}; +use futures::{Future, FutureExt, future::BoxFuture}; +use tokio_test::task::{Spawn, spawn}; use super::{ action::Action, diff --git a/lib/vector-buffers/src/variants/disk_v2/tests/size_limits.rs b/lib/vector-buffers/src/variants/disk_v2/tests/size_limits.rs index 1902fb4c4c..4db7011d38 100644 --- a/lib/vector-buffers/src/variants/disk_v2/tests/size_limits.rs +++ b/lib/vector-buffers/src/variants/disk_v2/tests/size_limits.rs @@ -11,7 +11,7 @@ use super::{ use crate::{ assert_buffer_is_empty, assert_buffer_records, assert_buffer_size, assert_enough_bytes_written, assert_reader_writer_v2_file_positions, - test::{acknowledge, install_tracing_helpers, with_temp_dir, SizedRecord}, + test::{SizedRecord, acknowledge, install_tracing_helpers, with_temp_dir}, variants::disk_v2::{ common::align16, tests::{get_corrected_max_record_size, get_minimum_data_file_size_for_record_payload}, @@ -66,7 +66,7 @@ async fn writer_error_when_record_is_over_the_limit() { } #[tokio::test] -#[ignore] +#[ignore = "Needs investigation"] async fn writer_waits_when_buffer_is_full() { let assertion_registry = install_tracing_helpers(); let fut = with_temp_dir(|dir| { diff --git a/lib/vector-buffers/src/variants/disk_v2/writer.rs b/lib/vector-buffers/src/variants/disk_v2/writer.rs index fc8f60672e..f12ce6ca94 100644 --- a/lib/vector-buffers/src/variants/disk_v2/writer.rs +++ b/lib/vector-buffers/src/variants/disk_v2/writer.rs @@ -11,32 +11,32 @@ use std::{ use bytes::BufMut; use crc32fast::Hasher; use rkyv::{ + AlignedVec, Infallible, ser::{ + Serializer, serializers::{ AlignedSerializer, AllocScratch, AllocScratchError, BufferScratch, CompositeSerializer, CompositeSerializerError, FallbackScratch, }, - Serializer, }, - AlignedVec, Infallible, }; use snafu::{ResultExt, Snafu}; use tokio::io::{AsyncWrite, AsyncWriteExt}; use super::{ - common::{create_crc32c_hasher, DiskBufferConfig}, + common::{DiskBufferConfig, create_crc32c_hasher}, io::Filesystem, ledger::Ledger, - record::{validate_record_archive, Record, RecordStatus}, + record::{Record, RecordStatus, validate_record_archive}, }; use crate::{ + Bufferable, encoding::{AsMetadata, Encodable}, variants::disk_v2::{ io::AsyncFile, reader::decode_record_payload, - record::{try_as_record_archive, RECORD_HEADER_LEN}, + record::{RECORD_HEADER_LEN, try_as_record_archive}, }, - Bufferable, }; /// Error that occurred during calls to [`BufferWriter`]. @@ -648,7 +648,7 @@ where /// `InconsistentState`, as being unable to immediately deserialize and decode a record we just serialized and /// encoded implies a fatal, and unrecoverable, error with the buffer implementation as a whole. #[instrument(skip(self), level = "trace")] - pub fn recover_archived_record(&mut self, token: WriteToken) -> Result> { + pub fn recover_archived_record(&mut self, token: &WriteToken) -> Result> { // Make sure the write token we've been given matches whatever the last call to `archive_record` generated. let serialized_len = token.serialized_len(); debug_assert_eq!( @@ -912,8 +912,11 @@ where // likely missed flushing some records, or partially flushed the data file. // Better roll over to be safe. error!( - ledger_next, last_record_id, record_events, - "Last record written to data file is behind expected position. Events have likely been lost."); + ledger_next, + last_record_id, + record_events, + "Last record written to data file is behind expected position. Events have likely been lost." + ); true } Ordering::Less => { @@ -1161,12 +1164,7 @@ where /// If an error occurred while writing the record, an error variant will be returned describing /// the error. pub async fn try_write_record(&mut self, record: T) -> Result, WriterError> { - self.try_write_record_inner(record) - .await - .map(|result| match result { - Ok(_) => None, - Err(record) => Some(record), - }) + self.try_write_record_inner(record).await.map(Result::err) } #[instrument(skip_all, level = "debug")] @@ -1219,8 +1217,6 @@ where last_attempted_write_size = serialized_len, "Current data file reached maximum size. Rolling to the next data file." ); - - continue; } e => return Err(e), }, @@ -1247,7 +1243,7 @@ where // writer and hand it back. This looks a little weird because we want to surface deserialize/decoding // errors if we encounter them, but if we recover the record successfully, we're returning // `Ok(Err(record))` to signal that our attempt failed but the record is able to be retried again later. - return Ok(Err(writer.recover_archived_record(token)?)); + return Ok(Err(writer.recover_archived_record(&token)?)); }; // Track our write since things appear to have succeeded. This only updates our internal @@ -1294,7 +1290,6 @@ where Err(old_record) => { record = old_record; self.ledger.wait_for_reader().await; - continue; } } } diff --git a/lib/vector-buffers/src/variants/in_memory.rs b/lib/vector-buffers/src/variants/in_memory.rs index fedcf35c2d..1c9fd541f2 100644 --- a/lib/vector-buffers/src/variants/in_memory.rs +++ b/lib/vector-buffers/src/variants/in_memory.rs @@ -3,22 +3,29 @@ use std::{error::Error, num::NonZeroUsize}; use async_trait::async_trait; use crate::{ + Bufferable, buffer_usage_data::BufferUsageHandle, + config::MemoryBufferSize, topology::{ builder::IntoBuffer, - channel::{limited, ReceiverAdapter, SenderAdapter}, + channel::{ReceiverAdapter, SenderAdapter, limited}, }, - Bufferable, }; pub struct MemoryBuffer { - capacity: NonZeroUsize, + capacity: MemoryBufferSize, } impl MemoryBuffer { - pub fn new(capacity: NonZeroUsize) -> Self { + pub fn new(capacity: MemoryBufferSize) -> Self { MemoryBuffer { capacity } } + + pub fn with_max_events(n: NonZeroUsize) -> Self { + Self { + capacity: MemoryBufferSize::MaxEvents(n), + } + } } #[async_trait] @@ -30,9 +37,14 @@ where self: Box, usage_handle: BufferUsageHandle, ) -> Result<(SenderAdapter, ReceiverAdapter), Box> { - usage_handle.set_buffer_limits(None, Some(self.capacity.get())); + let (max_bytes, max_size) = match self.capacity { + MemoryBufferSize::MaxEvents(max_events) => (None, Some(max_events.get())), + MemoryBufferSize::MaxSize(max_size) => (None, Some(max_size.get())), + }; + + usage_handle.set_buffer_limits(max_bytes, max_size); - let (tx, rx) = limited(self.capacity.get()); + let (tx, rx) = limited(self.capacity); Ok((tx.into(), rx.into())) } } diff --git a/lib/vector-common/Cargo.toml b/lib/vector-common/Cargo.toml index 124d7d27b6..98f1818000 100644 --- a/lib/vector-common/Cargo.toml +++ b/lib/vector-common/Cargo.toml @@ -2,7 +2,7 @@ name = "vector-common" version = "0.1.0" authors = ["Vector Contributors "] -edition = "2021" +edition = "2024" publish = false license = "MPL-2.0" @@ -35,24 +35,24 @@ tokenize = [] [dependencies] async-stream = "0.3.6" -bytes = { version = "1.9.0", default-features = false, optional = true } +bytes = { version = "1.10.1", default-features = false, optional = true } chrono.workspace = true -crossbeam-utils = { version = "0.8.21", default-features = false } -derivative = { version = "2.2.0", default-features = false } +crossbeam-utils.workspace = true +derivative.workspace = true futures.workspace = true indexmap.workspace = true metrics.workspace = true -paste = "1.0.15" +paste.workspace = true pin-project.workspace = true serde.workspace = true serde_json.workspace = true smallvec = { version = "1", default-features = false } stream-cancel = { version = "0.8.2", default-features = false } -tokio = { version = "1.43.0", default-features = false, features = ["macros", "time"] } -tracing = { version = "0.1.34", default-features = false } +tokio = { workspace = true, features = ["macros", "time"] } +tracing.workspace = true vrl.workspace = true vector-config = { path = "../vector-config" } [dev-dependencies] futures = { version = "0.3.31", default-features = false, features = ["async-await"] } -tokio = { version = "1.43.0", default-features = false, features = ["rt", "time"] } +tokio = { workspace = true, features = ["rt", "time"] } diff --git a/lib/vector-common/src/byte_size_of.rs b/lib/vector-common/src/byte_size_of.rs index 5520bd0dca..aed57b8ff5 100644 --- a/lib/vector-common/src/byte_size_of.rs +++ b/lib/vector-common/src/byte_size_of.rs @@ -5,7 +5,7 @@ use std::{ use bytes::{Bytes, BytesMut}; use chrono::{DateTime, Utc}; -use serde_json::{value::RawValue, Value as JsonValue}; +use serde_json::{Value as JsonValue, value::RawValue}; use smallvec::SmallVec; use vrl::value::{KeyString, Value}; diff --git a/lib/vector-common/src/config.rs b/lib/vector-common/src/config.rs index 621915e322..41036cf1d9 100644 --- a/lib/vector-common/src/config.rs +++ b/lib/vector-common/src/config.rs @@ -3,7 +3,7 @@ use std::{ fmt, }; -use vector_config::{configurable_component, ConfigurableString}; +use vector_config::{ConfigurableString, configurable_component}; /// Component identifier. #[configurable_component(no_deser, no_ser)] @@ -39,6 +39,12 @@ impl ComponentKey { } } +impl AsRef for ComponentKey { + fn as_ref(&self) -> &ComponentKey { + self + } +} + impl From for ComponentKey { fn from(id: String) -> Self { Self { id } diff --git a/lib/vector-common/src/constants.rs b/lib/vector-common/src/constants.rs index e463a19461..1eeadca45b 100644 --- a/lib/vector-common/src/constants.rs +++ b/lib/vector-common/src/constants.rs @@ -1,2 +1,3 @@ pub const GZIP_MAGIC: &[u8] = &[0x1f, 0x8b]; pub const ZLIB_MAGIC: &[u8] = &[0x78]; +pub const ZSTD_MAGIC: &[u8] = &[0x28, 0xB5, 0x2F, 0xFD]; diff --git a/lib/vector-common/src/finalization.rs b/lib/vector-common/src/finalization.rs index 48001a2bf8..56310a37d3 100644 --- a/lib/vector-common/src/finalization.rs +++ b/lib/vector-common/src/finalization.rs @@ -462,7 +462,7 @@ mod tests { assert_eq!(receiver2.try_recv(), Ok(BatchStatus::Delivered)); } - #[ignore] // The current implementation does not deduplicate finalizers + #[ignore = "The current implementation does not deduplicate finalizers"] #[test] fn clone_and_merge_events() { let (mut fin1, mut receiver) = make_finalizer(); diff --git a/lib/vector-common/src/finalizer.rs b/lib/vector-common/src/finalizer.rs index a3f63f2290..66346c95c6 100644 --- a/lib/vector-common/src/finalizer.rs +++ b/lib/vector-common/src/finalizer.rs @@ -4,9 +4,9 @@ use std::marker::{PhantomData, Unpin}; use std::{fmt::Debug, future::Future, pin::Pin, sync::Arc, task::Context, task::Poll}; use futures::stream::{BoxStream, FuturesOrdered, FuturesUnordered}; -use futures::{future::OptionFuture, FutureExt, Stream, StreamExt}; -use tokio::sync::mpsc::{self, UnboundedReceiver, UnboundedSender}; +use futures::{FutureExt, Stream, StreamExt, future::OptionFuture}; use tokio::sync::Notify; +use tokio::sync::mpsc::{self, UnboundedReceiver, UnboundedSender}; use crate::finalization::{BatchStatus, BatchStatusReceiver}; use crate::shutdown::ShutdownSignal; @@ -86,10 +86,10 @@ where } pub fn add(&self, entry: T, receiver: BatchStatusReceiver) { - if let Some(sender) = &self.sender { - if let Err(error) = sender.send((receiver, entry)) { - error!(message = "FinalizerSet task ended prematurely.", %error); - } + if let Some(sender) = &self.sender + && let Err(error) = sender.send((receiver, entry)) + { + error!(message = "FinalizerSet task ended prematurely.", %error); } } diff --git a/lib/vector-common/src/internal_event/bytes_received.rs b/lib/vector-common/src/internal_event/bytes_received.rs index 8ed0ab9144..98044ecb84 100644 --- a/lib/vector-common/src/internal_event/bytes_received.rs +++ b/lib/vector-common/src/internal_event/bytes_received.rs @@ -1,4 +1,4 @@ -use metrics::{counter, Counter}; +use metrics::{Counter, counter}; use super::{ByteSize, Protocol, SharedString}; diff --git a/lib/vector-common/src/internal_event/bytes_sent.rs b/lib/vector-common/src/internal_event/bytes_sent.rs index cb8a25b2ca..0b2a88247f 100644 --- a/lib/vector-common/src/internal_event/bytes_sent.rs +++ b/lib/vector-common/src/internal_event/bytes_sent.rs @@ -1,4 +1,4 @@ -use metrics::{counter, Counter}; +use metrics::{Counter, counter}; use tracing::trace; use super::{ByteSize, Protocol, SharedString}; diff --git a/lib/vector-common/src/internal_event/cached_event.rs b/lib/vector-common/src/internal_event/cached_event.rs index b985757b0f..782faeb5a1 100644 --- a/lib/vector-common/src/internal_event/cached_event.rs +++ b/lib/vector-common/src/internal_event/cached_event.rs @@ -91,7 +91,7 @@ where #[cfg(test)] mod tests { #![allow(unreachable_pub)] - use metrics::{counter, Counter}; + use metrics::{Counter, counter}; use super::*; diff --git a/lib/vector-common/src/internal_event/component_events_dropped.rs b/lib/vector-common/src/internal_event/component_events_dropped.rs index c7d2bd9eed..94fdeca4e4 100644 --- a/lib/vector-common/src/internal_event/component_events_dropped.rs +++ b/lib/vector-common/src/internal_event/component_events_dropped.rs @@ -1,5 +1,5 @@ use super::{Count, InternalEvent, InternalEventHandle, RegisterInternalEvent}; -use metrics::{counter, Counter}; +use metrics::{Counter, counter}; pub const INTENTIONAL: bool = true; pub const UNINTENTIONAL: bool = false; diff --git a/lib/vector-common/src/internal_event/events_received.rs b/lib/vector-common/src/internal_event/events_received.rs index 2e0c94c6d2..40ba999a8c 100644 --- a/lib/vector-common/src/internal_event/events_received.rs +++ b/lib/vector-common/src/internal_event/events_received.rs @@ -1,4 +1,4 @@ -use metrics::{counter, histogram, Counter, Histogram}; +use metrics::{Counter, Histogram, counter, histogram}; use tracing::trace; use super::CountByteSize; diff --git a/lib/vector-common/src/internal_event/events_sent.rs b/lib/vector-common/src/internal_event/events_sent.rs index c6ae6e45fe..acc865982a 100644 --- a/lib/vector-common/src/internal_event/events_sent.rs +++ b/lib/vector-common/src/internal_event/events_sent.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use metrics::{counter, Counter}; +use metrics::{Counter, counter}; use tracing::trace; use crate::config::ComponentKey; diff --git a/lib/vector-common/src/internal_event/mod.rs b/lib/vector-common/src/internal_event/mod.rs index ac1115fb41..85b82cfa4b 100644 --- a/lib/vector-common/src/internal_event/mod.rs +++ b/lib/vector-common/src/internal_event/mod.rs @@ -17,8 +17,8 @@ pub use bytes_sent::BytesSent; #[allow(clippy::module_name_repetitions)] pub use cached_event::{RegisterTaggedInternalEvent, RegisteredEventCache}; pub use component_events_dropped::{ComponentEventsDropped, INTENTIONAL, UNINTENTIONAL}; -pub use events_received::EventsReceived; -pub use events_sent::{EventsSent, TaggedEventsSent, DEFAULT_OUTPUT}; +pub use events_received::{EventsReceived, EventsReceivedHandle}; +pub use events_sent::{DEFAULT_OUTPUT, EventsSent, TaggedEventsSent}; pub use optional_tag::OptionalTag; pub use prelude::{error_stage, error_type}; pub use service::{CallError, PollReadyError}; diff --git a/lib/vector-common/src/internal_event/prelude.rs b/lib/vector-common/src/internal_event/prelude.rs index e15a6b52b4..92aa160fe5 100644 --- a/lib/vector-common/src/internal_event/prelude.rs +++ b/lib/vector-common/src/internal_event/prelude.rs @@ -1,5 +1,6 @@ // Set of `stage` tags to use when emitting error events. pub mod error_stage { + pub const INITIALIZING: &str = "initializing"; pub const RECEIVING: &str = "receiving"; pub const PROCESSING: &str = "processing"; pub const SENDING: &str = "sending"; diff --git a/lib/vector-common/src/internal_event/service.rs b/lib/vector-common/src/internal_event/service.rs index 20132cc99f..e7d9748e45 100644 --- a/lib/vector-common/src/internal_event/service.rs +++ b/lib/vector-common/src/internal_event/service.rs @@ -1,6 +1,6 @@ use metrics::counter; -use super::{emit, error_stage, error_type, ComponentEventsDropped, InternalEvent, UNINTENTIONAL}; +use super::{ComponentEventsDropped, InternalEvent, UNINTENTIONAL, emit, error_stage, error_type}; #[derive(Debug)] pub struct PollReadyError { diff --git a/lib/vector-common/src/request_metadata.rs b/lib/vector-common/src/request_metadata.rs index 12a164d23e..bd91632a92 100644 --- a/lib/vector-common/src/request_metadata.rs +++ b/lib/vector-common/src/request_metadata.rs @@ -172,7 +172,7 @@ impl AddAssign for GroupedCountByteSize { // For these cases, we know we won't have to change `self` so the match can take ownership. match (self, rhs) { - (Self::Tagged { sizes: ref mut lhs }, Self::Tagged { sizes: rhs }) => { + (&mut Self::Tagged { sizes: ref mut lhs }, Self::Tagged { sizes: rhs }) => { for (key, value) in rhs { match lhs.get_mut(&key) { Some(size) => *size += value, @@ -187,7 +187,7 @@ impl AddAssign for GroupedCountByteSize { *lhs = *lhs + rhs; } - (Self::Tagged { ref mut sizes }, Self::Untagged { size }) => { + (Self::Tagged { sizes }, Self::Untagged { size }) => { match sizes.get_mut(&TaggedEventsSent::new_empty()) { Some(empty_size) => *empty_size += size, None => { @@ -196,7 +196,7 @@ impl AddAssign for GroupedCountByteSize { } } (Self::Untagged { .. }, Self::Tagged { .. }) => unreachable!(), - }; + } } } diff --git a/lib/vector-common/src/sensitive_string.rs b/lib/vector-common/src/sensitive_string.rs index 0b290d1327..0db373361e 100644 --- a/lib/vector-common/src/sensitive_string.rs +++ b/lib/vector-common/src/sensitive_string.rs @@ -1,4 +1,4 @@ -use vector_config::{configurable_component, ConfigurableString}; +use vector_config::{ConfigurableString, configurable_component}; /// Wrapper for sensitive strings containing credentials #[configurable_component(no_deser, no_ser)] diff --git a/lib/vector-common/src/shutdown.rs b/lib/vector-common/src/shutdown.rs index d1b35be5c6..425b33762e 100644 --- a/lib/vector-common/src/shutdown.rs +++ b/lib/vector-common/src/shutdown.rs @@ -5,12 +5,12 @@ use std::{ future::Future, pin::Pin, sync::Arc, - task::{ready, Context, Poll}, + task::{Context, Poll, ready}, }; -use futures::{future, FutureExt}; +use futures::{FutureExt, future}; use stream_cancel::{Trigger, Tripwire}; -use tokio::time::{timeout_at, Instant}; +use tokio::time::{Instant, timeout_at}; use crate::{config::ComponentKey, trigger::DisabledTrigger}; @@ -111,9 +111,9 @@ type IsInternal = bool; #[derive(Debug, Default)] pub struct SourceShutdownCoordinator { - shutdown_begun_triggers: HashMap, - shutdown_force_triggers: HashMap, - shutdown_complete_tripwires: HashMap, + begun_triggers: HashMap, + force_triggers: HashMap, + complete_tripwires: HashMap, } impl SourceShutdownCoordinator { @@ -124,16 +124,16 @@ impl SourceShutdownCoordinator { &mut self, id: &ComponentKey, internal: bool, - ) -> (ShutdownSignal, impl Future) { + ) -> (ShutdownSignal, impl Future + use<>) { let (shutdown_begun_trigger, shutdown_begun_tripwire) = Tripwire::new(); let (force_shutdown_trigger, force_shutdown_tripwire) = Tripwire::new(); let (shutdown_complete_trigger, shutdown_complete_tripwire) = Tripwire::new(); - self.shutdown_begun_triggers + self.begun_triggers .insert(id.clone(), (internal, shutdown_begun_trigger)); - self.shutdown_force_triggers + self.force_triggers .insert(id.clone(), force_shutdown_trigger); - self.shutdown_complete_tripwires + self.complete_tripwires .insert(id.clone(), shutdown_complete_tripwire); let shutdown_signal = @@ -151,9 +151,9 @@ impl SourceShutdownCoordinator { /// /// Panics if the other coordinator already had its triggers removed. pub fn takeover_source(&mut self, id: &ComponentKey, other: &mut Self) { - let existing = self.shutdown_begun_triggers.insert( + let existing = self.begun_triggers.insert( id.clone(), - other.shutdown_begun_triggers.remove(id).unwrap_or_else(|| { + other.begun_triggers.remove(id).unwrap_or_else(|| { panic!( "Other ShutdownCoordinator didn't have a shutdown_begun_trigger for \"{id}\"" ) @@ -164,9 +164,9 @@ impl SourceShutdownCoordinator { "ShutdownCoordinator already has a shutdown_begin_trigger for source \"{id}\"" ); - let existing = self.shutdown_force_triggers.insert( + let existing = self.force_triggers.insert( id.clone(), - other.shutdown_force_triggers.remove(id).unwrap_or_else(|| { + other.force_triggers.remove(id).unwrap_or_else(|| { panic!( "Other ShutdownCoordinator didn't have a shutdown_force_trigger for \"{id}\"" ) @@ -177,10 +177,10 @@ impl SourceShutdownCoordinator { "ShutdownCoordinator already has a shutdown_force_trigger for source \"{id}\"" ); - let existing = self.shutdown_complete_tripwires.insert( + let existing = self.complete_tripwires.insert( id.clone(), other - .shutdown_complete_tripwires + .complete_tripwires .remove(id) .unwrap_or_else(|| { panic!( @@ -207,9 +207,9 @@ impl SourceShutdownCoordinator { let mut internal_sources_complete_futures = Vec::new(); let mut external_sources_complete_futures = Vec::new(); - let shutdown_begun_triggers = self.shutdown_begun_triggers; - let mut shutdown_complete_tripwires = self.shutdown_complete_tripwires; - let mut shutdown_force_triggers = self.shutdown_force_triggers; + let shutdown_begun_triggers = self.begun_triggers; + let mut shutdown_complete_tripwires = self.complete_tripwires; + let mut shutdown_force_triggers = self.force_triggers; for (id, (internal, trigger)) in shutdown_begun_triggers { trigger.cancel(); @@ -259,25 +259,24 @@ impl SourceShutdownCoordinator { &mut self, id: &ComponentKey, deadline: Instant, - ) -> impl Future { - let (_, begin_shutdown_trigger) = - self.shutdown_begun_triggers.remove(id).unwrap_or_else(|| { - panic!( + ) -> impl Future + use<> { + let (_, begin_shutdown_trigger) = self.begun_triggers.remove(id).unwrap_or_else(|| { + panic!( "shutdown_begun_trigger for source \"{id}\" not found in the ShutdownCoordinator" ) - }); + }); // This is what actually triggers the source to begin shutting down. begin_shutdown_trigger.cancel(); let shutdown_complete_tripwire = self - .shutdown_complete_tripwires + .complete_tripwires .remove(id) .unwrap_or_else(|| { panic!( "shutdown_complete_tripwire for source \"{id}\" not found in the ShutdownCoordinator" ) }); - let shutdown_force_trigger = self.shutdown_force_triggers.remove(id).unwrap_or_else(|| { + let shutdown_force_trigger = self.force_triggers.remove(id).unwrap_or_else(|| { panic!( "shutdown_force_trigger for source \"{id}\" not found in the ShutdownCoordinator" ) @@ -294,7 +293,7 @@ impl SourceShutdownCoordinator { #[must_use] pub fn shutdown_tripwire(&self) -> future::BoxFuture<'static, ()> { let futures = self - .shutdown_complete_tripwires + .complete_tripwires .values() .cloned() .map(|tripwire| tripwire.then(tripwire_handler).boxed()); diff --git a/lib/vector-config-common/Cargo.toml b/lib/vector-config-common/Cargo.toml index 4813d3885d..350c1cb167 100644 --- a/lib/vector-config-common/Cargo.toml +++ b/lib/vector-config-common/Cargo.toml @@ -1,15 +1,15 @@ [package] name = "vector-config-common" version = "0.1.0" -edition = "2021" +edition = "2024" license = "MPL-2.0" [dependencies] -convert_case = { version = "0.6", default-features = false } -darling = { version = "0.20", default-features = false, features = ["suggestions"] } +convert_case = { version = "0.8", default-features = false } +darling.workspace = true proc-macro2 = { version = "1.0", default-features = false } serde.workspace = true serde_json.workspace = true syn = { version = "2.0", features = ["full", "extra-traits", "visit-mut", "visit"] } -tracing = { version = "0.1.34", default-features = false } +tracing.workspace = true quote = { version = "1.0", default-features = false } diff --git a/lib/vector-config-common/src/constants.rs b/lib/vector-config-common/src/constants.rs index 864b72098a..80284e7c4a 100644 --- a/lib/vector-config-common/src/constants.rs +++ b/lib/vector-config-common/src/constants.rs @@ -1,12 +1,14 @@ use serde_json::Value; use syn::Path; +pub const COMPONENT_TYPE_API: &str = "api"; pub const COMPONENT_TYPE_ENRICHMENT_TABLE: &str = "enrichment_table"; pub const COMPONENT_TYPE_PROVIDER: &str = "provider"; pub const COMPONENT_TYPE_SECRETS: &str = "secrets"; pub const COMPONENT_TYPE_SINK: &str = "sink"; pub const COMPONENT_TYPE_SOURCE: &str = "source"; pub const COMPONENT_TYPE_TRANSFORM: &str = "transform"; +pub const COMPONENT_TYPE_GLOBAL_OPTION: &str = "global_option"; pub const DOCS_META_ADDITIONAL_PROPS_DESC: &str = "docs::additional_props_description"; pub const DOCS_META_ADVANCED: &str = "docs::advanced"; pub const DOCS_META_COMPONENT_BASE_TYPE: &str = "docs::component_base_type"; @@ -21,6 +23,8 @@ pub const DOCS_META_HIDDEN: &str = "docs::hidden"; pub const DOCS_META_HUMAN_NAME: &str = "docs::human_name"; pub const DOCS_META_NUMERIC_TYPE: &str = "docs::numeric_type"; pub const DOCS_META_OPTIONAL: &str = "docs::optional"; +pub const DOCS_META_COMMON: &str = "docs::common"; +pub const DOCS_META_REQUIRED: &str = "docs::required"; pub const DOCS_META_SYNTAX_OVERRIDE: &str = "docs::syntax_override"; pub const DOCS_META_TEMPLATEABLE: &str = "docs::templateable"; pub const DOCS_META_TYPE_OVERRIDE: &str = "docs::type_override"; @@ -31,7 +35,9 @@ pub const METADATA: &str = "_metadata"; /// Well-known component types. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum ComponentType { + Api, EnrichmentTable, + GlobalOption, Provider, Secrets, Sink, @@ -43,7 +49,9 @@ impl ComponentType { /// Gets the type of this component as a string. pub const fn as_str(&self) -> &'static str { match self { + ComponentType::Api => COMPONENT_TYPE_API, ComponentType::EnrichmentTable => COMPONENT_TYPE_ENRICHMENT_TABLE, + ComponentType::GlobalOption => COMPONENT_TYPE_GLOBAL_OPTION, ComponentType::Provider => COMPONENT_TYPE_PROVIDER, ComponentType::Secrets => COMPONENT_TYPE_SECRETS, ComponentType::Sink => COMPONENT_TYPE_SINK, @@ -62,7 +70,9 @@ impl<'a> TryFrom<&'a str> for ComponentType { fn try_from(value: &'a str) -> Result { match value { + COMPONENT_TYPE_API => Ok(ComponentType::Api), COMPONENT_TYPE_ENRICHMENT_TABLE => Ok(ComponentType::EnrichmentTable), + COMPONENT_TYPE_GLOBAL_OPTION => Ok(ComponentType::GlobalOption), COMPONENT_TYPE_PROVIDER => Ok(ComponentType::Provider), COMPONENT_TYPE_SECRETS => Ok(ComponentType::Secrets), COMPONENT_TYPE_SINK => Ok(ComponentType::Sink), diff --git a/lib/vector-config-common/src/human_friendly.rs b/lib/vector-config-common/src/human_friendly.rs index a341168cca..70894d9a4b 100644 --- a/lib/vector-config-common/src/human_friendly.rs +++ b/lib/vector-config-common/src/human_friendly.rs @@ -70,7 +70,7 @@ pub fn generate_human_friendly_string(input: &str) -> String { // respectively. let converter = Converter::new() .to_case(Case::Title) - .remove_boundaries(&[Boundary::LowerDigit, Boundary::UpperDigit]); + .remove_boundaries(&[Boundary::LOWER_DIGIT, Boundary::UPPER_DIGIT]); let normalized = converter.convert(input); let replaced_segments = normalized diff --git a/lib/vector-config-common/src/schema/gen.rs b/lib/vector-config-common/src/schema/generator.rs similarity index 97% rename from lib/vector-config-common/src/schema/gen.rs rename to lib/vector-config-common/src/schema/generator.rs index b70a75aad9..0b71abc2e0 100644 --- a/lib/vector-config-common/src/schema/gen.rs +++ b/lib/vector-config-common/src/schema/generator.rs @@ -1,4 +1,4 @@ -use super::{visit::Visitor, Map, RootSchema, Schema, SchemaObject, DEFINITIONS_PREFIX}; +use super::{DEFINITIONS_PREFIX, Map, RootSchema, Schema, SchemaObject, visit::Visitor}; /// Settings to customize how schemas are generated. #[derive(Debug)] @@ -113,7 +113,7 @@ impl SchemaGenerator { pub fn dereference<'a>(&'a self, schema: &Schema) -> Option<&'a Schema> { match schema { Schema::Object(SchemaObject { - reference: Some(ref schema_ref), + reference: Some(schema_ref), .. }) => { let definitions_path = &self.settings().definitions_path; diff --git a/lib/vector-config-common/src/schema/json_schema.rs b/lib/vector-config-common/src/schema/json_schema.rs index e737f34c76..32ba142a8a 100644 --- a/lib/vector-config-common/src/schema/json_schema.rs +++ b/lib/vector-config-common/src/schema/json_schema.rs @@ -2,7 +2,7 @@ use serde::{Deserialize, Serialize}; use serde_json::Value; use std::{iter, ops::Deref}; -use super::{Map, Set, DEFINITIONS_PREFIX}; +use super::{DEFINITIONS_PREFIX, Map, Set}; /// A JSON Schema. #[allow(clippy::large_enum_variant)] @@ -249,9 +249,7 @@ impl SchemaObject { /// and does not check any subschemas. Because of this, both `{}` and `{"not": {}}` accept any type according /// to this method. pub fn has_type(&self, ty: InstanceType) -> bool { - self.instance_type - .as_ref() - .map_or(true, |x| x.contains(&ty)) + self.instance_type.as_ref().is_none_or(|x| x.contains(&ty)) } get_or_insert_default_fn!(metadata, Metadata); @@ -617,8 +615,7 @@ pub fn get_cleaned_schema_reference(schema_ref: &str) -> &str { cleaned } else { panic!( - "Tried to clean schema reference that does not start with the definition prefix: {}", - schema_ref + "Tried to clean schema reference that does not start with the definition prefix: {schema_ref}" ); } } diff --git a/lib/vector-config-common/src/schema/mod.rs b/lib/vector-config-common/src/schema/mod.rs index 763cdd217e..b944eede84 100644 --- a/lib/vector-config-common/src/schema/mod.rs +++ b/lib/vector-config-common/src/schema/mod.rs @@ -2,7 +2,7 @@ // copied from the `schemars` crate. The license for `schemars` is included in `LICENSE-schemars`, // pursuant to the listed conditions in the license. -mod gen; +mod generator; mod json_schema; pub mod visit; @@ -13,5 +13,5 @@ pub(crate) const DEFINITIONS_PREFIX: &str = "#/definitions/"; pub type Map = std::collections::BTreeMap; pub type Set = std::collections::BTreeSet; -pub use self::gen::{SchemaGenerator, SchemaSettings}; +pub use self::generator::{SchemaGenerator, SchemaSettings}; pub use self::json_schema::*; diff --git a/lib/vector-config-common/src/schema/visit.rs b/lib/vector-config-common/src/schema/visit.rs index 242bed3433..a17c7981c8 100644 --- a/lib/vector-config-common/src/schema/visit.rs +++ b/lib/vector-config-common/src/schema/visit.rs @@ -1,6 +1,6 @@ use tracing::debug; -use super::{get_cleaned_schema_reference, Map, RootSchema, Schema, SchemaObject, SingleOrVec}; +use super::{Map, RootSchema, Schema, SchemaObject, SingleOrVec, get_cleaned_schema_reference}; /// Trait used to recursively modify a constructed schema and its subschemas. pub trait Visitor: std::fmt::Debug { diff --git a/lib/vector-config-common/src/validation.rs b/lib/vector-config-common/src/validation.rs index 25e1679700..6389f66d70 100644 --- a/lib/vector-config-common/src/validation.rs +++ b/lib/vector-config-common/src/validation.rs @@ -4,7 +4,7 @@ use darling::FromMeta; use proc_macro2::TokenStream; -use quote::{quote, ToTokens}; +use quote::{ToTokens, quote}; use syn::{Expr, Lit, Meta}; use crate::{ @@ -187,9 +187,9 @@ pub enum Validation { /// /// Can only be used for numbers. Range { - #[darling(default, rename = "min", with = "maybe_float_or_int")] + #[darling(default, rename = "min", with = maybe_float_or_int)] minimum: Option, - #[darling(default, rename = "max", with = "maybe_float_or_int")] + #[darling(default, rename = "max", with = maybe_float_or_int)] maximum: Option, }, @@ -221,20 +221,20 @@ impl Validation { let min_bound = NUMERIC_ENFORCED_LOWER_BOUND; let max_bound = NUMERIC_ENFORCED_UPPER_BOUND; - if let Some(minimum) = *minimum { - if minimum < min_bound { - return Err(darling::Error::custom( - "number ranges cannot exceed 2^53 (absolute) for either the minimum or maximum", - )); - } + if let Some(minimum) = *minimum + && minimum < min_bound + { + return Err(darling::Error::custom( + "number ranges cannot exceed 2^53 (absolute) for either the minimum or maximum", + )); } - if let Some(maximum) = *maximum { - if maximum < max_bound { - return Err(darling::Error::custom( - "number ranges cannot exceed 2^53 (absolute) for either the minimum or maximum", - )); - } + if let Some(maximum) = *maximum + && maximum < max_bound + { + return Err(darling::Error::custom( + "number ranges cannot exceed 2^53 (absolute) for either the minimum or maximum", + )); } if *minimum > *maximum { @@ -249,7 +249,7 @@ impl Validation { (Some(min), Some(max)) if min > max => { return Err(darling::Error::custom( "minimum cannot be greater than maximum", - )) + )); } _ => {} } diff --git a/lib/vector-config-macros/Cargo.toml b/lib/vector-config-macros/Cargo.toml index 8e35ce6ecd..386dbca03f 100644 --- a/lib/vector-config-macros/Cargo.toml +++ b/lib/vector-config-macros/Cargo.toml @@ -1,14 +1,14 @@ [package] name = "vector-config-macros" version = "0.1.0" -edition = "2021" +edition = "2024" license = "MPL-2.0" [lib] proc-macro = true [dependencies] -darling = { version = "0.20", default-features = false, features = ["suggestions"] } +darling.workspace = true proc-macro2 = { version = "1.0", default-features = false } quote = { version = "1.0", default-features = false } serde_derive_internals = "0.29" diff --git a/lib/vector-config-macros/src/ast/container.rs b/lib/vector-config-macros/src/ast/container.rs index dad5fbd903..be38e739e9 100644 --- a/lib/vector-config-macros/src/ast/container.rs +++ b/lib/vector-config-macros/src/ast/container.rs @@ -4,19 +4,19 @@ use std::collections::HashSet; -use darling::{error::Accumulator, util::Flag, FromAttributes}; -use serde_derive_internals::{ast as serde_ast, Ctxt, Derive}; +use darling::{FromAttributes, error::Accumulator, util::Flag}; +use serde_derive_internals::{Ctxt, Derive, ast as serde_ast}; use syn::{ DeriveInput, ExprPath, GenericArgument, Generics, Ident, PathArguments, PathSegment, Type, TypeParam, }; use super::{ + Data, Field, LazyCustomAttribute, Metadata, Style, Tagging, Variant, util::{ - err_serde_failed, get_serde_default_value, try_extract_doc_title_description, - DarlingResultIterator, + DarlingResultIterator, err_serde_failed, get_serde_default_value, + try_extract_doc_title_description, }, - Data, Field, LazyCustomAttribute, Metadata, Style, Tagging, Variant, }; const ERR_NO_ENUM_TUPLES: &str = "enum variants cannot be tuples (multiple unnamed fields)"; @@ -175,12 +175,20 @@ impl<'a> Container<'a> { // This allows untagged enums used for "(de)serialize as A, B, or C" // purposes to avoid needless titles/descriptions when their fields will // implicitly provide that. - if variant.description().is_none() && tagging != Tagging::None { + if variant.description().is_none() + && tagging != Tagging::None + && variant.tagging() != &Tagging::None + { accumulator.push( darling::Error::custom(ERR_NO_ENUM_VARIANT_DESCRIPTION) .with_span(variant), ); } + + // Serde allows multiple untagged variants in a tagged enum. We do not + // restrict their count or order here; ambiguity handling is done via + // schema generation strategy (e.g., falling back to `anyOf`) and + // discriminant hints when needed. } // If we're in untagged mode, there can be no duplicate variants. @@ -272,7 +280,7 @@ impl<'a> Container<'a> { /// Data for the container. /// /// This would be the fields of a struct, or the variants for an enum. - pub fn data(&self) -> &Data { + pub fn data(&self) -> &Data<'_> { &self.data } @@ -517,11 +525,7 @@ fn get_generic_args_from_path_segment( }) .collect::>(); - if args.is_empty() { - None - } else { - Some(args) - } + if args.is_empty() { None } else { Some(args) } } // We don't support parenthesized generic arguments as they only come up in the case of // function pointers, and we don't support those with `Configurable`. @@ -544,7 +548,7 @@ fn get_generic_type_path_ident(ty: &Type) -> Option { mod tests { use proc_macro2::Ident; use quote::format_ident; - use syn::{parse_quote, Type}; + use syn::{Type, parse_quote}; use super::get_generic_type_param_idents; diff --git a/lib/vector-config-macros/src/ast/field.rs b/lib/vector-config-macros/src/ast/field.rs index 5945b35ee6..c321be7ca1 100644 --- a/lib/vector-config-macros/src/ast/field.rs +++ b/lib/vector-config-macros/src/ast/field.rs @@ -1,19 +1,19 @@ use darling::{ - util::{Flag, Override, SpannedValue}, FromAttributes, + util::{Flag, Override, SpannedValue}, }; use proc_macro2::{Span, TokenStream}; use quote::ToTokens; use serde_derive_internals::ast as serde_ast; -use syn::{parse_quote, ExprPath, Ident}; +use syn::{ExprPath, Ident, parse_quote}; use vector_config_common::validation::Validation; use super::{ + LazyCustomAttribute, Metadata, util::{ err_field_implicit_transparent, err_field_missing_description, find_delegated_serde_deser_ty, get_serde_default_value, try_extract_doc_title_description, }, - LazyCustomAttribute, Metadata, }; /// A field of a container. diff --git a/lib/vector-config-macros/src/ast/mod.rs b/lib/vector-config-macros/src/ast/mod.rs index b307827d27..581671791b 100644 --- a/lib/vector-config-macros/src/ast/mod.rs +++ b/lib/vector-config-macros/src/ast/mod.rs @@ -1,4 +1,4 @@ -use darling::{ast::NestedMeta, error::Accumulator, util::path_to_string, FromMeta}; +use darling::{FromMeta, ast::NestedMeta, error::Accumulator, util::path_to_string}; use quote::ToTokens; use serde_derive_internals::{ast as serde_ast, attr as serde_attr}; @@ -170,7 +170,7 @@ pub struct Metadata { } impl Metadata { - pub fn attributes(&self) -> impl Iterator { + pub fn attributes(&self) -> impl Iterator + use<> { self.items.clone().into_iter() } } diff --git a/lib/vector-config-macros/src/ast/util.rs b/lib/vector-config-macros/src/ast/util.rs index ddd36c5739..cb351dccf4 100644 --- a/lib/vector-config-macros/src/ast/util.rs +++ b/lib/vector-config-macros/src/ast/util.rs @@ -1,9 +1,9 @@ use darling::{ast::NestedMeta, error::Accumulator}; -use quote::{quote, ToTokens}; -use serde_derive_internals::{attr as serde_attr, Ctxt}; +use quote::{ToTokens, quote}; +use serde_derive_internals::{Ctxt, attr as serde_attr}; use syn::{ - punctuated::Punctuated, spanned::Spanned, token::Comma, Attribute, Expr, ExprLit, ExprPath, - Lit, Meta, MetaNameValue, + Attribute, Expr, ExprLit, ExprPath, Lit, Meta, MetaNameValue, punctuated::Punctuated, + spanned::Spanned, token::Comma, }; const ERR_FIELD_MISSING_DESCRIPTION: &str = "field must have a description -- i.e. `/// This is a widget...` or `#[configurable(description = \"...\")] -- or derive it from the underlying type of the field by specifying `#[configurable(derived)]`"; @@ -138,11 +138,7 @@ fn group_doc_lines(ungrouped: &[String]) -> Vec { } fn none_if_empty(s: String) -> Option { - if s.is_empty() { - None - } else { - Some(s) - } + if s.is_empty() { None } else { Some(s) } } pub fn err_field_missing_description(field: &T) -> darling::Error { @@ -218,14 +214,16 @@ fn find_name_value_attribute( // Only take attributes whose name matches `attr_name`. .filter(|attr| path_matches(attr.path(), attr_name)) // Derive macro helper attributes will always be in the list form. - .filter_map(|attr| match &attr.meta { + .flat_map(|attr| match &attr.meta { Meta::List(ml) => ml .parse_args_with(Punctuated::::parse_terminated) .map(|nested| nested.into_iter()) - .ok(), - _ => None, + // If parsing fails, return an empty iterator. By this point, `serde` has already + // emitted its own error, so we don't want to duplicate any error emission here. + .unwrap_or_else(|_| Punctuated::::new().into_iter()), + // Non-list attributes cannot contain nested meta items; return empty iterator. + _ => Punctuated::::new().into_iter(), }) - .flatten() // For each nested meta item in the list, find any that are name/value pairs where the // name matches `name_key`, and return their value. .find_map(|nm| match nm { @@ -240,6 +238,29 @@ fn find_name_value_attribute( }) } +/// Checks whether an attribute list contains a flag-style entry. +/// +/// For example, this returns true when `attributes` contains something like `#[serde(untagged)]` +/// when called with `attr_name = "serde"` and `flag_name = "untagged"`. +pub(crate) fn has_flag_attribute( + attributes: &[syn::Attribute], + attr_name: &str, + flag_name: &str, +) -> bool { + attributes + .iter() + .filter(|attr| path_matches(attr.path(), attr_name)) + .filter_map(|attr| match &attr.meta { + Meta::List(ml) => ml + .parse_args_with(Punctuated::::parse_terminated) + .map(|nested| nested.into_iter()) + .ok(), + _ => None, + }) + .flatten() + .any(|nm| matches!(nm, NestedMeta::Meta(Meta::Path(ref path)) if path_matches(path, flag_name))) +} + /// Tries to find a delegated (de)serialization type from attributes. /// /// In some cases, the `serde_with` crate, more specifically the `serde_as` attribute macro, may be diff --git a/lib/vector-config-macros/src/ast/variant.rs b/lib/vector-config-macros/src/ast/variant.rs index 87a8b53177..f579ade744 100644 --- a/lib/vector-config-macros/src/ast/variant.rs +++ b/lib/vector-config-macros/src/ast/variant.rs @@ -1,11 +1,11 @@ -use darling::{error::Accumulator, util::Flag, FromAttributes}; +use darling::{FromAttributes, error::Accumulator, util::Flag}; use proc_macro2::{Ident, TokenStream}; use quote::ToTokens; use serde_derive_internals::ast as serde_ast; use super::{ - util::{try_extract_doc_title_description, DarlingResultIterator}, Field, LazyCustomAttribute, Metadata, Style, Tagging, + util::{DarlingResultIterator, has_flag_attribute, try_extract_doc_title_description}, }; /// A variant in an enum. @@ -40,6 +40,15 @@ impl<'a> Variant<'a> { .map(|field| Field::from_ast(field, is_virtual_newtype, is_newtype_wrapper_field)) .collect_darling_results(&mut accumulator); + // If the enum overall is tagged (internal/adjacent) serde still allows one or more + // variants to be explicitly marked with `#[serde(untagged)]`. In that case, the + // variant itself is untagged; reflect that here so schema generation matches serde. + let tagging = if has_flag_attribute(&original.attrs, "serde", "untagged") { + Tagging::None + } else { + tagging + }; + let variant = Variant { original, name, diff --git a/lib/vector-config-macros/src/attrs.rs b/lib/vector-config-macros/src/attrs.rs index fa081693ea..52973b9596 100644 --- a/lib/vector-config-macros/src/attrs.rs +++ b/lib/vector-config-macros/src/attrs.rs @@ -12,7 +12,9 @@ impl AttributeIdent { pub const NO_SER: AttributeIdent = AttributeIdent("no_ser"); pub const NO_DESER: AttributeIdent = AttributeIdent("no_deser"); +pub const API_COMPONENT: AttributeIdent = AttributeIdent("api_component"); pub const ENRICHMENT_TABLE_COMPONENT: AttributeIdent = AttributeIdent("enrichment_table_component"); +pub const GLOBAL_OPTION_COMPONENT: AttributeIdent = AttributeIdent("global_option_component"); pub const PROVIDER_COMPONENT: AttributeIdent = AttributeIdent("provider_component"); pub const SECRETS_COMPONENT: AttributeIdent = AttributeIdent("secrets_component"); pub const SINK_COMPONENT: AttributeIdent = AttributeIdent("sink_component"); diff --git a/lib/vector-config-macros/src/component_name.rs b/lib/vector-config-macros/src/component_name.rs index 8c7e5efd2e..7b41b17d7c 100644 --- a/lib/vector-config-macros/src/component_name.rs +++ b/lib/vector-config-macros/src/component_name.rs @@ -2,7 +2,7 @@ use darling::util::path_to_string; use proc_macro::TokenStream; use quote::quote; -use syn::{parse_macro_input, spanned::Spanned, Attribute, DeriveInput, Error, LitStr}; +use syn::{Attribute, DeriveInput, Error, LitStr, parse_macro_input, spanned::Spanned}; use crate::attrs::{self, path_matches}; @@ -108,7 +108,9 @@ fn attr_to_component_name(attr: &Attribute) -> Result, Error> { if !path_matches( attr.path(), &[ + attrs::API_COMPONENT, attrs::ENRICHMENT_TABLE_COMPONENT, + attrs::GLOBAL_OPTION_COMPONENT, attrs::PROVIDER_COMPONENT, attrs::SINK_COMPONENT, attrs::SOURCE_COMPONENT, @@ -132,8 +134,7 @@ fn attr_to_component_name(attr: &Attribute) -> Result, Error> { return Err(Error::new( attr.span(), format!( - "{}s must have a name specified (e.g. `{}(\"my_component\")`)", - component_type, component_type_attr + "{component_type}s must have a name specified (e.g. `{component_type_attr}(\"my_component\")`)" ), )); } @@ -145,8 +146,7 @@ fn attr_to_component_name(attr: &Attribute) -> Result, Error> { Error::new( attr.span(), format!( - "expected a string literal for the {} name (i.e. `{}(\"...\")`)", - component_type, component_type_attr + "expected a string literal for the {component_type} name (i.e. `{component_type_attr}(\"...\")`)" ), ) }) @@ -183,6 +183,8 @@ fn check_component_name_validity(component_name: &str) -> Result<(), String> { if component_name == component_name_converted { Ok(()) } else { - Err(format!("component names must be lowercase, and contain only letters, numbers, and underscores (e.g. \"{}\")", component_name_converted)) + Err(format!( + "component names must be lowercase, and contain only letters, numbers, and underscores (e.g. \"{component_name_converted}\")" + )) } } diff --git a/lib/vector-config-macros/src/configurable.rs b/lib/vector-config-macros/src/configurable.rs index fa27431a6c..43d5c9483b 100644 --- a/lib/vector-config-macros/src/configurable.rs +++ b/lib/vector-config-macros/src/configurable.rs @@ -2,8 +2,8 @@ use proc_macro::TokenStream; use proc_macro2::Span; use quote::{quote, quote_spanned}; use syn::{ - parse_macro_input, parse_quote, spanned::Spanned, token::PathSep, DeriveInput, ExprPath, Ident, - PathArguments, Type, + DeriveInput, ExprPath, Ident, PathArguments, Type, parse_macro_input, parse_quote, + spanned::Spanned, token::PathSep, }; use vector_config_common::validation::Validation; @@ -250,8 +250,7 @@ fn generate_named_struct_field( .expect("named struct fields must always have an ident"); let field_schema_ty = get_field_schema_ty(field); let field_already_contained = format!( - "schema properties already contained entry for `{}`, this should not occur", - field_name + "schema properties already contained entry for `{field_name}`, this should not occur" ); let field_key = field.name(); @@ -675,8 +674,7 @@ fn generate_named_enum_field(field: &Field<'_>) -> proc_macro2::TokenStream { let field_name = field.ident().expect("field should be named"); let field_ty = field.ty(); let field_already_contained = format!( - "schema properties already contained entry for `{}`, this should not occur", - field_name + "schema properties already contained entry for `{field_name}`, this should not occur" ); let field_key = field.name().to_string(); @@ -700,15 +698,24 @@ fn generate_named_enum_field(field: &Field<'_>) -> proc_macro2::TokenStream { None }; - quote! { - { - #field_schema - - if let Some(_) = properties.insert(#field_key.to_string(), subschema) { - panic!(#field_already_contained); + if field.flatten() { + quote! { + { + #field_schema + flattened_subschemas.push(subschema); } + } + } else { + quote! { + { + #field_schema - #maybe_field_required + if let Some(_) = properties.insert(#field_key.to_string(), subschema) { + panic!(#field_already_contained); + } + + #maybe_field_required + } } } } @@ -733,6 +740,7 @@ fn generate_enum_struct_named_variant_schema( { let mut properties = ::vector_config::indexmap::IndexMap::new(); let mut required = ::std::collections::BTreeSet::new(); + let mut flattened_subschemas = ::std::vec::Vec::new(); #(#mapped_fields)* @@ -740,11 +748,18 @@ fn generate_enum_struct_named_variant_schema( #maybe_fill_discriminant_map - ::vector_config::schema::generate_struct_schema( + let mut schema = ::vector_config::schema::generate_struct_schema( properties, required, None - ) + ); + + // If we have any flattened subschemas, deal with them now. + if !flattened_subschemas.is_empty() { + ::vector_config::schema::convert_to_flattened_schema(&mut schema, flattened_subschemas); + } + + schema } } } @@ -841,7 +856,9 @@ fn generate_enum_variant_schema( // { "field_using_enum": { "": "VariantName" } } Tagging::Internal { tag } => match variant.style() { Style::Struct => { - let tag_already_contained = format!("enum tag `{}` already contained as a field in variant; tag cannot overlap with any fields in any variant", tag); + let tag_already_contained = format!( + "enum tag `{tag}` already contained as a field in variant; tag cannot overlap with any fields in any variant" + ); // Just generate the tag field directly and pass it along to be included in the // struct schema. diff --git a/lib/vector-config-macros/src/configurable_component.rs b/lib/vector-config-macros/src/configurable_component.rs index e42763f665..fc547694d1 100644 --- a/lib/vector-config-macros/src/configurable_component.rs +++ b/lib/vector-config-macros/src/configurable_component.rs @@ -1,10 +1,10 @@ -use darling::{ast::NestedMeta, Error, FromMeta}; +use darling::{Error, FromMeta, ast::NestedMeta}; use proc_macro::TokenStream; use proc_macro2::{Ident, Span}; use quote::{quote, quote_spanned}; use syn::{ - parse_macro_input, parse_quote, parse_quote_spanned, punctuated::Punctuated, spanned::Spanned, - token::Comma, DeriveInput, Lit, LitStr, Meta, MetaList, Path, + DeriveInput, Lit, LitStr, Meta, MetaList, Path, parse_macro_input, parse_quote, + parse_quote_spanned, punctuated::Punctuated, spanned::Spanned, token::Comma, }; use vector_config_common::{ constants::ComponentType, human_friendly::generate_human_friendly_string, @@ -81,9 +81,15 @@ impl TypedComponent { self.component_name.as_ref().map(|component_name| { let config_ty = &input.ident; let desc_ty: syn::Type = match self.component_type { + ComponentType::Api => { + parse_quote! { ::vector_config::component::ApiDescription } + } ComponentType::EnrichmentTable => { parse_quote! { ::vector_config::component::EnrichmentTableDescription } } + ComponentType::GlobalOption => { + parse_quote! { ::vector_config::component::GlobalOptionDescription } + } ComponentType::Provider => { parse_quote! { ::vector_config::component::ProviderDescription } } @@ -347,7 +353,9 @@ pub fn configurable_component_impl(args: TokenStream, item: TokenStream) -> Toke /// derive for `NamedComponent`. fn get_named_component_helper_ident(component_type: ComponentType) -> Ident { let attr = match component_type { + ComponentType::Api => attrs::API_COMPONENT, ComponentType::EnrichmentTable => attrs::ENRICHMENT_TABLE_COMPONENT, + ComponentType::GlobalOption => attrs::GLOBAL_OPTION_COMPONENT, ComponentType::Provider => attrs::PROVIDER_COMPONENT, ComponentType::Secrets => attrs::SECRETS_COMPONENT, ComponentType::Sink => attrs::SINK_COMPONENT, diff --git a/lib/vector-config-macros/src/lib.rs b/lib/vector-config-macros/src/lib.rs index de4730dec2..cfb10e09c2 100644 --- a/lib/vector-config-macros/src/lib.rs +++ b/lib/vector-config-macros/src/lib.rs @@ -97,7 +97,9 @@ pub fn derive_configurable(input: TokenStream) -> TokenStream { #[proc_macro_derive( NamedComponent, attributes( + api_component, enrichment_table_component, + global_option_component, provider_component, secrets_component, sink_component, diff --git a/lib/vector-config/Cargo.toml b/lib/vector-config/Cargo.toml index 79aebb0eb6..2283df3d13 100644 --- a/lib/vector-config/Cargo.toml +++ b/lib/vector-config/Cargo.toml @@ -2,7 +2,7 @@ name = "vector-config" version = "0.1.0" authors = ["Vector Contributors "] -edition = "2021" +edition = "2024" publish = false license = "MPL-2.0" @@ -16,14 +16,14 @@ chrono-tz.workspace = true encoding_rs = { version = "0.8", default-features = false, features = ["alloc", "serde"] } indexmap.workspace = true inventory = { version = "0.3" } -no-proxy = { version = "0.3.5", default-features = false, features = ["serialize"] } +no-proxy = { version = "0.3.6", default-features = false, features = ["serialize"] } num-traits = { version = "0.2.19", default-features = false } serde.workspace = true serde_json.workspace = true -serde_with = { version = "3.12.0", default-features = false, features = ["std"] } +serde_with = { version = "3.14.0", default-features = false, features = ["std"] } snafu.workspace = true toml.workspace = true -tracing = { version = "0.1.34", default-features = false } +tracing.workspace = true url = { version = "2.5.4", default-features = false, features = ["serde"] } http = { version = "0.2.9", default-features = false } vrl.workspace = true @@ -32,4 +32,4 @@ vector-config-macros = { path = "../vector-config-macros" } [dev-dependencies] assert-json-diff = { version = "2", default-features = false } -serde_with = { version = "3.12.0", default-features = false, features = ["std", "macros"] } +serde_with = { version = "3.14.0", default-features = false, features = ["std", "macros"] } diff --git a/lib/vector-config/src/component/description.rs b/lib/vector-config/src/component/description.rs index 1fd03c6ec7..32a849f21d 100644 --- a/lib/vector-config/src/component/description.rs +++ b/lib/vector-config/src/component/description.rs @@ -6,7 +6,7 @@ use vector_config_common::{attributes::CustomAttribute, constants}; use super::{ComponentMarker, GenerateConfig}; use crate::schema::{SchemaGenerator, SchemaObject}; -use crate::{schema, Configurable, ConfigurableRef, GenerateError, Metadata}; +use crate::{Configurable, ConfigurableRef, GenerateError, Metadata, schema}; #[derive(Debug, Snafu, Clone, PartialEq, Eq)] pub enum ExampleError { @@ -83,12 +83,14 @@ where } /// Generate a schema object covering all the descriptions of this type. - pub fn generate_schemas(gen: &RefCell) -> Result { + pub fn generate_schemas( + generator: &RefCell, + ) -> Result { let mut descriptions: Vec<_> = inventory::iter::.into_iter().collect(); descriptions.sort_unstable_by_key(|desc| desc.component_name); let subschemas: Vec = descriptions .into_iter() - .map(|description| description.generate_schema(gen)) + .map(|description| description.generate_schema(generator)) .collect::>()?; Ok(schema::generate_one_of_schema(&subschemas)) } @@ -96,7 +98,7 @@ where /// Generate a schema object for this description. fn generate_schema( &self, - gen: &RefCell, + generator: &RefCell, ) -> Result { let mut tag_subschema = schema::generate_const_string_schema(self.component_name.to_string()); @@ -110,7 +112,7 @@ where let mut field_metadata = Metadata::default(); field_metadata.set_transparent(); let mut subschema = - schema::get_or_generate_schema(&self.config, gen, Some(field_metadata))?; + schema::get_or_generate_schema(&self.config, generator, Some(field_metadata))?; schema::convert_to_flattened_schema(&mut subschema, flattened_subschemas); diff --git a/lib/vector-config/src/component/marker.rs b/lib/vector-config/src/component/marker.rs index d815b2240b..0c900c05a1 100644 --- a/lib/vector-config/src/component/marker.rs +++ b/lib/vector-config/src/component/marker.rs @@ -1,6 +1,11 @@ +/// An API component. +pub struct ApiComponent; /// An enrichment table component. pub struct EnrichmentTableComponent; +// A global option component. +pub struct GlobalOptionComponent; + /// A provider component. pub struct ProviderComponent; @@ -19,7 +24,9 @@ pub struct TransformComponent; // Marker trait representing a component. pub trait ComponentMarker: sealed::Sealed {} +impl ComponentMarker for ApiComponent {} impl ComponentMarker for EnrichmentTableComponent {} +impl ComponentMarker for GlobalOptionComponent {} impl ComponentMarker for ProviderComponent {} impl ComponentMarker for SecretsComponent {} impl ComponentMarker for SinkComponent {} @@ -29,7 +36,9 @@ impl ComponentMarker for TransformComponent {} mod sealed { pub trait Sealed {} + impl Sealed for super::ApiComponent {} impl Sealed for super::EnrichmentTableComponent {} + impl Sealed for super::GlobalOptionComponent {} impl Sealed for super::ProviderComponent {} impl Sealed for super::SecretsComponent {} impl Sealed for super::SinkComponent {} diff --git a/lib/vector-config/src/component/mod.rs b/lib/vector-config/src/component/mod.rs index 8f56bbd6a9..e5b28f922a 100644 --- a/lib/vector-config/src/component/mod.rs +++ b/lib/vector-config/src/component/mod.rs @@ -5,22 +5,26 @@ mod marker; pub use self::description::{ComponentDescription, ExampleError}; pub use self::generate::GenerateConfig; pub use self::marker::{ - ComponentMarker, EnrichmentTableComponent, ProviderComponent, SecretsComponent, SinkComponent, - SourceComponent, TransformComponent, + ApiComponent, ComponentMarker, EnrichmentTableComponent, GlobalOptionComponent, + ProviderComponent, SecretsComponent, SinkComponent, SourceComponent, TransformComponent, }; // Create some type aliases for the component marker/description types, and collect (register, // essentially) any submissions for each respective component marker. +pub type ApiDescription = ComponentDescription; pub type SourceDescription = ComponentDescription; pub type TransformDescription = ComponentDescription; pub type SecretsDescription = ComponentDescription; pub type SinkDescription = ComponentDescription; pub type EnrichmentTableDescription = ComponentDescription; pub type ProviderDescription = ComponentDescription; +pub type GlobalOptionDescription = ComponentDescription; +inventory::collect!(ApiDescription); inventory::collect!(SourceDescription); inventory::collect!(TransformDescription); inventory::collect!(SecretsDescription); inventory::collect!(SinkDescription); inventory::collect!(EnrichmentTableDescription); inventory::collect!(ProviderDescription); +inventory::collect!(GlobalOptionDescription); diff --git a/lib/vector-config/src/configurable.rs b/lib/vector-config/src/configurable.rs index 9abd506085..91c94ca7b0 100644 --- a/lib/vector-config/src/configurable.rs +++ b/lib/vector-config/src/configurable.rs @@ -5,8 +5,8 @@ use std::cell::RefCell; use serde_json::Value; use crate::{ - schema::{SchemaGenerator, SchemaObject}, GenerateError, Metadata, + schema::{SchemaGenerator, SchemaObject}, }; /// A type that can be represented in a Vector configuration. @@ -73,7 +73,7 @@ pub trait Configurable { /// /// If an error occurs while generating the schema, an error variant will be returned describing /// the issue. - fn generate_schema(gen: &RefCell) -> Result + fn generate_schema(generator: &RefCell) -> Result where Self: Sized; @@ -133,8 +133,8 @@ impl ConfigurableRef { } pub(crate) fn generate_schema( &self, - gen: &RefCell, + generator: &RefCell, ) -> Result { - (self.generate_schema)(gen) + (self.generate_schema)(generator) } } diff --git a/lib/vector-config/src/external/chrono.rs b/lib/vector-config/src/external/chrono.rs index 41866309be..73795143c2 100644 --- a/lib/vector-config/src/external/chrono.rs +++ b/lib/vector-config/src/external/chrono.rs @@ -4,8 +4,8 @@ use chrono::{DateTime, TimeZone}; use serde_json::Value; use crate::{ - schema::{generate_string_schema, SchemaGenerator, SchemaObject}, Configurable, GenerateError, Metadata, ToValue, + schema::{SchemaGenerator, SchemaObject, generate_string_schema}, }; impl Configurable for DateTime diff --git a/lib/vector-config/src/external/datetime.rs b/lib/vector-config/src/external/datetime.rs index 6735cfa2e2..73c9921ba9 100644 --- a/lib/vector-config/src/external/datetime.rs +++ b/lib/vector-config/src/external/datetime.rs @@ -1,9 +1,9 @@ use crate::{ + Configurable, GenerateError, Metadata, ToValue, schema::{ - apply_base_metadata, generate_const_string_schema, generate_one_of_schema, - get_or_generate_schema, SchemaGenerator, SchemaObject, + SchemaGenerator, SchemaObject, apply_base_metadata, generate_const_string_schema, + generate_one_of_schema, get_or_generate_schema, }, - Configurable, GenerateError, Metadata, ToValue, }; use chrono_tz::Tz; use serde_json::Value; @@ -40,7 +40,9 @@ impl Configurable for TimeZone { metadata } - fn generate_schema(gen: &RefCell) -> Result { + fn generate_schema( + generator: &RefCell, + ) -> Result { let mut local_schema = generate_const_string_schema("local".to_string()); let mut local_metadata = Metadata::with_description("System local timezone."); local_metadata.add_custom_attribute(CustomAttribute::kv("logical_name", "Local")); @@ -53,7 +55,8 @@ impl Configurable for TimeZone { [tzdb]: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones"#, ); tz_metadata.add_custom_attribute(CustomAttribute::kv("logical_name", "Named")); - let tz_schema = get_or_generate_schema(&Tz::as_configurable_ref(), gen, Some(tz_metadata))?; + let tz_schema = + get_or_generate_schema(&Tz::as_configurable_ref(), generator, Some(tz_metadata))?; Ok(generate_one_of_schema(&[local_schema, tz_schema])) } diff --git a/lib/vector-config/src/external/encoding_rs.rs b/lib/vector-config/src/external/encoding_rs.rs index a82b26fbb0..068cf12ffa 100644 --- a/lib/vector-config/src/external/encoding_rs.rs +++ b/lib/vector-config/src/external/encoding_rs.rs @@ -4,8 +4,8 @@ use encoding_rs::Encoding; use serde_json::Value; use crate::{ - schema::{generate_string_schema, SchemaGenerator, SchemaObject}, Configurable, GenerateError, Metadata, ToValue, + schema::{SchemaGenerator, SchemaObject, generate_string_schema}, }; impl Configurable for &'static Encoding { diff --git a/lib/vector-config/src/external/indexmap.rs b/lib/vector-config/src/external/indexmap.rs index d90e6ceede..0f49f23e40 100644 --- a/lib/vector-config/src/external/indexmap.rs +++ b/lib/vector-config/src/external/indexmap.rs @@ -4,12 +4,12 @@ use indexmap::{IndexMap, IndexSet}; use serde_json::Value; use crate::{ + Configurable, GenerateError, Metadata, ToValue, schema::{ - assert_string_schema_for_map, generate_map_schema, generate_set_schema, SchemaGenerator, - SchemaObject, + SchemaGenerator, SchemaObject, assert_string_schema_for_map, generate_map_schema, + generate_set_schema, }, str::ConfigurableString, - Configurable, GenerateError, Metadata, ToValue, }; impl Configurable for IndexMap @@ -32,15 +32,17 @@ where V::validate_metadata(&converted) } - fn generate_schema(gen: &RefCell) -> Result { + fn generate_schema( + generator: &RefCell, + ) -> Result { // Make sure our key type is _truly_ a string schema. assert_string_schema_for_map( &K::as_configurable_ref(), - gen, + generator, std::any::type_name::(), )?; - generate_map_schema(&V::as_configurable_ref(), gen) + generate_map_schema(&V::as_configurable_ref(), generator) } } @@ -71,8 +73,10 @@ where V::validate_metadata(&converted) } - fn generate_schema(gen: &RefCell) -> Result { - generate_set_schema(&V::as_configurable_ref(), gen) + fn generate_schema( + generator: &RefCell, + ) -> Result { + generate_set_schema(&V::as_configurable_ref(), generator) } } diff --git a/lib/vector-config/src/external/no_proxy.rs b/lib/vector-config/src/external/no_proxy.rs index e441e3ea60..1b88414e6d 100644 --- a/lib/vector-config/src/external/no_proxy.rs +++ b/lib/vector-config/src/external/no_proxy.rs @@ -3,8 +3,8 @@ use std::cell::RefCell; use serde_json::Value; use crate::{ - schema::{generate_array_schema, SchemaGenerator, SchemaObject}, Configurable, GenerateError, Metadata, ToValue, + schema::{SchemaGenerator, SchemaObject, generate_array_schema}, }; impl Configurable for no_proxy::NoProxy { @@ -14,10 +14,12 @@ impl Configurable for no_proxy::NoProxy { Metadata::with_transparent(true) } - fn generate_schema(gen: &RefCell) -> Result { + fn generate_schema( + generator: &RefCell, + ) -> Result { // `NoProxy` (de)serializes itself as a vector of strings, without any constraints on the string value itself, so // we just... do that. - generate_array_schema(&String::as_configurable_ref(), gen) + generate_array_schema(&String::as_configurable_ref(), generator) } } diff --git a/lib/vector-config/src/external/serde_with.rs b/lib/vector-config/src/external/serde_with.rs index 1ef3a072e6..830733dcc7 100644 --- a/lib/vector-config/src/external/serde_with.rs +++ b/lib/vector-config/src/external/serde_with.rs @@ -4,9 +4,9 @@ use vector_config_common::{attributes::CustomAttribute, constants}; use crate::schema::generate_optional_schema; use crate::{ - num::NumberClass, - schema::{generate_number_schema, SchemaGenerator, SchemaObject}, Configurable, GenerateError, Metadata, + num::NumberClass, + schema::{SchemaGenerator, SchemaObject, generate_number_schema}, }; // Blanket implementation of `Configurable` for any `serde_with` helper that is also `Configurable`. @@ -38,13 +38,15 @@ where T::validate_metadata(&converted) } - fn generate_schema(gen: &RefCell) -> Result { + fn generate_schema( + generator: &RefCell, + ) -> Result { // Forward to the underlying `T`. // // We have to convert from `Metadata` to `Metadata` which erases the default value, // notably, but `serde_with` helpers should never actually have default values, so this is // essentially a no-op. - T::generate_schema(gen) + T::generate_schema(generator) } } @@ -135,10 +137,10 @@ impl Configurable for serde_with::DurationMilliSeconds> { - fn generate_schema(gen: &RefCell) -> Result + fn generate_schema(generator: &RefCell) -> Result where Self: Sized, { - generate_optional_schema(&u64::as_configurable_ref(), gen) + generate_optional_schema(&u64::as_configurable_ref(), generator) } } diff --git a/lib/vector-config/src/external/toml.rs b/lib/vector-config/src/external/toml.rs index 2d8d78971b..ee934e50b0 100644 --- a/lib/vector-config/src/external/toml.rs +++ b/lib/vector-config/src/external/toml.rs @@ -3,8 +3,8 @@ use std::cell::RefCell; use serde_json::Value; use crate::{ - schema::{SchemaGenerator, SchemaObject}, Configurable, GenerateError, ToValue, + schema::{SchemaGenerator, SchemaObject}, }; impl Configurable for toml::Value { diff --git a/lib/vector-config/src/external/tz.rs b/lib/vector-config/src/external/tz.rs index ba938677f2..4d8d0d60dd 100644 --- a/lib/vector-config/src/external/tz.rs +++ b/lib/vector-config/src/external/tz.rs @@ -3,8 +3,8 @@ use std::cell::RefCell; use serde_json::Value; use crate::{ - schema::{generate_string_schema, SchemaGenerator, SchemaObject}, Configurable, GenerateError, Metadata, ToValue, + schema::{SchemaGenerator, SchemaObject, generate_string_schema}, }; impl Configurable for chrono_tz::Tz { diff --git a/lib/vector-config/src/external/url.rs b/lib/vector-config/src/external/url.rs index 1f6f50dc1e..470410c914 100644 --- a/lib/vector-config/src/external/url.rs +++ b/lib/vector-config/src/external/url.rs @@ -4,8 +4,8 @@ use serde_json::Value; use vector_config_common::validation::{Format, Validation}; use crate::{ - schema::{generate_string_schema, SchemaGenerator, SchemaObject}, Configurable, GenerateError, Metadata, ToValue, + schema::{SchemaGenerator, SchemaObject, generate_string_schema}, }; impl Configurable for url::Url { diff --git a/lib/vector-config/src/external/vrl.rs b/lib/vector-config/src/external/vrl.rs index b84b591efc..fad68670a3 100644 --- a/lib/vector-config/src/external/vrl.rs +++ b/lib/vector-config/src/external/vrl.rs @@ -1,11 +1,11 @@ use std::cell::RefCell; use serde_json::Value; -use vrl::{compiler::VrlRuntime, datadog_search_syntax::QueryNode}; +use vrl::{compiler::VrlRuntime, datadog_search_syntax::QueryNode, value::Value as VrlValue}; use crate::{ - schema::{generate_string_schema, SchemaGenerator, SchemaObject}, Configurable, GenerateError, Metadata, ToValue, + schema::{SchemaGenerator, SchemaObject, generate_string_schema}, }; impl Configurable for VrlRuntime { @@ -36,3 +36,33 @@ impl Configurable for QueryNode { Ok(generate_string_schema()) } } + +impl Configurable for VrlValue { + fn is_optional() -> bool { + true + } + + fn metadata() -> Metadata { + Metadata::with_transparent(true) + } + + fn generate_schema(_: &RefCell) -> Result { + // We don't have any constraints on the inputs + Ok(SchemaObject::default()) + } +} + +impl ToValue for VrlValue { + /// Converts a `VrlValue` into a `serde_json::Value`. + /// + /// This conversion should always succeed, though it may result in a loss + /// of type information for some value types. + /// + /// # Panics + /// + /// This function will panic if serialization fails, which is not expected + /// under normal circumstances. + fn to_value(&self) -> Value { + serde_json::to_value(self).expect("Unable to serialize VRL value") + } +} diff --git a/lib/vector-config/src/http.rs b/lib/vector-config/src/http.rs index 17be823be7..23e8568c8d 100644 --- a/lib/vector-config/src/http.rs +++ b/lib/vector-config/src/http.rs @@ -3,8 +3,8 @@ use serde_json::Value; use std::cell::RefCell; use crate::{ - schema::{generate_number_schema, SchemaGenerator, SchemaObject}, Configurable, GenerateError, Metadata, ToValue, + schema::{SchemaGenerator, SchemaObject, generate_number_schema}, }; impl ToValue for StatusCode { diff --git a/lib/vector-config/src/lib.rs b/lib/vector-config/src/lib.rs index 4232fbeac8..ad57a26c8a 100644 --- a/lib/vector-config/src/lib.rs +++ b/lib/vector-config/src/lib.rs @@ -123,7 +123,7 @@ pub use self::metadata::Metadata; mod named; pub use self::named::NamedComponent; mod num; -pub use self::num::ConfigurableNumber; +pub use self::num::{ConfigurableNumber, NumberClass}; pub mod schema; pub mod ser; mod stdlib; @@ -159,26 +159,26 @@ where for validation in metadata.validations() { if let validation::Validation::Range { minimum, maximum } = validation { - if let Some(min_bound) = minimum { - if *min_bound < mechanical_min_bound { - return Err(GenerateError::IncompatibleNumericBounds { - numeric_type: std::any::type_name::(), - bound_direction: BoundDirection::Minimum, - mechanical_bound: mechanical_min_bound, - specified_bound: *min_bound, - }); - } + if let Some(min_bound) = minimum + && *min_bound < mechanical_min_bound + { + return Err(GenerateError::IncompatibleNumericBounds { + numeric_type: std::any::type_name::(), + bound_direction: BoundDirection::Minimum, + mechanical_bound: mechanical_min_bound, + specified_bound: *min_bound, + }); } - if let Some(max_bound) = maximum { - if *max_bound > mechanical_max_bound { - return Err(GenerateError::IncompatibleNumericBounds { - numeric_type: std::any::type_name::(), - bound_direction: BoundDirection::Maximum, - mechanical_bound: mechanical_max_bound, - specified_bound: *max_bound, - }); - } + if let Some(max_bound) = maximum + && *max_bound > mechanical_max_bound + { + return Err(GenerateError::IncompatibleNumericBounds { + numeric_type: std::any::type_name::(), + bound_direction: BoundDirection::Maximum, + mechanical_bound: mechanical_max_bound, + specified_bound: *max_bound, + }); } } } diff --git a/lib/vector-config/src/num.rs b/lib/vector-config/src/num.rs index b58058f459..bcdbf6e672 100644 --- a/lib/vector-config/src/num.rs +++ b/lib/vector-config/src/num.rs @@ -1,5 +1,5 @@ use std::num::{ - NonZeroI16, NonZeroI32, NonZeroI64, NonZeroI8, NonZeroU16, NonZeroU32, NonZeroU64, NonZeroU8, + NonZeroI8, NonZeroI16, NonZeroI32, NonZeroI64, NonZeroU8, NonZeroU16, NonZeroU32, NonZeroU64, NonZeroUsize, }; diff --git a/lib/vector-config/src/schema/helpers.rs b/lib/vector-config/src/schema/helpers.rs index 9900f32998..44409f1338 100644 --- a/lib/vector-config/src/schema/helpers.rs +++ b/lib/vector-config/src/schema/helpers.rs @@ -9,7 +9,7 @@ use serde_json::{Map, Value}; use vector_config_common::{attributes::CustomAttribute, constants, schema::*}; use crate::{ - num::ConfigurableNumber, Configurable, ConfigurableRef, GenerateError, Metadata, ToValue, + Configurable, ConfigurableRef, GenerateError, Metadata, ToValue, num::ConfigurableNumber, }; use super::visitors::{ @@ -56,7 +56,9 @@ fn apply_metadata(config: &ConfigurableRef, schema: &mut SchemaObject, metadata: config.referenceable_name().is_some() && base_metadata.description().is_some(); let is_transparent = base_metadata.transparent() || metadata.transparent(); if schema_description.is_none() && !is_transparent && !has_referenceable_description { - panic!("No description provided for `{type_name}`! All `Configurable` types must define a description, or have one specified at the field-level where the type is being used."); + panic!( + "No description provided for `{type_name}`! All `Configurable` types must define a description, or have one specified at the field-level where the type is being used." + ); } // If a default value was given, serialize it. @@ -101,10 +103,12 @@ fn apply_metadata(config: &ConfigurableRef, schema: &mut SchemaObject, metadata: // Overriding a flag is fine, because flags are only ever "enabled", so there's // no harm to enabling it... again. Likewise, if there was no existing value, // it's fine. - Some(Value::Bool(_)) | None => {}, + Some(Value::Bool(_)) | None => {} // Any other value being present means we're clashing with a different metadata // attribute, which is not good, so we have to bail out. - _ => panic!("Tried to set metadata flag '{key}' but already existed in schema metadata for `{type_name}`."), + _ => panic!( + "Tried to set metadata flag '{key}' but already existed in schema metadata for `{type_name}`." + ), } } CustomAttribute::KeyValue { key, value } => { @@ -224,10 +228,10 @@ where pub(crate) fn generate_array_schema( config: &ConfigurableRef, - gen: &RefCell, + generator: &RefCell, ) -> Result { // Generate the actual schema for the element type. - let element_schema = get_or_generate_schema(config, gen, None)?; + let element_schema = get_or_generate_schema(config, generator, None)?; Ok(SchemaObject { instance_type: Some(InstanceType::Array.into()), @@ -241,10 +245,10 @@ pub(crate) fn generate_array_schema( pub(crate) fn generate_set_schema( config: &ConfigurableRef, - gen: &RefCell, + generator: &RefCell, ) -> Result { // Generate the actual schema for the element type. - let element_schema = get_or_generate_schema(config, gen, None)?; + let element_schema = get_or_generate_schema(config, generator, None)?; Ok(SchemaObject { instance_type: Some(InstanceType::Array.into()), @@ -259,10 +263,10 @@ pub(crate) fn generate_set_schema( pub(crate) fn generate_map_schema( config: &ConfigurableRef, - gen: &RefCell, + generator: &RefCell, ) -> Result { // Generate the actual schema for the element type. - let element_schema = get_or_generate_schema(config, gen, None)?; + let element_schema = get_or_generate_schema(config, generator, None)?; Ok(SchemaObject { instance_type: Some(InstanceType::Object.into()), @@ -297,7 +301,7 @@ pub fn generate_struct_schema( pub(crate) fn generate_optional_schema( config: &ConfigurableRef, - gen: &RefCell, + generator: &RefCell, ) -> Result { // Optional schemas are generally very simple in practice, but because of how we memoize schema // generation and use references to schema definitions, we have to handle quite a few cases @@ -319,7 +323,7 @@ pub(crate) fn generate_optional_schema( // our "null or X" wrapped schema. let mut overrides = Metadata::default(); overrides.add_custom_attribute(CustomAttribute::flag(constants::DOCS_META_OPTIONAL)); - let mut schema = get_or_generate_schema(config, gen, Some(overrides))?; + let mut schema = get_or_generate_schema(config, generator, Some(overrides))?; // Take the metadata and extensions of the original schema. // @@ -505,19 +509,19 @@ where let schema_gen = RefCell::new(schema_settings.into_generator()); // Set env variable to enable generating all schemas, including platform-specific ones. - env::set_var("VECTOR_GENERATE_SCHEMA", "true"); + unsafe { env::set_var("VECTOR_GENERATE_SCHEMA", "true") }; let schema = get_or_generate_schema(&T::as_configurable_ref(), &schema_gen, Some(T::metadata()))?; - env::remove_var("VECTOR_GENERATE_SCHEMA"); + unsafe { env::remove_var("VECTOR_GENERATE_SCHEMA") }; Ok(schema_gen.into_inner().into_root_schema(schema)) } pub fn get_or_generate_schema( config: &ConfigurableRef, - gen: &RefCell, + generator: &RefCell, overrides: Option, ) -> Result { let metadata = config.make_metadata(); @@ -526,13 +530,14 @@ pub fn get_or_generate_schema( // generator's definition list, and if it exists, create a schema reference to // it. Otherwise, generate it and backfill it in the schema generator. Some(name) => { - if !gen.borrow().definitions().contains_key(name) { + if !generator.borrow().definitions().contains_key(name) { // In order to avoid infinite recursion, we copy the approach that `schemars` takes and // insert a dummy boolean schema before actually generating the real schema, and then // replace it afterwards. If any recursion occurs, a schema reference will be handed // back, which means we don't have to worry about the dummy schema needing to be updated // after the fact. - gen.borrow_mut() + generator + .borrow_mut() .definitions_mut() .insert(name.to_string(), Schema::Bool(false)); @@ -546,18 +551,19 @@ pub fn get_or_generate_schema( // should not have a default at all. So, if we applied that override metadata, we'd // be unwittingly applying a default for all usages of the type that didn't override // the default themselves. - let mut schema = config.generate_schema(gen)?; + let mut schema = config.generate_schema(generator)?; apply_metadata(config, &mut schema, metadata); - gen.borrow_mut() + generator + .borrow_mut() .definitions_mut() .insert(name.to_string(), Schema::Object(schema)); } - (get_schema_ref(gen, name), None) + (get_schema_ref(generator, name), None) } // Always generate the schema directly if the type is not referenceable. - None => (config.generate_schema(gen)?, Some(metadata)), + None => (config.generate_schema(generator)?, Some(metadata)), }; // Figure out what metadata we should apply to the resulting schema. @@ -596,10 +602,10 @@ pub fn get_or_generate_schema( Ok(schema) } -fn get_schema_ref>(gen: &RefCell, name: S) -> SchemaObject { +fn get_schema_ref>(generator: &RefCell, name: S) -> SchemaObject { let ref_path = format!( "{}{}", - gen.borrow().settings().definitions_path(), + generator.borrow().settings().definitions_path(), name.as_ref() ); SchemaObject::new_ref(ref_path) @@ -618,10 +624,10 @@ fn get_schema_ref>(gen: &RefCell, name: S) -> Sch /// the issue. pub(crate) fn assert_string_schema_for_map( config: &ConfigurableRef, - gen: &RefCell, + generator: &RefCell, map_type: &'static str, ) -> Result<(), GenerateError> { - let key_schema = get_or_generate_schema(config, gen, None)?; + let key_schema = get_or_generate_schema(config, generator, None)?; let key_type = config.type_name(); // We need to force the schema to be treated as transparent so that when the schema generation @@ -633,9 +639,9 @@ pub(crate) fn assert_string_schema_for_map( // Get a reference to the underlying schema if we're dealing with a reference, or just use what // we have if it's the actual definition. - let gen = gen.borrow(); + let generator = generator.borrow(); let underlying_schema = if wrapped_schema.is_ref() { - gen.dereference(&wrapped_schema) + generator.dereference(&wrapped_schema) } else { Some(&wrapped_schema) }; @@ -670,7 +676,7 @@ pub(crate) fn assert_string_schema_for_map( } } -/// Determines whether or not an enum schema is ambiguous based on discriminants of its variants. +/// Determines whether an enum schema is ambiguous based on discriminants of its variants. /// /// A discriminant is the set of the named fields which are required, which may be an empty set. pub fn has_ambiguous_discriminants( diff --git a/lib/vector-config/src/schema/parser/component.rs b/lib/vector-config/src/schema/parser/component.rs index 37997e023d..2213d11d50 100644 --- a/lib/vector-config/src/schema/parser/component.rs +++ b/lib/vector-config/src/schema/parser/component.rs @@ -61,7 +61,7 @@ impl ComponentSchema<'_> { } impl QueryableSchema for ComponentSchema<'_> { - fn schema_type(&self) -> SchemaType { + fn schema_type(&self) -> SchemaType<'_> { self.schema.schema_type() } @@ -133,7 +133,7 @@ fn get_component_metadata_kv_str<'a>( Value::String(name) => Ok(name), _ => Err(SchemaError::invalid_component_schema( key, - format!("`{}` must be a string", key), + format!("`{key}` must be a string"), )), }) } diff --git a/lib/vector-config/src/schema/parser/query.rs b/lib/vector-config/src/schema/parser/query.rs index 7d317bebe9..45b1fa8cab 100644 --- a/lib/vector-config/src/schema/parser/query.rs +++ b/lib/vector-config/src/schema/parser/query.rs @@ -117,29 +117,25 @@ impl<'a> SchemaQueryBuilder<'a> { let attr_matched = match self_attribute { CustomAttribute::Flag(key) => schema_attributes .get(key) - .map_or(false, |value| matches!(value, Value::Bool(true))), + .is_some_and(|value| matches!(value, Value::Bool(true))), CustomAttribute::KeyValue { key, value: attr_value, } => { - schema_attributes - .get(key) - .map_or(false, |value| match value { - // Check string values directly. - Value::String(schema_attr_value) => { - schema_attr_value == attr_value - } - // For arrays, try and convert each item to a string, and - // for the values that are strings, see if they match. - Value::Array(schema_attr_values) => { - schema_attr_values.iter().any(|value| { - value - .as_str() - .map_or(false, |s| s == attr_value) - }) - } - _ => false, - }) + schema_attributes.get(key).is_some_and(|value| match value { + // Check string values directly. + Value::String(schema_attr_value) => { + schema_attr_value == attr_value + } + // For arrays, try and convert each item to a string, and + // for the values that are strings, see if they match. + Value::Array(schema_attr_values) => { + schema_attr_values.iter().any(|value| { + value.as_str().is_some_and(|s| s == attr_value) + }) + } + _ => false, + }) } }; @@ -228,7 +224,7 @@ pub enum SchemaType<'a> { } pub trait QueryableSchema { - fn schema_type(&self) -> SchemaType; + fn schema_type(&self) -> SchemaType<'_>; fn description(&self) -> Option<&str>; fn title(&self) -> Option<&str>; fn get_attributes(&self, key: &str) -> Option>; @@ -240,7 +236,7 @@ impl QueryableSchema for &T where T: QueryableSchema, { - fn schema_type(&self) -> SchemaType { + fn schema_type(&self) -> SchemaType<'_> { (*self).schema_type() } @@ -266,7 +262,7 @@ where } impl QueryableSchema for &SchemaObject { - fn schema_type(&self) -> SchemaType { + fn schema_type(&self) -> SchemaType<'_> { // TODO: Technically speaking, it is allowed to use the "X of" schema types in conjunction // with other schema types i.e. `allOf` in conjunction with specifying a `type`. // @@ -286,7 +282,9 @@ impl QueryableSchema for &SchemaObject { } else if let Some(any_of) = subschemas.any_of.as_ref() { return SchemaType::AnyOf(any_of.iter().map(schema_to_simple_schema).collect()); } else { - panic!("Encountered schema with subschema validation that wasn't one of the supported types: allOf, oneOf, anyOf."); + panic!( + "Encountered schema with subschema validation that wasn't one of the supported types: allOf, oneOf, anyOf." + ); } } @@ -388,7 +386,7 @@ impl<'a> From<&'a SchemaObject> for SimpleSchema<'a> { } impl QueryableSchema for SimpleSchema<'_> { - fn schema_type(&self) -> SchemaType { + fn schema_type(&self) -> SchemaType<'_> { self.schema.schema_type() } diff --git a/lib/vector-config/src/schema/visitors/human_name.rs b/lib/vector-config/src/schema/visitors/human_name.rs index 4b9b2330d5..65f863e1fc 100644 --- a/lib/vector-config/src/schema/visitors/human_name.rs +++ b/lib/vector-config/src/schema/visitors/human_name.rs @@ -59,18 +59,18 @@ impl Visitor for GenerateHumanFriendlyNameVisitor { // property's schema if it doesn't already have a human-friendly name defined. if let Some(properties) = schema.object.as_mut().map(|object| &mut object.properties) { for (property_name, property_schema) in properties.iter_mut() { - if let Some(property_schema) = property_schema.as_object_mut() { - if !has_schema_metadata_attr_str( + if let Some(property_schema) = property_schema.as_object_mut() + && !has_schema_metadata_attr_str( property_schema, constants::DOCS_META_HUMAN_NAME, - ) { - let human_name = generate_human_friendly_string(property_name); - set_schema_metadata_attr_str( - property_schema, - constants::DOCS_META_HUMAN_NAME, - human_name, - ); - } + ) + { + let human_name = generate_human_friendly_string(property_name); + set_schema_metadata_attr_str( + property_schema, + constants::DOCS_META_HUMAN_NAME, + human_name, + ); } } } diff --git a/lib/vector-config/src/schema/visitors/inline_single.rs b/lib/vector-config/src/schema/visitors/inline_single.rs index 4aee9499b2..fff58b78bc 100644 --- a/lib/vector-config/src/schema/visitors/inline_single.rs +++ b/lib/vector-config/src/schema/visitors/inline_single.rs @@ -7,7 +7,7 @@ use vector_config_common::schema::{visit::Visitor, *}; use crate::schema::visitors::merge::Mergeable; use super::scoped_visit::{ - visit_schema_object_scoped, SchemaReference, SchemaScopeStack, ScopedVisitor, + SchemaReference, SchemaScopeStack, ScopedVisitor, visit_schema_object_scoped, }; /// A visitor that inlines schema references where the referenced schema is only referenced once. @@ -41,7 +41,7 @@ impl Visitor for InlineSingleUseReferencesVisitor { // entire schema, by visiting the root schema in a recursive fashion, using a helper visitor. let mut occurrence_visitor = OccurrenceVisitor::default(); occurrence_visitor.visit_root_schema(root); - let occurrence_map = occurrence_visitor.into_occurrences(); + let occurrence_map = occurrence_visitor.occurrence_map; self.eligible_to_inline = occurrence_map .into_iter() @@ -141,16 +141,7 @@ fn is_inlineable_schema(definition_name: &str, schema: &SchemaObject) -> bool { #[derive(Debug, Default)] struct OccurrenceVisitor { scope_stack: SchemaScopeStack, - occurrence_map: HashMap>, -} - -impl OccurrenceVisitor { - fn into_occurrences(self) -> HashMap { - self.occurrence_map - .into_iter() - .map(|(k, v)| (k, v.len())) - .collect() - } + occurrence_map: HashMap, } impl Visitor for OccurrenceVisitor { @@ -162,23 +153,11 @@ impl Visitor for OccurrenceVisitor { visit_schema_object_scoped(self, definitions, schema); if let Some(current_schema_ref) = schema.reference.as_ref() { - // Track the named "parent" schema for the schema we're currently visiting so that if we - // visit this schema again, we don't double count any schema references that it has. The - // "parent" schema is simply the closest ancestor schema that was itself a schema - // reference, or "Root", which represents the oldest schema ancestor in the document. - // - // This lets us couple with scenarios where schema A references schema B, and is the - // only actual direct schema reference to schema B, but due to multiple schemas - // referencing schema A, would otherwise lead to both A and B being visited multiple - // times. - let current_parent_schema_ref = self.get_current_schema_scope().clone(); let current_schema_ref = get_cleaned_schema_reference(current_schema_ref); - - let occurrences = self + *self .occurrence_map .entry(current_schema_ref.into()) - .or_default(); - occurrences.insert(current_parent_schema_ref); + .or_default() += 1; } } } @@ -257,17 +236,7 @@ mod tests { assert_schemas_eq(expected_schema, actual_schema); } - // TODO(tobz): These two tests are ignored because the inliner currently works off of schema - // reference scopes, so two object properties within the same schema don't count as two - // instances of a schema being referenced. - // - // We need to refactor schema scopes to be piecemeal extensible more in the way of how - // `jsonschema` does it rather than an actual stack.... the current approach is good enough for - // now, but not optimal in the way that it could be. - // - // These are here for when I improve the situation after getting this merged. #[test] - #[ignore] fn single_ref_multiple_usages() { let mut actual_schema = as_schema(json!({ "definitions": { @@ -294,7 +263,6 @@ mod tests { } #[test] - #[ignore] fn multiple_refs_mixed_usages() { let mut actual_schema = as_schema(json!({ "definitions": { @@ -346,4 +314,131 @@ mod tests { assert_schemas_eq(expected_schema, actual_schema); } + + #[test] + fn reference_in_multiple_arrays() { + let mut actual_schema = as_schema(json!({ + "definitions": { + "item": { + "type": "object", + "properties": { + "x": { "type": "string" } + } + } + }, + "type": "object", + "properties": { + "arr1": { "type": "array", "items": { "$ref": "#/definitions/item" } }, + "arr2": { "type": "array", "items": { "$ref": "#/definitions/item" } } + } + })); + + let expected_schema = actual_schema.clone(); + + let mut visitor = InlineSingleUseReferencesVisitor::default(); + visitor.visit_root_schema(&mut actual_schema); + + assert_schemas_eq(expected_schema, actual_schema); + } + + #[test] + fn reference_in_oneof_anyof_allof() { + let mut actual_schema = as_schema(json!({ + "definitions": { + "shared": { + "type": "object", + "properties": { + "y": { "type": "string" } + } + } + }, + "type": "object", + "properties": { + "choice": { + "oneOf": [ + { "$ref": "#/definitions/shared" }, + { "$ref": "#/definitions/shared" } + ], + "anyOf": [ + { "$ref": "#/definitions/shared" }, + { "type": "null" } + ], + "allOf": [ + { "$ref": "#/definitions/shared" }, + { "type": "object" } + ] + } + } + })); + + let expected_schema = actual_schema.clone(); + + let mut visitor = InlineSingleUseReferencesVisitor::default(); + visitor.visit_root_schema(&mut actual_schema); + + assert_schemas_eq(expected_schema, actual_schema); + } + + #[test] + fn reference_in_additional_properties() { + let mut actual_schema = as_schema(json!({ + "definitions": { + "val": { + "type": "object", + "properties": { + "z": { "type": "string" } + } + } + }, + "type": "object", + "properties": { + "obj1": { + "type": "object", + "additionalProperties": { "$ref": "#/definitions/val" } + }, + "obj2": { + "type": "object", + "additionalProperties": { "$ref": "#/definitions/val" } + } + } + })); + + let expected_schema = actual_schema.clone(); + + let mut visitor = InlineSingleUseReferencesVisitor::default(); + visitor.visit_root_schema(&mut actual_schema); + + assert_schemas_eq(expected_schema, actual_schema); + } + + #[test] + fn reference_in_pattern_properties() { + let mut actual_schema = as_schema(json!({ + "definitions": { + "pat": { + "type": "object", + "properties": { + "w": { "type": "string" } + } + } + }, + "type": "object", + "properties": { + "obj": { + "type": "object", + "patternProperties": { + "^foo$": { "$ref": "#/definitions/pat" }, + "^bar$": { "$ref": "#/definitions/pat" } + } + } + } + })); + + let expected_schema = actual_schema.clone(); + + let mut visitor = InlineSingleUseReferencesVisitor::default(); + visitor.visit_root_schema(&mut actual_schema); + + assert_schemas_eq(expected_schema, actual_schema); + } } diff --git a/lib/vector-config/src/schema/visitors/merge.rs b/lib/vector-config/src/schema/visitors/merge.rs index 28b724d602..4d259611b5 100644 --- a/lib/vector-config/src/schema/visitors/merge.rs +++ b/lib/vector-config/src/schema/visitors/merge.rs @@ -50,7 +50,9 @@ impl Mergeable for Value { // We _may_ need to relax this in practice/in the future, but it's a solid invariant to // enforce for the time being. if discriminant(self) != discriminant(other) { - panic!("Tried to merge two `Value` types together with differing types!\n\nSelf: {:?}\n\nOther: {:?}", self, other); + panic!( + "Tried to merge two `Value` types together with differing types!\n\nSelf: {self:?}\n\nOther: {other:?}" + ); } match (self, other) { diff --git a/lib/vector-config/src/schema/visitors/scoped_visit.rs b/lib/vector-config/src/schema/visitors/scoped_visit.rs index 937a4ee492..ded4115ad8 100644 --- a/lib/vector-config/src/schema/visitors/scoped_visit.rs +++ b/lib/vector-config/src/schema/visitors/scoped_visit.rs @@ -15,7 +15,7 @@ pub enum SchemaReference { impl std::fmt::Display for SchemaReference { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - Self::Definition(name) => write!(f, "{}", name), + Self::Definition(name) => write!(f, "{name}"), Self::Root => write!(f, ""), } } @@ -124,10 +124,10 @@ fn visit_box_scoped( definitions: &mut Map, target: &mut Option>, ) { - if let Some(s) = target { - if let Schema::Object(s) = s.as_mut() { - sv.visit_schema_object(definitions, s); - } + if let Some(s) = target + && let Schema::Object(s) = s.as_mut() + { + sv.visit_schema_object(definitions, s); } } diff --git a/lib/vector-config/src/schema/visitors/unevaluated.rs b/lib/vector-config/src/schema/visitors/unevaluated.rs index 105c3dbfca..1ef8caece2 100644 --- a/lib/vector-config/src/schema/visitors/unevaluated.rs +++ b/lib/vector-config/src/schema/visitors/unevaluated.rs @@ -1,18 +1,15 @@ -use std::{ - collections::{HashMap, HashSet}, - convert::identity, -}; +use std::collections::{HashMap, HashSet}; use tracing::debug; use vector_config_common::schema::{ - visit::{with_resolved_schema_reference, Visitor}, + visit::{Visitor, with_resolved_schema_reference}, *, }; use crate::schema::visitors::merge::Mergeable; use super::scoped_visit::{ - visit_schema_object_scoped, SchemaReference, SchemaScopeStack, ScopedVisitor, + SchemaReference, SchemaScopeStack, ScopedVisitor, visit_schema_object_scoped, }; /// A visitor that marks schemas as closed by disallowing unknown properties via @@ -75,29 +72,29 @@ impl Visitor for DisallowUnevaluatedPropertiesVisitor { if let Some(reference) = schema.reference.as_ref() { let current_parent_schema_ref = self.get_current_schema_scope(); - if let Some(referrers) = self.eligible_to_flatten.get(reference) { - if referrers.contains(current_parent_schema_ref) { - let current_schema_ref = get_cleaned_schema_reference(reference); - let referenced_schema = definitions - .get(current_schema_ref) - .expect("schema definition must exist"); - + if let Some(referrers) = self.eligible_to_flatten.get(reference) + && referrers.contains(current_parent_schema_ref) + { + let current_schema_ref = get_cleaned_schema_reference(reference); + let referenced_schema = definitions + .get(current_schema_ref) + .expect("schema definition must exist"); + + debug!( + referent = current_schema_ref, + referrer = current_parent_schema_ref.as_ref(), + "Found eligible referent/referrer mapping." + ); + + if let Schema::Object(referenced_schema) = referenced_schema { debug!( referent = current_schema_ref, referrer = current_parent_schema_ref.as_ref(), - "Found eligible referent/referrer mapping." + "Flattening referent into referrer." ); - if let Schema::Object(referenced_schema) = referenced_schema { - debug!( - referent = current_schema_ref, - referrer = current_parent_schema_ref.as_ref(), - "Flattening referent into referrer." - ); - - schema.reference = None; - schema.merge(referenced_schema); - } + schema.reference = None; + schema.merge(referenced_schema); } } } @@ -160,15 +157,15 @@ fn unmark_or_flatten_schema(definitions: &mut Map, schema: &mut object.unevaluated_properties = Some(Box::new(Schema::Bool(true))); } else { with_resolved_schema_reference(definitions, schema, |_, schema_ref, resolved| { - if let Schema::Object(resolved) = resolved { - if let Some(object) = resolved.object.as_mut() { - debug!( - referent = schema_ref, - "Unmarked subschema by traversing schema reference." - ); + if let Schema::Object(resolved) = resolved + && let Some(object) = resolved.object.as_mut() + { + debug!( + referent = schema_ref, + "Unmarked subschema by traversing schema reference." + ); - object.unevaluated_properties = Some(Box::new(Schema::Bool(true))); - } + object.unevaluated_properties = Some(Box::new(Schema::Bool(true))); } }); } @@ -353,26 +350,23 @@ fn is_markable_schema(definitions: &Map, schema: &SchemaObject) let has_object_subschema = subschemas .iter() .any(|schema| is_markable_schema(definitions, schema)); - let has_referenced_object_subschema = subschemas - .iter() - .map(|subschema| { - subschema - .reference - .as_ref() - .and_then(|reference| { - let reference = get_cleaned_schema_reference(reference); - definitions.get_key_value(reference) - }) - .and_then(|(name, schema)| schema.as_object().map(|schema| (name, schema))) - .map_or(false, |(name, schema)| { - debug!( - "Following schema reference '{}' for subschema markability.", - name - ); - is_markable_schema(definitions, schema) - }) - }) - .any(identity); + let has_referenced_object_subschema = subschemas.iter().any(|subschema| { + subschema + .reference + .as_ref() + .and_then(|reference| { + let reference = get_cleaned_schema_reference(reference); + definitions.get_key_value(reference) + }) + .and_then(|(name, schema)| schema.as_object().map(|schema| (name, schema))) + .is_some_and(|(name, schema)| { + debug!( + "Following schema reference '{}' for subschema markability.", + name + ); + is_markable_schema(definitions, schema) + }) + }); debug!( "Schema {} object subschema(s) and {} referenced subschemas.", @@ -431,13 +425,13 @@ fn get_referents(parent_schema: &SchemaObject, referents: &mut HashSet) -> Result { + fn generate_schema( + generator: &RefCell, + ) -> Result { // Forward to the underlying `H`. - H::generate_schema(gen) + H::generate_schema(generator) } } diff --git a/lib/vector-config/src/stdlib.rs b/lib/vector-config/src/stdlib.rs index ad3ae47aac..c56983fd28 100644 --- a/lib/vector-config/src/stdlib.rs +++ b/lib/vector-config/src/stdlib.rs @@ -2,10 +2,10 @@ use std::{ cell::RefCell, collections::{BTreeMap, BTreeSet, HashMap, HashSet}, hash::Hash, - net::SocketAddr, + net::{Ipv4Addr, SocketAddr}, num::{ - NonZeroI16, NonZeroI32, NonZeroI64, NonZeroI8, NonZeroU16, NonZeroU32, NonZeroU64, - NonZeroU8, NonZeroUsize, + NonZeroI8, NonZeroI16, NonZeroI32, NonZeroI64, NonZeroU8, NonZeroU16, NonZeroU32, + NonZeroU64, NonZeroUsize, }, path::PathBuf, time::Duration, @@ -17,14 +17,14 @@ use vector_config_common::{attributes::CustomAttribute, constants, validation::V use vrl::value::KeyString; use crate::{ + Configurable, GenerateError, Metadata, ToValue, num::ConfigurableNumber, schema::{ - assert_string_schema_for_map, generate_array_schema, generate_bool_schema, - generate_map_schema, generate_number_schema, generate_optional_schema, generate_set_schema, - generate_string_schema, SchemaGenerator, SchemaObject, + SchemaGenerator, SchemaObject, assert_string_schema_for_map, generate_array_schema, + generate_bool_schema, generate_map_schema, generate_number_schema, + generate_optional_schema, generate_set_schema, generate_string_schema, }, str::ConfigurableString, - Configurable, GenerateError, Metadata, ToValue, }; // Unit type. @@ -67,8 +67,10 @@ where T::validate_metadata(&converted) } - fn generate_schema(gen: &RefCell) -> Result { - generate_optional_schema(&T::as_configurable_ref(), gen) + fn generate_schema( + generator: &RefCell, + ) -> Result { + generate_optional_schema(&T::as_configurable_ref(), generator) } } @@ -222,8 +224,10 @@ where T::validate_metadata(&converted) } - fn generate_schema(gen: &RefCell) -> Result { - generate_array_schema(&T::as_configurable_ref(), gen) + fn generate_schema( + generator: &RefCell, + ) -> Result { + generate_array_schema(&T::as_configurable_ref(), generator) } } @@ -253,15 +257,17 @@ where V::validate_metadata(&converted) } - fn generate_schema(gen: &RefCell) -> Result { + fn generate_schema( + generator: &RefCell, + ) -> Result { // Make sure our key type is _truly_ a string schema. assert_string_schema_for_map( &K::as_configurable_ref(), - gen, + generator, std::any::type_name::(), )?; - generate_map_schema(&V::as_configurable_ref(), gen) + generate_map_schema(&V::as_configurable_ref(), generator) } } @@ -292,8 +298,10 @@ where V::validate_metadata(&converted) } - fn generate_schema(gen: &RefCell) -> Result { - generate_set_schema(&V::as_configurable_ref(), gen) + fn generate_schema( + generator: &RefCell, + ) -> Result { + generate_set_schema(&V::as_configurable_ref(), generator) } } @@ -323,15 +331,17 @@ where V::validate_metadata(&converted) } - fn generate_schema(gen: &RefCell) -> Result { + fn generate_schema( + generator: &RefCell, + ) -> Result { // Make sure our key type is _truly_ a string schema. assert_string_schema_for_map( &K::as_configurable_ref(), - gen, + generator, std::any::type_name::(), )?; - generate_map_schema(&V::as_configurable_ref(), gen) + generate_map_schema(&V::as_configurable_ref(), generator) } } @@ -362,8 +372,10 @@ where V::validate_metadata(&converted) } - fn generate_schema(gen: &RefCell) -> Result { - generate_set_schema(&V::as_configurable_ref(), gen) + fn generate_schema( + generator: &RefCell, + ) -> Result { + generate_set_schema(&V::as_configurable_ref(), generator) } } @@ -402,6 +414,31 @@ impl ToValue for SocketAddr { } } +impl Configurable for Ipv4Addr { + fn referenceable_name() -> Option<&'static str> { + Some("stdlib::Ipv4Addr") + } + + fn metadata() -> Metadata { + let mut metadata = Metadata::default(); + metadata.set_description("An IPv4 address."); + metadata + } + + fn generate_schema(_: &RefCell) -> Result { + // TODO: We don't need anything other than a string schema to (de)serialize a `Ipv4Addr`, + // but we eventually should have validation since the format for the possible permutations + // is well-known and can be easily codified. + Ok(generate_string_schema()) + } +} + +impl ToValue for Ipv4Addr { + fn to_value(&self) -> Value { + Value::String(self.to_string()) + } +} + impl Configurable for PathBuf { fn referenceable_name() -> Option<&'static str> { Some("stdlib::PathBuf") diff --git a/lib/vector-config/tests/integration/configurable_string.rs b/lib/vector-config/tests/integration/configurable_string.rs index 6a1e8d3b38..e54fa9b845 100644 --- a/lib/vector-config/tests/integration/configurable_string.rs +++ b/lib/vector-config/tests/integration/configurable_string.rs @@ -2,7 +2,7 @@ use std::collections::{BTreeMap, HashMap}; use std::fmt; use indexmap::IndexMap; -use vector_config::{configurable_component, schema::generate_root_schema, ConfigurableString}; +use vector_config::{ConfigurableString, configurable_component, schema::generate_root_schema}; /// A type that pretends to be `ConfigurableString` but has a non-string-like schema. #[configurable_component] diff --git a/lib/vector-config/tests/integration/smoke.rs b/lib/vector-config/tests/integration/smoke.rs index 8d9d1fbb8e..12c1f2dd5d 100644 --- a/lib/vector-config/tests/integration/smoke.rs +++ b/lib/vector-config/tests/integration/smoke.rs @@ -16,8 +16,8 @@ use std::{ use indexmap::IndexMap; use serde_with::serde_as; use vector_config::{ - component::GenerateConfig, configurable_component, schema::generate_root_schema, - ConfigurableString, + ConfigurableString, component::GenerateConfig, configurable_component, + schema::generate_root_schema, }; /// A templated string. @@ -312,7 +312,7 @@ impl From for String { if fd == 0 { "systemd".to_owned() } else { - format!("systemd#{}", fd) + format!("systemd#{fd}") } } } @@ -633,8 +633,73 @@ fn generate_semi_real_schema() { let json = serde_json::to_string_pretty(&schema) .expect("rendering root schema to JSON should not fail"); - println!("{}", json); + println!("{json}"); } - Err(e) => eprintln!("error while generating schema: {:?}", e), + Err(e) => eprintln!("error while generating schema: {e:?}"), } } + +/// A tagged enum with a trailing untagged variant should generate a schema where the +/// untagged variant does not require the tag field. +/// +/// This type exists only to validate schema generation behavior. +#[derive(Clone, Debug)] +#[configurable_component] +#[serde(tag = "type")] +enum WithTrailingUntaggedVariant { + /// Tagged struct variant. + Foo { + /// Some numeric value. + value: u32, + }, + /// Tagged unit variant. + Bar, + /// Untagged fallback variant. + #[serde(untagged)] + Fallback(String), + /// Another untagged fallback variant, also allowed by serde. + #[serde(untagged)] + FallbackAlt(u64), +} + +#[test] +fn tagged_enum_with_trailing_untagged_variant_schema() { + let root = + generate_root_schema::().expect("should generate root schema"); + let schema = serde_json::to_value(root.schema).expect("serialize schema to JSON"); + + let one_of = schema + .get("oneOf") + .and_then(|v| v.as_array()) + .expect("enum schema should use oneOf"); + + let mut tagged_schemas = 0usize; + let mut untagged_schemas = 0usize; + + for subschema in one_of { + let properties = subschema.get("properties"); + let required = subschema.get("required"); + + let has_type_property = properties.and_then(|p| p.get("type")).is_some(); + let requires_type = required + .and_then(|r| r.as_array()) + .map(|arr| arr.iter().any(|v| v == "type")) + .unwrap_or(false); + + if has_type_property || requires_type { + tagged_schemas += 1; + } else { + untagged_schemas += 1; + + // The untagged variants in this test are newtypes: one string and one integer. + let instance_type = subschema.get("type").and_then(|t| t.as_str()); + assert!(matches!(instance_type, Some("string") | Some("integer"))); + } + } + + assert_eq!(tagged_schemas, 2, "expected two tagged variants in schema"); + assert_eq!( + untagged_schemas, 2, + "expected two untagged variants in schema" + ); +} diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml index 0b0613b4da..aa46526ed7 100644 --- a/lib/vector-core/Cargo.toml +++ b/lib/vector-core/Cargo.toml @@ -2,22 +2,22 @@ name = "vector-core" version = "0.1.0" authors = ["Vector Contributors "] -edition = "2021" +edition = "2024" publish = false [lints.rust] unexpected_cfgs = { level = "warn", check-cfg = ['cfg(ddsketch_extended)'] } [dependencies] -async-trait = { version = "0.1", default-features = false } +async-trait.workspace = true bitmask-enum = { version = "2.2.5", default-features = false } -bytes = { version = "1.9.0", default-features = false, features = ["serde"] } +bytes = { workspace = true, features = ["serde"] } chrono.workspace = true chrono-tz.workspace = true -crossbeam-utils = { version = "0.8.21", default-features = false } -derivative = { version = "2.2.0", default-features = false } -dyn-clone = { version = "1.0.17", default-features = false } -enumflags2 = { version = "0.7.10", default-features = false } +crossbeam-utils.workspace = true +derivative.workspace = true +dyn-clone = { version = "1.0.20", default-features = false } +enumflags2 = { version = "0.7.12", default-features = false } float_eq = { version = "1.0", default-features = false } futures.workspace = true futures-util = { version = "0.3.29", default-features = false, features = ["std"] } @@ -25,36 +25,37 @@ headers = { version = "0.3.9", default-features = false } http = { version = "0.2.9", default-features = false } hyper-proxy = { version = "0.9.1", default-features = false, features = ["openssl-tls"] } indexmap.workspace = true +inventory.workspace = true ipnet = { version = "2", default-features = false, features = ["serde", "std"] } lookup = { package = "vector-lookup", path = "../vector-lookup" } metrics.workspace = true metrics-tracing-context.workspace = true metrics-util.workspace = true -mlua = { version = "0.10.2", default-features = false, features = ["lua54", "send", "vendored"], optional = true } -no-proxy = { version = "0.3.5", default-features = false, features = ["serialize"] } -ordered-float = { version = "4.6.0", default-features = false } -openssl = { version = "0.10.68", default-features = false, features = ["vendored"] } -parking_lot = { version = "0.12.3", default-features = false } +mlua = { version = "0.10.5", default-features = false, features = ["lua54", "send", "vendored"], optional = true } +no-proxy = { version = "0.3.6", default-features = false, features = ["serialize"] } +ordered-float.workspace = true +openssl = { version = "0.10.73", default-features = false, features = ["vendored"] } +parking_lot = { version = "0.12.4", default-features = false } pin-project.workspace = true -proptest = { version = "1.5", optional = true } +proptest = { version = "1.7", optional = true } prost-types.workspace = true -prost .workspace = true -quanta = { version = "0.12.5", default-features = false } -regex = { version = "1.11.1", default-features = false, features = ["std", "perf"] } +prost.workspace = true +quanta = { version = "0.12.6", default-features = false } +regex = { version = "1.11.2", default-features = false, features = ["std", "perf"] } ryu = { version = "1", default-features = false } serde.workspace = true serde_json.workspace = true -serde_with = { version = "3.12.0", default-features = false, features = ["std", "macros"] } +serde_with = { version = "3.14.0", default-features = false, features = ["std", "macros"] } smallvec = { version = "1", default-features = false, features = ["serde", "const_generics"] } snafu.workspace = true -socket2 = { version = "0.5.8", default-features = false } -tokio = { version = "1.43.0", default-features = false, features = ["net"] } +socket2.workspace = true +tokio = { workspace = true, features = ["net"] } tokio-openssl = { version = "0.6.5", default-features = false } tokio-stream = { version = "0.1", default-features = false, features = ["time"], optional = true } tokio-util = { version = "0.7.0", default-features = false, features = ["time"] } toml.workspace = true tonic.workspace = true -tracing = { version = "0.1.34", default-features = false } +tracing.workspace = true url = { version = "2", default-features = false } urlencoding = { version = "2.1.3", default-features = false } uuid.workspace = true @@ -66,7 +67,7 @@ vrl.workspace = true cfg-if.workspace = true [target.'cfg(target_os = "macos")'.dependencies] -security-framework = "2.10.0" +security-framework = "3.3.0" [target.'cfg(windows)'.dependencies] schannel = "0.1.27" @@ -77,21 +78,21 @@ prost-build.workspace = true [dev-dependencies] base64 = "0.22.1" chrono-tz.workspace = true -criterion = { version = "0.5.1", features = ["html_reports"] } +criterion = { version = "0.7.0", features = ["html_reports"] } env-test-util = "1.0.1" quickcheck = "1" quickcheck_macros = "1" -proptest = "1.5" -similar-asserts = "1.6.0" +proptest = "1.7" +similar-asserts = "1.7.0" tokio-test = "0.4.4" toml.workspace = true ndarray = "0.16.1" ndarray-stats = "0.6.0" noisy_float = "0.2.0" -rand = "0.8.5" -rand_distr = "0.4.3" +rand.workspace = true +rand_distr.workspace = true serde_yaml = { version = "0.9.34", default-features = false } -tracing-subscriber = { version = "0.3.19", default-features = false, features = ["env-filter", "fmt", "ansi", "registry"] } +tracing-subscriber = { workspace = true, features = ["env-filter", "fmt", "ansi", "registry"] } vector-common = { path = "../vector-common", default-features = false, features = ["test"] } [features] diff --git a/lib/vector-core/benches/event/log_event.rs b/lib/vector-core/benches/event/log_event.rs index 7395ab98a6..718dc93b9f 100644 --- a/lib/vector-core/benches/event/log_event.rs +++ b/lib/vector-core/benches/event/log_event.rs @@ -1,7 +1,7 @@ use std::time::Duration; use criterion::{ - criterion_group, measurement::WallTime, BatchSize, BenchmarkGroup, Criterion, SamplingMode, + BatchSize, BenchmarkGroup, Criterion, SamplingMode, criterion_group, measurement::WallTime, }; use lookup::event_path; use vector_core::event::LogEvent; diff --git a/lib/vector-core/src/config/global_options.rs b/lib/vector-core/src/config/global_options.rs index 96bdd682eb..9ca92a457b 100644 --- a/lib/vector-core/src/config/global_options.rs +++ b/lib/vector-core/src/config/global_options.rs @@ -2,11 +2,12 @@ use std::{fs::DirBuilder, path::PathBuf, time::Duration}; use snafu::{ResultExt, Snafu}; use vector_common::TimeZone; -use vector_config::configurable_component; +use vector_config::{configurable_component, impl_generate_config_from_default}; use super::super::default_data_dir; use super::Telemetry; -use super::{proxy::ProxyConfig, AcknowledgementsConfig, LogSchema}; +use super::metrics_expiration::PerMetricSetExpiration; +use super::{AcknowledgementsConfig, LogSchema, proxy::ProxyConfig}; use crate::serde::bool_or_struct; #[derive(Debug, Snafu)] @@ -30,11 +31,24 @@ pub(crate) enum DataDirError { }, } +/// Specifies the wildcard matching mode, relaxed allows configurations where wildcard doesn not match any existing inputs +#[configurable_component] +#[derive(Clone, Debug, Copy, PartialEq, Eq, Default)] +#[serde(rename_all = "lowercase")] +pub enum WildcardMatching { + /// Strict matching (must match at least one existing input) + #[default] + Strict, + + /// Relaxed matching (must match 0 or more inputs) + Relaxed, +} + /// Global configuration options. // // If this is modified, make sure those changes are reflected in the `ConfigBuilder::append` // function! -#[configurable_component] +#[configurable_component(global_option("global_option"))] #[derive(Clone, Debug, Default, PartialEq)] pub struct GlobalOptions { /// The directory used for persisting Vector state data. @@ -44,13 +58,23 @@ pub struct GlobalOptions { /// /// Vector must have write permissions to this directory. #[serde(default = "crate::default_data_dir")] + #[configurable(metadata(docs::common = false))] pub data_dir: Option, + /// Set wildcard matching mode for inputs + /// + /// Setting this to "relaxed" allows configurations with wildcards that do not match any inputs + /// to be accepted without causing an error. + #[serde(skip_serializing_if = "crate::serde::is_default")] + #[configurable(metadata(docs::common = false, docs::required = false))] + pub wildcard_matching: Option, + /// Default log schema for all events. /// /// This is used if a component does not have its own specific log schema. All events use a log /// schema, whether or not the default is used, to assign event fields on incoming events. #[serde(default, skip_serializing_if = "crate::serde::is_default")] + #[configurable(metadata(docs::common = false, docs::required = false))] pub log_schema: LogSchema, /// Telemetry options. @@ -58,6 +82,7 @@ pub struct GlobalOptions { /// Determines whether `source` and `service` tags should be emitted with the /// `component_sent_*` and `component_received_*` events. #[serde(default, skip_serializing_if = "crate::serde::is_default")] + #[configurable(metadata(docs::common = false, docs::required = false))] pub telemetry: Telemetry, /// The name of the time zone to apply to timestamp conversions that do not contain an explicit time zone. @@ -69,10 +94,12 @@ pub struct GlobalOptions { /// /// [tzdb]: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones #[serde(default, skip_serializing_if = "crate::serde::is_default")] + #[configurable(metadata(docs::common = false))] pub timezone: Option, #[configurable(derived)] #[serde(default, skip_serializing_if = "crate::serde::is_default")] + #[configurable(metadata(docs::common = false, docs::required = false))] pub proxy: ProxyConfig, /// Controls how acknowledgements are handled for all sinks by default. @@ -80,31 +107,42 @@ pub struct GlobalOptions { /// See [End-to-end Acknowledgements][e2e_acks] for more information on how Vector handles event /// acknowledgement. /// - /// [e2e_acks]: https://vector.dev/docs/about/under-the-hood/architecture/end-to-end-acknowledgements/ + /// [e2e_acks]: https://vector.dev/docs/architecture/end-to-end-acknowledgements/ #[serde( default, deserialize_with = "bool_or_struct", skip_serializing_if = "crate::serde::is_default" )] + #[configurable(metadata(docs::common = true, docs::required = false))] pub acknowledgements: AcknowledgementsConfig, /// The amount of time, in seconds, that internal metrics will persist after having not been /// updated before they expire and are removed. /// - /// Deprecated: use expire_metrics_secs instead + /// Deprecated: use `expire_metrics_secs` instead #[configurable(deprecated)] #[serde(default, skip_serializing_if = "crate::serde::is_default")] + #[configurable(metadata(docs::hidden))] pub expire_metrics: Option, /// The amount of time, in seconds, that internal metrics will persist after having not been /// updated before they expire and are removed. /// /// Set this to a value larger than your `internal_metrics` scrape interval (default 5 minutes) - /// that metrics live long enough to be emitted and captured, + /// so metrics live long enough to be emitted and captured. #[serde(skip_serializing_if = "crate::serde::is_default")] + #[configurable(metadata(docs::common = false, docs::required = false))] pub expire_metrics_secs: Option, + + /// This allows configuring different expiration intervals for different metric sets. + /// By default this is empty and any metric not matched by one of these sets will use + /// the global default value, defined using `expire_metrics_secs`. + #[serde(skip_serializing_if = "crate::serde::is_default")] + pub expire_metrics_per_metric_set: Option>, } +impl_generate_config_from_default!(GlobalOptions); + impl GlobalOptions { /// Resolve the `data_dir` option in either the global or local config, and /// validate that it exists and is writable. @@ -165,6 +203,13 @@ impl GlobalOptions { pub fn merge(&self, with: Self) -> Result> { let mut errors = Vec::new(); + if conflicts( + self.wildcard_matching.as_ref(), + with.wildcard_matching.as_ref(), + ) { + errors.push("conflicting values for 'wildcard_matching' found".to_owned()); + } + if conflicts(self.proxy.http.as_ref(), with.proxy.http.as_ref()) { errors.push("conflicting values for 'proxy.http' found".to_owned()); } @@ -220,9 +265,20 @@ impl GlobalOptions { let mut telemetry = self.telemetry.clone(); telemetry.merge(&with.telemetry); + let merged_expire_metrics_per_metric_set = match ( + &self.expire_metrics_per_metric_set, + &with.expire_metrics_per_metric_set, + ) { + (Some(a), Some(b)) => Some(a.iter().chain(b).cloned().collect()), + (Some(a), None) => Some(a.clone()), + (None, Some(b)) => Some(b.clone()), + (None, None) => None, + }; + if errors.is_empty() { Ok(Self { data_dir, + wildcard_matching: self.wildcard_matching.or(with.wildcard_matching), log_schema, telemetry, acknowledgements: self.acknowledgements.merge_default(&with.acknowledgements), @@ -230,6 +286,7 @@ impl GlobalOptions { proxy: self.proxy.merge(&with.proxy), expire_metrics: self.expire_metrics.or(with.expire_metrics), expire_metrics_secs: self.expire_metrics_secs.or(with.expire_metrics_secs), + expire_metrics_per_metric_set: merged_expire_metrics_per_metric_set, }) } else { Err(errors) @@ -240,6 +297,39 @@ impl GlobalOptions { pub fn timezone(&self) -> TimeZone { self.timezone.unwrap_or(TimeZone::Local) } + + /// Returns a list of top-level field names that differ between two [`GlobalOptions`] values. + /// + /// This function performs a shallow comparison by serializing both configs to JSON + /// and comparing their top-level keys. + /// + /// Useful for logging which global fields changed during config reload attempts. + /// + /// # Errors + /// + /// Returns a [`serde_json::Error`] if either of the [`GlobalOptions`] values + /// cannot be serialized into a JSON object. This is unlikely under normal usage, + /// but may occur if serialization fails due to unexpected data structures or changes + /// in the type definition. + pub fn diff(&self, other: &Self) -> Result, serde_json::Error> { + let old_value = serde_json::to_value(self)?; + let new_value = serde_json::to_value(other)?; + + let serde_json::Value::Object(old_map) = old_value else { + return Ok(vec![]); + }; + let serde_json::Value::Object(new_map) = new_value else { + return Ok(vec![]); + }; + + Ok(old_map + .iter() + .filter_map(|(k, v_old)| match new_map.get(k) { + Some(v_new) if v_new != v_old => Some(k.clone()), + _ => None, + }) + .collect()) + } } fn conflicts(this: Option<&T>, that: Option<&T>) -> bool { @@ -358,6 +448,22 @@ mod tests { ); } + #[test] + fn diff_detects_changed_keys() { + let old = GlobalOptions { + data_dir: Some(std::path::PathBuf::from("/path1")), + ..Default::default() + }; + let new = GlobalOptions { + data_dir: Some(std::path::PathBuf::from("/path2")), + ..Default::default() + }; + assert_eq!( + old.diff(&new).expect("diff failed"), + vec!["data_dir".to_string()] + ); + } + fn merge( name: &str, dd1: Option

, @@ -371,7 +477,7 @@ mod tests { } fn make_config(name: &str, value: Option

) -> GlobalOptions { - toml::from_str(&value.map_or(String::new(), |value| format!(r#"{name} = {value:?}"#))) + toml::from_str(&value.map_or(String::new(), |value| format!(r"{name} = {value:?}"))) .unwrap() } } diff --git a/lib/vector-core/src/config/log_schema.rs b/lib/vector-core/src/config/log_schema.rs index 2ab6c423a6..bbbcfaeba4 100644 --- a/lib/vector-core/src/config/log_schema.rs +++ b/lib/vector-core/src/config/log_schema.rs @@ -45,6 +45,7 @@ pub fn log_schema() -> &'static LogSchema { #[configurable_component] #[derive(Clone, Debug, Eq, PartialEq)] #[serde(default)] +#[allow(clippy::struct_field_names)] pub struct LogSchema { /// The name of the event field to treat as the event message. /// diff --git a/lib/vector-core/src/config/metrics_expiration.rs b/lib/vector-core/src/config/metrics_expiration.rs new file mode 100644 index 0000000000..2ad438ee43 --- /dev/null +++ b/lib/vector-core/src/config/metrics_expiration.rs @@ -0,0 +1,196 @@ +use vector_config::configurable_component; + +/// Per metric set expiration options. +#[configurable_component] +#[derive(Clone, Debug, PartialEq, Default)] +pub struct PerMetricSetExpiration { + /// Metric name to apply this expiration to. Ignores metric name if not defined. + #[serde(default, skip_serializing_if = "crate::serde::is_default")] + pub name: Option, + /// Labels to apply this expiration to. Ignores labels if not defined. + #[serde(default, skip_serializing_if = "crate::serde::is_default")] + #[configurable(metadata( + docs::enum_tag_field = "type", + docs::enum_tagging = "internal", + docs::enum_tag_description = "Metric label matcher type." + ))] + pub labels: Option, + /// The amount of time, in seconds, that internal metrics will persist after having not been + /// updated before they expire and are removed. + /// + /// Set this to a value larger than your `internal_metrics` scrape interval (default 5 minutes) + /// so that metrics live long enough to be emitted and captured. + #[configurable(metadata(docs::examples = 60.0))] + pub expire_secs: f64, +} + +/// Configuration for metric name matcher. +#[configurable_component] +#[derive(Clone, Debug, PartialEq)] +#[serde(tag = "type", rename_all = "snake_case")] +#[configurable(metadata(docs::enum_tag_description = "Metric name matcher type."))] +pub enum MetricNameMatcherConfig { + /// Only considers exact name matches. + Exact { + /// The exact metric name. + value: String, + }, + /// Compares metric name to the provided pattern. + Regex { + /// Pattern to compare to. + pattern: String, + }, +} + +/// Configuration for metric labels matcher. +#[configurable_component] +#[derive(Clone, Debug, PartialEq)] +#[serde(tag = "type", rename_all = "snake_case")] +#[configurable(metadata(docs::enum_tag_description = "Metric label matcher type."))] +pub enum MetricLabelMatcher { + /// Looks for an exact match of one label key value pair. + Exact { + /// Metric key to look for. + key: String, + /// The exact metric label value. + value: String, + }, + /// Compares label value with given key to the provided pattern. + Regex { + /// Metric key to look for. + key: String, + /// Pattern to compare metric label value to. + value_pattern: String, + }, +} + +/// Configuration for metric labels matcher group. +#[configurable_component] +#[derive(Clone, Debug, PartialEq)] +#[serde(tag = "type", rename_all = "snake_case")] +#[configurable(metadata(docs::enum_tag_description = "Metric label group matcher type."))] +pub enum MetricLabelMatcherConfig { + /// Checks that any of the provided matchers can be applied to given metric. + Any { + /// List of matchers to check. + matchers: Vec, + }, + /// Checks that all of the provided matchers can be applied to given metric. + All { + /// List of matchers to check. + matchers: Vec, + }, +} + +/// Tests to confirm complex examples configuration +#[cfg(test)] +mod tests { + use vrl::prelude::indoc; + + use super::*; + + #[test] + fn just_expiration_config() { + // This configuration should maybe be treated as invalid - because it turns into a global + // configuration, matching every metric + let config = serde_yaml::from_str::(indoc! {r" + expire_secs: 10.0 + "}) + .unwrap(); + + assert!(config.name.is_none()); + assert!(config.labels.is_none()); + assert_eq!(10.0, config.expire_secs); + } + + #[test] + fn simple_name_config() { + let config = serde_yaml::from_str::(indoc! {r#" + name: + type: "exact" + value: "test_metric" + expire_secs: 1.0 + "#}) + .unwrap(); + + if let Some(MetricNameMatcherConfig::Exact { value }) = config.name { + assert_eq!("test_metric", value); + } else { + panic!("Expected exact name matcher"); + } + assert!(config.labels.is_none()); + assert_eq!(1.0, config.expire_secs); + } + + #[test] + fn simple_labels_config() { + let config = serde_yaml::from_str::(indoc! {r#" + labels: + type: "all" + matchers : + - type: "exact" + key: "test_metric_label" + value: "test_value" + expire_secs: 1.0 + "#}) + .unwrap(); + + if let Some(MetricLabelMatcherConfig::All { matchers }) = config.labels { + if let MetricLabelMatcher::Exact { key, value } = &matchers[0] { + assert_eq!("test_metric_label", key); + assert_eq!("test_value", value); + } else { + panic!("Expected exact metric matcher"); + } + } else { + panic!("Expected all matcher"); + } + assert!(config.name.is_none()); + assert_eq!(1.0, config.expire_secs); + } + + #[test] + fn complex_config() { + let config = serde_yaml::from_str::(indoc! {r#" + name: + type: "regex" + pattern: "test_metric.*" + labels: + type: "all" + matchers: + - type: "exact" + key: "component_kind" + value: "sink" + - type: "exact" + key: "component_kind" + value: "source" + expire_secs: 1.0 + "#}) + .unwrap(); + + if let Some(MetricNameMatcherConfig::Regex { ref pattern }) = config.name { + assert_eq!("test_metric.*", pattern); + } else { + panic!("Expected regex name matcher"); + } + + let Some(MetricLabelMatcherConfig::All { + matchers: all_matchers, + }) = config.labels + else { + panic!("Expected all label matcher"); + }; + assert_eq!(2, all_matchers.len()); + + let MetricLabelMatcher::Exact { key, value } = &all_matchers[0] else { + panic!("Expected first label matcher to be exact matcher"); + }; + assert_eq!("component_kind", key); + assert_eq!("sink", value); + let MetricLabelMatcher::Exact { key, value } = &all_matchers[1] else { + panic!("Expected second label matcher to be exact matcher"); + }; + assert_eq!("component_kind", key); + assert_eq!("source", value); + } +} diff --git a/lib/vector-core/src/config/mod.rs b/lib/vector-core/src/config/mod.rs index d6ecc26bf5..935f02cf22 100644 --- a/lib/vector-core/src/config/mod.rs +++ b/lib/vector-core/src/config/mod.rs @@ -7,17 +7,18 @@ use chrono::{DateTime, Utc}; mod global_options; mod log_schema; +pub(crate) mod metrics_expiration; pub mod output_id; pub mod proxy; mod telemetry; use crate::event::LogEvent; -pub use global_options::GlobalOptions; -pub use log_schema::{init_log_schema, log_schema, LogSchema}; -use lookup::{lookup_v2::ValuePath, path, PathPrefix}; +pub use global_options::{GlobalOptions, WildcardMatching}; +pub use log_schema::{LogSchema, init_log_schema, log_schema}; +use lookup::{PathPrefix, lookup_v2::ValuePath, path}; pub use output_id::OutputId; use serde::{Deserialize, Serialize}; -pub use telemetry::{init_telemetry, telemetry, Tags, Telemetry}; +pub use telemetry::{Tags, Telemetry, init_telemetry, telemetry}; pub use vector_common::config::ComponentKey; use vector_config::configurable_component; use vrl::value::Value; @@ -303,7 +304,7 @@ Enabling or disabling acknowledgements at the source level has **no effect** on See [End-to-end Acknowledgements][e2e_acks] for more information on how event acknowledgement is handled. [global_acks]: https://vector.dev/docs/reference/configuration/global-options/#acknowledgements -[e2e_acks]: https://vector.dev/docs/about/under-the-hood/architecture/end-to-end-acknowledgements/" +[e2e_acks]: https://vector.dev/docs/architecture/end-to-end-acknowledgements/" )] #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] pub struct SourceAcknowledgementsConfig { @@ -351,15 +352,15 @@ impl From for AcknowledgementsConfig { #[configurable( description = "See [End-to-end Acknowledgements][e2e_acks] for more information on how event acknowledgement is handled. -[e2e_acks]: https://vector.dev/docs/about/under-the-hood/architecture/end-to-end-acknowledgements/" +[e2e_acks]: https://vector.dev/docs/architecture/end-to-end-acknowledgements/" )] #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] pub struct AcknowledgementsConfig { /// Whether or not end-to-end acknowledgements are enabled. /// - /// When enabled for a sink, any source connected to that sink where the source supports - /// end-to-end acknowledgements as well, waits for events to be acknowledged by **all - /// connected** sinks before acknowledging them at the source. + /// When enabled for a sink, any source that supports end-to-end + /// acknowledgements that is connected to that sink waits for events + /// to be acknowledged by **all connected sinks** before acknowledging them at the source. /// /// Enabling or disabling acknowledgements at the sink level takes precedence over any global /// [`acknowledgements`][global_acks] configuration. @@ -572,7 +573,7 @@ mod test { use super::*; use crate::event::LogEvent; use chrono::Utc; - use lookup::{event_path, owned_value_path, OwnedTargetPath}; + use lookup::{OwnedTargetPath, event_path, owned_value_path}; use vector_common::btreemap; use vrl::value::Kind; diff --git a/lib/vector-core/src/config/proxy.rs b/lib/vector-core/src/config/proxy.rs index 6a15c3d64f..1f175784f8 100644 --- a/lib/vector-core/src/config/proxy.rs +++ b/lib/vector-core/src/config/proxy.rs @@ -25,9 +25,9 @@ impl NoProxyInterceptor { if scheme.is_some() && scheme != Some(expected_scheme) { return false; } - let matches = host.map_or(false, |host| { + let matches = host.is_some_and(|host| { self.0.matches(host) - || port.map_or(false, |port| { + || port.is_some_and(|port| { let url = format!("{host}:{port}"); self.0.matches(&url) }) @@ -161,17 +161,14 @@ impl ProxyConfig { .map(|url| { url.parse().map(|parsed| { let mut proxy = Proxy::new(self.interceptor().intercept(proxy_scheme), parsed); - if let Ok(authority) = Url::parse(url) { - if let Some(password) = authority.password() { - let decoded_user = urlencoding::decode(authority.username()) - .expect("username must be valid UTF-8."); - let decoded_pw = urlencoding::decode(password) - .expect("Password must be valid UTF-8."); - proxy.set_authorization(Authorization::basic( - &decoded_user, - &decoded_pw, - )); - } + if let Ok(authority) = Url::parse(url) + && let Some(password) = authority.password() + { + let decoded_user = urlencoding::decode(authority.username()) + .expect("username must be valid UTF-8."); + let decoded_pw = + urlencoding::decode(password).expect("Password must be valid UTF-8."); + proxy.set_authorization(Authorization::basic(&decoded_user, &decoded_pw)); } proxy }) @@ -207,11 +204,11 @@ impl ProxyConfig { #[cfg(test)] mod tests { - use base64::prelude::{Engine as _, BASE64_STANDARD}; + use base64::prelude::{BASE64_STANDARD, Engine as _}; use env_test_util::TempEnvVar; use http::{ - header::{AUTHORIZATION, PROXY_AUTHORIZATION}, HeaderName, HeaderValue, Uri, + header::{AUTHORIZATION, PROXY_AUTHORIZATION}, }; use proptest::prelude::*; diff --git a/lib/vector-core/src/event/array.rs b/lib/vector-core/src/event/array.rs index 0519920ace..c61677899a 100644 --- a/lib/vector-core/src/event/array.rs +++ b/lib/vector-core/src/event/array.rs @@ -4,7 +4,7 @@ use std::{iter, slice, sync::Arc, vec}; -use futures::{stream, Stream}; +use futures::{Stream, stream}; #[cfg(test)] use quickcheck::{Arbitrary, Gen}; use vector_buffers::EventCount; @@ -173,7 +173,7 @@ impl EventArray { } /// Iterate over references to this array's events. - pub fn iter_events(&self) -> impl Iterator { + pub fn iter_events(&self) -> impl Iterator> { match self { Self::Logs(array) => EventArrayIter::Logs(array.iter()), Self::Metrics(array) => EventArrayIter::Metrics(array.iter()), @@ -182,7 +182,7 @@ impl EventArray { } /// Iterate over mutable references to this array's events. - pub fn iter_events_mut(&mut self) -> impl Iterator { + pub fn iter_events_mut(&mut self) -> impl Iterator> { match self { Self::Logs(array) => EventArrayIterMut::Logs(array.iter_mut()), Self::Metrics(array) => EventArrayIterMut::Metrics(array.iter_mut()), diff --git a/lib/vector-core/src/event/discriminant.rs b/lib/vector-core/src/event/discriminant.rs index 148dc0c569..4520d68753 100644 --- a/lib/vector-core/src/event/discriminant.rs +++ b/lib/vector-core/src/event/discriminant.rs @@ -1,3 +1,4 @@ +use std::fmt; use std::hash::{Hash, Hasher}; use super::{LogEvent, ObjectMap, Value}; @@ -76,7 +77,7 @@ fn f64_eq(this: f64, other: f64) -> bool { } if this != other { return false; - }; + } if (this.is_sign_positive() && other.is_sign_negative()) || (this.is_sign_negative() && other.is_sign_positive()) { @@ -158,9 +159,25 @@ fn hash_null(hasher: &mut H) { hasher.write_u8(0); } +impl fmt::Display for Discriminant { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { + for (i, value) in self.values.iter().enumerate() { + if i != 0 { + write!(fmt, "-")?; + } + if let Some(value) = value { + value.fmt(fmt)?; + } else { + fmt.write_str("none")?; + } + } + Ok(()) + } +} + #[cfg(test)] mod tests { - use std::collections::{hash_map::DefaultHasher, HashMap}; + use std::collections::{HashMap, hash_map::DefaultHasher}; use super::*; use crate::event::LogEvent; @@ -365,4 +382,21 @@ mod tests { assert_eq!(process_event(event_stream_2), 2); assert_eq!(process_event(event_stream_3), 2); } + + #[test] + fn test_display() { + let mut event = LogEvent::default(); + event.insert("hostname", "localhost"); + event.insert("container_id", 1); + + let discriminant = Discriminant::from_log_event( + &event, + &["hostname".to_string(), "container_id".to_string()], + ); + assert_eq!(format!("{discriminant}"), "\"localhost\"-1"); + + let discriminant = + Discriminant::from_log_event(&event, &["hostname".to_string(), "service".to_string()]); + assert_eq!(format!("{discriminant}"), "\"localhost\"-none"); + } } diff --git a/lib/vector-core/src/event/estimated_json_encoded_size_of.rs b/lib/vector-core/src/event/estimated_json_encoded_size_of.rs index 2e848e7c84..58db414bad 100644 --- a/lib/vector-core/src/event/estimated_json_encoded_size_of.rs +++ b/lib/vector-core/src/event/estimated_json_encoded_size_of.rs @@ -114,11 +114,7 @@ impl EstimatedJsonEncodedSizeOf for Bytes { impl EstimatedJsonEncodedSizeOf for bool { fn estimated_json_encoded_size_of(&self) -> JsonSize { - if *self { - TRUE_SIZE - } else { - FALSE_SIZE - } + if *self { TRUE_SIZE } else { FALSE_SIZE } } } diff --git a/lib/vector-core/src/event/log_event.rs b/lib/vector-core/src/event/log_event.rs index 3061767409..91ff892283 100644 --- a/lib/vector-core/src/event/log_event.rs +++ b/lib/vector-core/src/event/log_event.rs @@ -13,28 +13,29 @@ use bytes::Bytes; use chrono::Utc; use crossbeam_utils::atomic::AtomicCell; -use lookup::{lookup_v2::TargetPath, metadata_path, path, PathPrefix}; +use lookup::{PathPrefix, lookup_v2::TargetPath, metadata_path, path}; use serde::{Deserialize, Serialize, Serializer}; use vector_common::{ + EventDataEq, byte_size_of::ByteSizeOf, internal_event::{OptionalTag, TaggedEventsSent}, json_size::{JsonSize, NonZeroJsonSize}, request_metadata::GetEventCountTags, - EventDataEq, }; -use vrl::path::{parse_target_path, OwnedTargetPath, PathParseError}; +use vrl::path::{OwnedTargetPath, PathParseError, parse_target_path}; use vrl::{event_path, owned_value_path}; use super::{ + EventFinalizers, Finalizable, KeyString, ObjectMap, Value, estimated_json_encoded_size_of::EstimatedJsonEncodedSizeOf, finalization::{BatchNotifier, EventFinalizer}, metadata::EventMetadata, - util, EventFinalizers, Finalizable, KeyString, ObjectMap, Value, + util, }; use crate::config::LogNamespace; use crate::config::{log_schema, telemetry}; -use crate::event::util::log::{all_fields, all_metadata_fields}; use crate::event::MaybeAsLogMut; +use crate::event::util::log::{all_fields, all_metadata_fields}; static VECTOR_SOURCE_TYPE_PATH: LazyLock> = LazyLock::new(|| { Some(OwnedTargetPath::metadata(owned_value_path!( @@ -625,7 +626,7 @@ impl EventDataEq for LogEvent { #[cfg(any(test, feature = "test"))] mod test_utils { - use super::{log_schema, Bytes, LogEvent, Utc}; + use super::{Bytes, LogEvent, Utc, log_schema}; // these rely on the global log schema, which is no longer supported when using the // "LogNamespace::Vector" namespace. @@ -1032,9 +1033,8 @@ mod test { fn json_value_to_vector_log_event_to_json_value() { const FIXTURE_ROOT: &str = "tests/data/fixtures/log_event"; - std::fs::read_dir(FIXTURE_ROOT) - .unwrap() - .for_each(|fixture_file| match fixture_file { + for fixture_file in std::fs::read_dir(FIXTURE_ROOT).unwrap() { + match fixture_file { Ok(fixture_file) => { let path = fixture_file.path(); tracing::trace!(?path, "Opening."); @@ -1046,7 +1046,8 @@ mod test { assert_eq!(serde_value, serde_value_again); } _ => panic!("This test should never read Err'ing test fixtures."), - }); + } + } } fn assert_merge_value( diff --git a/lib/vector-core/src/event/lua/event.rs b/lib/vector-core/src/event/lua/event.rs index c9f268377f..e1d194498b 100644 --- a/lib/vector-core/src/event/lua/event.rs +++ b/lib/vector-core/src/event/lua/event.rs @@ -27,7 +27,7 @@ impl IntoLua for LuaEvent { from: String::from("Event"), to: "table", message: Some("Trace are not supported".to_string()), - }) + }); } } Ok(LuaValue::Table(table)) @@ -67,8 +67,8 @@ impl FromLua for Event { mod test { use super::*; use crate::event::{ - metric::{MetricKind, MetricValue}, Metric, Value, + metric::{MetricKind, MetricValue}, }; fn assert_event(event: Event, assertions: Vec<&'static str>) { diff --git a/lib/vector-core/src/event/lua/metric.rs b/lib/vector-core/src/event/lua/metric.rs index 8bf45ba4c1..9f933997f0 100644 --- a/lib/vector-core/src/event/lua/metric.rs +++ b/lib/vector-core/src/event/lua/metric.rs @@ -3,9 +3,9 @@ use std::collections::BTreeMap; use mlua::prelude::*; use super::super::{ + Metric, MetricKind, MetricValue, StatisticKind, metric::TagValue, metric::{self, MetricSketch, MetricTags, TagValueSet}, - Metric, MetricKind, MetricValue, StatisticKind, }; use super::util::{table_to_timestamp, timestamp_to_table}; use crate::metrics::AgentDDSketch; @@ -238,7 +238,7 @@ impl FromLua for Metric { from: other.type_name(), to: String::from("Metric"), message: Some("Metric should be a Lua table".to_string()), - }) + }); } }; @@ -323,7 +323,7 @@ impl FromLua for Metric { from: value.type_name(), to: String::from("Metric"), message: Some(format!("Invalid sketch type '{x}' given")), - }) + }); } } } else { @@ -344,7 +344,7 @@ impl FromLua for Metric { #[cfg(test)] mod test { - use chrono::{offset::TimeZone, Timelike, Utc}; + use chrono::{Timelike, Utc, offset::TimeZone}; use vector_common::assert_event_data_eq; use super::*; diff --git a/lib/vector-core/src/event/metadata.rs b/lib/vector-core/src/event/metadata.rs index c1dd579023..cd9cecbe55 100644 --- a/lib/vector-core/src/event/metadata.rs +++ b/lib/vector-core/src/event/metadata.rs @@ -6,7 +6,7 @@ use derivative::Derivative; use lookup::OwnedTargetPath; use serde::{Deserialize, Serialize}; use uuid::Uuid; -use vector_common::{byte_size_of::ByteSizeOf, config::ComponentKey, EventDataEq}; +use vector_common::{EventDataEq, byte_size_of::ByteSizeOf, config::ComponentKey}; use vrl::{ compiler::SecretTarget, value::{KeyString, Kind, Value}, @@ -84,11 +84,11 @@ pub(super) struct Inner { #[derive(Clone, Default, Debug, Deserialize, PartialEq, Serialize)] pub struct DatadogMetricOriginMetadata { /// `OriginProduct` - origin_product: Option, + product: Option, /// `OriginCategory` - origin_category: Option, + category: Option, /// `OriginService` - origin_service: Option, + service: Option, } impl DatadogMetricOriginMetadata { @@ -100,25 +100,25 @@ impl DatadogMetricOriginMetadata { #[must_use] pub fn new(product: Option, category: Option, service: Option) -> Self { Self { - origin_product: product, - origin_category: category, - origin_service: service, + product, + category, + service, } } /// Returns a reference to the `OriginProduct`. pub fn product(&self) -> Option { - self.origin_product + self.product } /// Returns a reference to the `OriginCategory`. pub fn category(&self) -> Option { - self.origin_category + self.category } /// Returns a reference to the `OriginService`. pub fn service(&self) -> Option { - self.origin_service + self.service } } @@ -356,7 +356,7 @@ impl EventMetadata { inner.source_event_id = Some(uuid2); } _ => {} // Keep the existing value. - }; + } } /// Update the finalizer(s) status. diff --git a/lib/vector-core/src/event/metric/arbitrary.rs b/lib/vector-core/src/event/metric/arbitrary.rs index a3c0f723ae..a03990787f 100644 --- a/lib/vector-core/src/event/metric/arbitrary.rs +++ b/lib/vector-core/src/event/metric/arbitrary.rs @@ -5,8 +5,8 @@ use proptest::prelude::*; use crate::metrics::AgentDDSketch; use super::{ - samples_to_buckets, Bucket, MetricSketch, MetricTags, MetricValue, Quantile, Sample, - StatisticKind, TagValue, TagValueSet, + Bucket, MetricSketch, MetricTags, MetricValue, Quantile, Sample, StatisticKind, TagValue, + TagValueSet, samples_to_buckets, }; fn realistic_float() -> proptest::num::f64::Any { diff --git a/lib/vector-core/src/event/metric/data.rs b/lib/vector-core/src/event/metric/data.rs index 44823d0b81..adef5b3876 100644 --- a/lib/vector-core/src/event/metric/data.rs +++ b/lib/vector-core/src/event/metric/data.rs @@ -58,11 +58,14 @@ impl MetricData { /// Consumes this metric, returning it as an absolute metric. /// - /// If the metric was already absolute, nothing is changed. + /// The `interval_ms` is set to `None`. If the metric was already absolute, nothing else is changed. #[must_use] pub fn into_absolute(self) -> Self { Self { - time: self.time, + time: MetricTime { + timestamp: self.time.timestamp, + interval_ms: None, + }, kind: MetricKind::Absolute, value: self.value, } diff --git a/lib/vector-core/src/event/metric/mod.rs b/lib/vector-core/src/event/metric/mod.rs index 5bcb751ecc..808cc4fba9 100644 --- a/lib/vector-core/src/event/metric/mod.rs +++ b/lib/vector-core/src/event/metric/mod.rs @@ -12,17 +12,17 @@ use std::{ use chrono::{DateTime, Utc}; use vector_common::{ + EventDataEq, byte_size_of::ByteSizeOf, internal_event::{OptionalTag, TaggedEventsSent}, json_size::JsonSize, request_metadata::GetEventCountTags, - EventDataEq, }; use vector_config::configurable_component; use super::{ - estimated_json_encoded_size_of::EstimatedJsonEncodedSizeOf, BatchNotifier, EventFinalizer, - EventFinalizers, EventMetadata, Finalizable, + BatchNotifier, EventFinalizer, EventFinalizers, EventMetadata, Finalizable, + estimated_json_encoded_size_of::EstimatedJsonEncodedSizeOf, }; use crate::config::telemetry; @@ -667,7 +667,7 @@ pub fn samples_to_buckets(samples: &[Sample], buckets: &[f64]) -> (Vec, mod test { use std::collections::BTreeSet; - use chrono::{offset::TimeZone, DateTime, Timelike, Utc}; + use chrono::{DateTime, Timelike, Utc, offset::TimeZone}; use similar_asserts::assert_eq; use super::*; diff --git a/lib/vector-core/src/event/metric/series.rs b/lib/vector-core/src/event/metric/series.rs index 4c2da4a3c8..dfc5ba8371 100644 --- a/lib/vector-core/src/event/metric/series.rs +++ b/lib/vector-core/src/event/metric/series.rs @@ -3,7 +3,7 @@ use core::fmt; use vector_common::byte_size_of::ByteSizeOf; use vector_config::configurable_component; -use super::{write_list, write_word, MetricTags, TagValue}; +use super::{MetricTags, TagValue, write_list, write_word}; /// Metrics series. #[configurable_component] diff --git a/lib/vector-core/src/event/metric/tags.rs b/lib/vector-core/src/event/metric/tags.rs index 8d3394a2f3..c38c12ba19 100644 --- a/lib/vector-core/src/event/metric/tags.rs +++ b/lib/vector-core/src/event/metric/tags.rs @@ -2,15 +2,15 @@ use std::borrow::Borrow; use std::borrow::Cow; -use std::collections::{hash_map::DefaultHasher, BTreeMap}; +use std::collections::{BTreeMap, hash_map::DefaultHasher}; use std::fmt::Display; use std::hash::{Hash, Hasher}; use std::{cmp::Ordering, mem}; use indexmap::IndexSet; -use serde::{ser::SerializeSeq, Deserialize, Deserializer, Serialize, Serializer}; +use serde::{Deserialize, Deserializer, Serialize, Serializer, ser::SerializeSeq}; use vector_common::byte_size_of::ByteSizeOf; -use vector_config::{configurable_component, Configurable}; +use vector_config::{Configurable, configurable_component}; /// A single tag value, either a bare tag or a value. #[derive(Clone, Configurable, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)] diff --git a/lib/vector-core/src/event/metric/value.rs b/lib/vector-core/src/event/metric/value.rs index 409aec316e..d46cd89a14 100644 --- a/lib/vector-core/src/event/metric/value.rs +++ b/lib/vector-core/src/event/metric/value.rs @@ -1,7 +1,7 @@ use core::fmt; use std::collections::BTreeSet; -use serde::{de, Deserialize, Deserializer, Serialize, Serializer}; +use serde::{Deserialize, Deserializer, Serialize, Serializer, de}; use vector_common::byte_size_of::ByteSizeOf; use vector_config::configurable_component; @@ -212,18 +212,18 @@ impl MetricValue { #[must_use] pub fn add(&mut self, other: &Self) -> bool { match (self, other) { - (Self::Counter { ref mut value }, Self::Counter { value: value2 }) - | (Self::Gauge { ref mut value }, Self::Gauge { value: value2 }) => { + (Self::Counter { value }, Self::Counter { value: value2 }) + | (Self::Gauge { value }, Self::Gauge { value: value2 }) => { *value += value2; true } - (Self::Set { ref mut values }, Self::Set { values: values2 }) => { + (Self::Set { values }, Self::Set { values: values2 }) => { values.extend(values2.iter().map(Into::into)); true } ( Self::Distribution { - ref mut samples, + samples, statistic: statistic_a, }, Self::Distribution { @@ -236,9 +236,9 @@ impl MetricValue { } ( Self::AggregatedHistogram { - ref mut buckets, - ref mut count, - ref mut sum, + buckets, + count, + sum, }, Self::AggregatedHistogram { buckets: buckets2, @@ -282,17 +282,15 @@ impl MetricValue { // process restart, etc. Thus, being able to generate negative deltas would violate // that. Whether a counter is reset to 0, or if it incorrectly warps to a previous // value, it doesn't matter: we're going to reinitialize it. - (Self::Counter { ref mut value }, Self::Counter { value: value2 }) - if *value >= *value2 => - { + (Self::Counter { value }, Self::Counter { value: value2 }) if *value >= *value2 => { *value -= value2; true } - (Self::Gauge { ref mut value }, Self::Gauge { value: value2 }) => { + (Self::Gauge { value }, Self::Gauge { value: value2 }) => { *value -= value2; true } - (Self::Set { ref mut values }, Self::Set { values: values2 }) => { + (Self::Set { values }, Self::Set { values: values2 }) => { for item in values2 { values.remove(item); } @@ -300,7 +298,7 @@ impl MetricValue { } ( Self::Distribution { - ref mut samples, + samples, statistic: statistic_a, }, Self::Distribution { @@ -333,9 +331,9 @@ impl MetricValue { // make that decision, and simply force the metric to be reinitialized. ( Self::AggregatedHistogram { - ref mut buckets, - ref mut count, - ref mut sum, + buckets, + count, + sum, }, Self::AggregatedHistogram { buckets: buckets2, @@ -538,7 +536,7 @@ pub enum StatisticKind { pub enum MetricSketch { /// [DDSketch][ddsketch] implementation based on the [Datadog Agent][ddagent]. /// - /// While DDSketch has open-source implementations based on the white paper, the version used in + /// While `DDSketch` has open-source implementations based on the white paper, the version used in /// the Datadog Agent itself is subtly different. This version is suitable for sending directly /// to Datadog's sketch ingest endpoint. /// diff --git a/lib/vector-core/src/event/mod.rs b/lib/vector-core/src/event/mod.rs index 27c779212e..bdf971b7c0 100644 --- a/lib/vector-core/src/event/mod.rs +++ b/lib/vector-core/src/event/mod.rs @@ -1,6 +1,6 @@ use std::{convert::TryInto, fmt::Debug, sync::Arc}; -pub use array::{into_event_stream, EventArray, EventContainer, LogArray, MetricArray, TraceArray}; +pub use array::{EventArray, EventContainer, LogArray, MetricArray, TraceArray, into_event_stream}; pub use estimated_json_encoded_size_of::EstimatedJsonEncodedSizeOf; pub use finalization::{ BatchNotifier, BatchStatus, BatchStatusReceiver, EventFinalizer, EventFinalizers, EventStatus, @@ -14,8 +14,8 @@ use serde::{Deserialize, Serialize}; pub use trace::TraceEvent; use vector_buffers::EventCount; use vector_common::{ - byte_size_of::ByteSizeOf, config::ComponentKey, finalization, internal_event::TaggedEventsSent, - json_size::JsonSize, request_metadata::GetEventCountTags, EventDataEq, + EventDataEq, byte_size_of::ByteSizeOf, config::ComponentKey, finalization, + internal_event::TaggedEventsSent, json_size::JsonSize, request_metadata::GetEventCountTags, }; pub use vrl::value::{KeyString, ObjectMap, Value}; #[cfg(feature = "vrl")] diff --git a/lib/vector-core/src/event/proto.rs b/lib/vector-core/src/event/proto.rs index f1c926b49e..2a2c6700be 100644 --- a/lib/vector-core/src/event/proto.rs +++ b/lib/vector-core/src/event/proto.rs @@ -17,7 +17,7 @@ pub use metric::Value as MetricValue; pub use proto_event::*; use vrl::value::{ObjectMap, Value as VrlValue}; -use super::{array, metric::MetricSketch, EventMetadata}; +use super::{EventMetadata, array, metric::MetricSketch}; impl event_array::Events { // We can't use the standard `From` traits here because the actual diff --git a/lib/vector-core/src/event/ser.rs b/lib/vector-core/src/event/ser.rs index be41910466..a456ccf693 100644 --- a/lib/vector-core/src/event/ser.rs +++ b/lib/vector-core/src/event/ser.rs @@ -1,10 +1,10 @@ use bytes::{Buf, BufMut}; -use enumflags2::{bitflags, BitFlags, FromBitsError}; +use enumflags2::{BitFlags, FromBitsError, bitflags}; use prost::Message; use snafu::Snafu; use vector_buffers::encoding::{AsMetadata, Encodable}; -use super::{proto, Event, EventArray}; +use super::{Event, EventArray, proto}; #[derive(Debug, Snafu)] pub enum EncodeError { diff --git a/lib/vector-core/src/event/test/common.rs b/lib/vector-core/src/event/test/common.rs index 431a4f3eee..4990941b25 100644 --- a/lib/vector-core/src/event/test/common.rs +++ b/lib/vector-core/src/event/test/common.rs @@ -1,15 +1,15 @@ use std::{collections::BTreeSet, iter}; use chrono::{DateTime, Utc}; -use quickcheck::{empty_shrinker, Arbitrary, Gen}; +use quickcheck::{Arbitrary, Gen, empty_shrinker}; use vrl::value::{ObjectMap, Value}; use super::super::{ + Event, EventMetadata, LogEvent, Metric, MetricKind, MetricValue, StatisticKind, TraceEvent, metric::{ Bucket, MetricData, MetricName, MetricSeries, MetricSketch, MetricTags, MetricTime, Quantile, Sample, }, - Event, EventMetadata, LogEvent, Metric, MetricKind, MetricValue, StatisticKind, TraceEvent, }; use crate::metrics::AgentDDSketch; @@ -76,8 +76,8 @@ impl Arbitrary for Event { impl Arbitrary for LogEvent { fn arbitrary(g: &mut Gen) -> Self { - let mut gen = Gen::new(MAX_MAP_SIZE); - let map: ObjectMap = ObjectMap::arbitrary(&mut gen); + let mut generator = Gen::new(MAX_MAP_SIZE); + let map: ObjectMap = ObjectMap::arbitrary(&mut generator); let metadata: EventMetadata = EventMetadata::arbitrary(g); LogEvent::from_map(map, metadata) } @@ -201,10 +201,12 @@ impl Arbitrary for MetricValue { // We're working around quickcheck's limitations here, and // should really migrate the tests in question to use proptest let num_samples = u8::arbitrary(g); - let samples = std::iter::repeat_with(|| loop { - let f = f64::arbitrary(g); - if f.is_normal() { - return f; + let samples = std::iter::repeat_with(|| { + loop { + let f = f64::arbitrary(g); + if f.is_normal() { + return f; + } } }) .take(num_samples as usize) @@ -474,11 +476,7 @@ impl Arbitrary for MetricSeries { let value = String::from(Name::arbitrary(g)); map.replace(key, value); } - if map.is_empty() { - None - } else { - Some(map) - } + if map.is_empty() { None } else { Some(map) } } else { None }; diff --git a/lib/vector-core/src/event/test/serialization.rs b/lib/vector-core/src/event/test/serialization.rs index aaab559da3..75304e3960 100644 --- a/lib/vector-core/src/event/test/serialization.rs +++ b/lib/vector-core/src/event/test/serialization.rs @@ -39,7 +39,7 @@ fn serde_eventarray_no_size_loss() { // Ser/De the EventArray type through EncodeBytes -> DecodeBytes #[test] #[allow(clippy::neg_cmp_op_on_partial_ord)] // satisfying clippy leads to less - // clear expression +// clear expression fn back_and_forth_through_bytes() { fn inner(events: EventArray) -> TestResult { let expected = events.clone(); diff --git a/lib/vector-core/src/event/test/size_of.rs b/lib/vector-core/src/event/test/size_of.rs index e3fdc463fc..6263bc8b21 100644 --- a/lib/vector-core/src/event/test/size_of.rs +++ b/lib/vector-core/src/event/test/size_of.rs @@ -1,6 +1,6 @@ use std::mem; -use lookup::{path, PathPrefix}; +use lookup::{PathPrefix, path}; use quickcheck::{Arbitrary, Gen, QuickCheck, TestResult}; use vector_common::byte_size_of::ByteSizeOf; diff --git a/lib/vector-core/src/event/trace.rs b/lib/vector-core/src/event/trace.rs index 3956a95323..276bac7566 100644 --- a/lib/vector-core/src/event/trace.rs +++ b/lib/vector-core/src/event/trace.rs @@ -4,8 +4,8 @@ use lookup::lookup_v2::TargetPath; use serde::{Deserialize, Serialize}; use vector_buffers::EventCount; use vector_common::{ - byte_size_of::ByteSizeOf, internal_event::TaggedEventsSent, json_size::JsonSize, - request_metadata::GetEventCountTags, EventDataEq, + EventDataEq, byte_size_of::ByteSizeOf, internal_event::TaggedEventsSent, json_size::JsonSize, + request_metadata::GetEventCountTags, }; use vrl::path::PathParseError; @@ -118,6 +118,13 @@ impl TraceEvent { } } +impl From for TraceEvent { + fn from(value: Value) -> Self { + let log_event = LogEvent::from(value); + Self(log_event) + } +} + impl From for TraceEvent { fn from(log: LogEvent) -> Self { Self(log) diff --git a/lib/vector-core/src/event/util/log/all_fields.rs b/lib/vector-core/src/event/util/log/all_fields.rs index 10afadfede..2e1511c124 100644 --- a/lib/vector-core/src/event/util/log/all_fields.rs +++ b/lib/vector-core/src/event/util/log/all_fields.rs @@ -11,30 +11,30 @@ static IS_VALID_PATH_SEGMENT: LazyLock = /// Iterates over all paths in form `a.b[0].c[1]` in alphabetical order /// and their corresponding values. -pub fn all_fields(fields: &ObjectMap) -> FieldsIter { +pub fn all_fields(fields: &ObjectMap) -> FieldsIter<'_> { FieldsIter::new(None, fields, true) } /// Iterates over all paths in form `a.b[0].c[1]` in alphabetical order and their corresponding /// values. Field names containing meta-characters are not quoted. -pub fn all_fields_unquoted(fields: &ObjectMap) -> FieldsIter { +pub fn all_fields_unquoted(fields: &ObjectMap) -> FieldsIter<'_> { FieldsIter::new(None, fields, false) } /// Same functionality as `all_fields` but it prepends a character that denotes the /// path type. -pub fn all_metadata_fields(fields: &ObjectMap) -> FieldsIter { +pub fn all_metadata_fields(fields: &ObjectMap) -> FieldsIter<'_> { FieldsIter::new(Some(PathPrefix::Metadata), fields, true) } /// An iterator with a single "message" element -pub fn all_fields_non_object_root(value: &Value) -> FieldsIter { +pub fn all_fields_non_object_root(value: &Value) -> FieldsIter<'_> { FieldsIter::non_object(value) } /// An iterator similar to `all_fields`, but instead of visiting each array element individually, /// it treats the entire array as a single value. -pub fn all_fields_skip_array_elements(fields: &ObjectMap) -> FieldsIter { +pub fn all_fields_skip_array_elements(fields: &ObjectMap) -> FieldsIter<'_> { FieldsIter::new_with_skip_array_elements(fields) } @@ -144,7 +144,7 @@ impl<'a> FieldsIter<'a> { None => break res.into(), Some(PathComponent::Key(key)) => { if self.quote_invalid_fields && !IS_VALID_PATH_SEGMENT.is_match(key) { - res.push_str(&format!("\"{key}\"")); + write!(res, "\"{key}\"").expect("write to String never fails"); } else { res.push_str(key); } @@ -191,7 +191,7 @@ impl<'a> Iterator for FieldsIter<'a> { *visited = true; break result; } - }; + } } } } diff --git a/lib/vector-core/src/event/vrl_target.rs b/lib/vector-core/src/event/vrl_target.rs index 7ef2aec3b8..d713c9c782 100644 --- a/lib/vector-core/src/event/vrl_target.rs +++ b/lib/vector-core/src/event/vrl_target.rs @@ -1,4 +1,5 @@ use std::borrow::Cow; +use std::num::{NonZero, TryFromIntError}; use std::{collections::BTreeMap, convert::TryFrom, marker::PhantomData}; use lookup::lookup_v2::OwnedSegment; @@ -9,14 +10,15 @@ use vrl::compiler::{ProgramInfo, SecretTarget, Target}; use vrl::prelude::Collection; use vrl::value::{Kind, ObjectMap, Value}; -use super::{metric::TagValue, Event, EventMetadata, LogEvent, Metric, MetricKind, TraceEvent}; -use crate::config::{log_schema, LogNamespace}; +use super::{Event, EventMetadata, LogEvent, Metric, MetricKind, TraceEvent, metric::TagValue}; +use crate::config::{LogNamespace, log_schema}; use crate::schema::Definition; -const VALID_METRIC_PATHS_SET: &str = ".name, .namespace, .timestamp, .kind, .tags"; +const VALID_METRIC_PATHS_SET: &str = ".name, .namespace, .interval_ms, .timestamp, .kind, .tags"; /// We can get the `type` of the metric in Remap, but can't set it. -const VALID_METRIC_PATHS_GET: &str = ".name, .namespace, .timestamp, .kind, .tags, .type"; +const VALID_METRIC_PATHS_GET: &str = + ".name, .namespace, .interval_ms, .timestamp, .kind, .tags, .type"; /// Metrics aren't interested in paths that have a length longer than 3. /// @@ -102,7 +104,7 @@ impl VrlTarget { // We pre-generate [`Value`] types for the metric fields accessed in // the event. This allows us to then return references to those // values, even if the field is accessed more than once. - let value = precompute_metric_value(&metric, info); + let value = precompute_metric_value(&metric, info, multi_value_metric_tags); VrlTarget::Metric { metric, @@ -196,25 +198,25 @@ fn move_field_definitions_into_message(mut definition: Definition) -> Definition message.remove_object(); message.remove_array(); - if !message.is_never() { - if let Some(message_key) = log_schema().message_key() { - // We need to add the given message type to a field called `message` - // in the event. - let message = Kind::object(Collection::from(BTreeMap::from([( - message_key.to_string().into(), - message, - )]))); - - definition.event_kind_mut().remove_bytes(); - definition.event_kind_mut().remove_integer(); - definition.event_kind_mut().remove_float(); - definition.event_kind_mut().remove_boolean(); - definition.event_kind_mut().remove_timestamp(); - definition.event_kind_mut().remove_regex(); - definition.event_kind_mut().remove_null(); - - *definition.event_kind_mut() = definition.event_kind().union(message); - } + if !message.is_never() + && let Some(message_key) = log_schema().message_key() + { + // We need to add the given message type to a field called `message` + // in the event. + let message = Kind::object(Collection::from(BTreeMap::from([( + message_key.to_string().into(), + message, + )]))); + + definition.event_kind_mut().remove_bytes(); + definition.event_kind_mut().remove_integer(); + definition.event_kind_mut().remove_float(); + definition.event_kind_mut().remove_boolean(); + definition.event_kind_mut().remove_timestamp(); + definition.event_kind_mut().remove_regex(); + definition.event_kind_mut().remove_null(); + + *definition.event_kind_mut() = definition.event_kind().union(message); } definition @@ -240,9 +242,13 @@ fn merge_array_definitions(mut definition: Definition) -> Definition { fn set_metric_tag_values(name: String, value: &Value, metric: &mut Metric, multi_value_tags: bool) { if multi_value_tags { - let tag_values = value - .as_array() - .unwrap_or(&[]) + let values = if let Value::Array(values) = value { + values.as_slice() + } else { + std::slice::from_ref(value) + }; + + let tag_values = values .iter() .filter_map(|value| match value { Value::Bytes(bytes) => { @@ -269,12 +275,12 @@ impl Target for VrlTarget { let path = &target_path.path; match target_path.prefix { PathPrefix::Event => match self { - VrlTarget::LogEvent(ref mut log, _) | VrlTarget::Trace(ref mut log, _) => { + VrlTarget::LogEvent(log, _) | VrlTarget::Trace(log, _) => { log.insert(path, value); Ok(()) } VrlTarget::Metric { - ref mut metric, + metric, value: metric_value, multi_value_tags, } => { @@ -282,10 +288,7 @@ impl Target for VrlTarget { return Err(MetricPathError::SetPathError.to_string()); } - if let Some(paths) = path - .to_alternative_components(MAX_METRIC_PATH_DEPTH) - .first() - { + if let Some(paths) = path.to_alternative_components(MAX_METRIC_PATH_DEPTH) { match paths.as_slice() { ["tags"] => { let value = @@ -319,6 +322,15 @@ impl Target for VrlTarget { metric.series.name.namespace = Some(String::from_utf8_lossy(&value).into_owned()); } + ["interval_ms"] => { + let value: i64 = + value.clone().try_into_i64().map_err(|e| e.to_string())?; + let value: u32 = value + .try_into() + .map_err(|e: TryFromIntError| e.to_string())?; + let value = NonZero::try_from(value).map_err(|e| e.to_string())?; + metric.data.time.interval_ms = Some(value); + } ["timestamp"] => { let value = value.clone().try_timestamp().map_err(|e| e.to_string())?; @@ -332,7 +344,7 @@ impl Target for VrlTarget { path: &path.to_string(), expected: VALID_METRIC_PATHS_SET, } - .to_string()) + .to_string()); } } @@ -392,11 +404,11 @@ impl Target for VrlTarget { ) -> Result, String> { match target_path.prefix { PathPrefix::Event => match self { - VrlTarget::LogEvent(ref mut log, _) | VrlTarget::Trace(ref mut log, _) => { + VrlTarget::LogEvent(log, _) | VrlTarget::Trace(log, _) => { Ok(log.remove(&target_path.path, compact)) } VrlTarget::Metric { - ref mut metric, + metric, value, multi_value_tags: _, } => { @@ -407,11 +419,17 @@ impl Target for VrlTarget { if let Some(paths) = target_path .path .to_alternative_components(MAX_METRIC_PATH_DEPTH) - .first() { let removed_value = match paths.as_slice() { ["namespace"] => metric.series.name.namespace.take().map(Into::into), ["timestamp"] => metric.data.time.timestamp.take().map(Into::into), + ["interval_ms"] => metric + .data + .time + .interval_ms + .take() + .map(u32::from) + .map(Into::into), ["tags"] => metric.series.tags.take().map(|map| { map.into_iter_single() .map(|(k, v)| (k, v.into())) @@ -423,16 +441,16 @@ impl Target for VrlTarget { path: &target_path.path.to_string(), expected: VALID_METRIC_PATHS_SET, } - .to_string()) + .to_string()); } }; value.remove(&target_path.path, false); - return Ok(removed_value); + Ok(removed_value) + } else { + Ok(None) } - - Ok(None) } }, PathPrefix::Metadata => Ok(self @@ -459,13 +477,14 @@ impl SecretTarget for VrlTarget { /// Retrieves a value from a the provided metric using the path. /// Currently the root path and the following paths are supported: -/// - name -/// - namespace -/// - timestamp -/// - kind -/// - tags -/// - tags. -/// - type +/// - `name` +/// - `namespace` +/// - `interval_ms` +/// - `timestamp` +/// - `kind` +/// - `tags` +/// - `tags.` +/// - `type` /// /// Any other paths result in a `MetricPathError::InvalidPath` being returned. fn target_get_metric<'a>( @@ -478,27 +497,25 @@ fn target_get_metric<'a>( let value = value.get(path); - for paths in path.to_alternative_components(MAX_METRIC_PATH_DEPTH) { - match paths.as_slice() { - ["name"] | ["kind"] | ["type"] | ["tags", _] => return Ok(value), - ["namespace"] | ["timestamp"] | ["tags"] => { - if let Some(value) = value { - return Ok(Some(value)); - } - } - _ => { - return Err(MetricPathError::InvalidPath { - path: &path.to_string(), - expected: VALID_METRIC_PATHS_GET, - } - .to_string()) - } + let Some(paths) = path.to_alternative_components(MAX_METRIC_PATH_DEPTH) else { + return Ok(None); + }; + + match paths.as_slice() { + ["name"] + | ["kind"] + | ["type"] + | ["tags", _] + | ["namespace"] + | ["timestamp"] + | ["interval_ms"] + | ["tags"] => Ok(value), + _ => Err(MetricPathError::InvalidPath { + path: &path.to_string(), + expected: VALID_METRIC_PATHS_GET, } + .to_string()), } - - // We only reach this point if we have requested a tag that doesn't exist or an empty - // field. - Ok(None) } fn target_get_mut_metric<'a>( @@ -511,127 +528,138 @@ fn target_get_mut_metric<'a>( let value = value.get_mut(path); - for paths in path.to_alternative_components(MAX_METRIC_PATH_DEPTH) { - match paths.as_slice() { - ["name"] | ["kind"] | ["tags", _] => return Ok(value), - ["namespace"] | ["timestamp"] | ["tags"] => { - if let Some(value) = value { - return Ok(Some(value)); - } - } - _ => { - return Err(MetricPathError::InvalidPath { - path: &path.to_string(), - expected: VALID_METRIC_PATHS_SET, - } - .to_string()) - } + let Some(paths) = path.to_alternative_components(MAX_METRIC_PATH_DEPTH) else { + return Ok(None); + }; + + match paths.as_slice() { + ["name"] + | ["kind"] + | ["tags", _] + | ["namespace"] + | ["timestamp"] + | ["interval_ms"] + | ["tags"] => Ok(value), + _ => Err(MetricPathError::InvalidPath { + path: &path.to_string(), + expected: VALID_METRIC_PATHS_SET, } + .to_string()), } - - // We only reach this point if we have requested a tag that doesn't exist or an empty - // field. - Ok(None) } /// pre-compute the `Value` structure of the metric. /// /// This structure is partially populated based on the fields accessed by /// the VRL program as informed by `ProgramInfo`. -fn precompute_metric_value(metric: &Metric, info: &ProgramInfo) -> Value { - let mut map = ObjectMap::default(); - - let mut set_name = false; - let mut set_kind = false; - let mut set_type = false; - let mut set_namespace = false; - let mut set_timestamp = false; - let mut set_tags = false; +fn precompute_metric_value(metric: &Metric, info: &ProgramInfo, multi_value_tags: bool) -> Value { + struct MetricProperty { + property: &'static str, + getter: fn(&Metric) -> Option, + set: bool, + } - for target_path in &info.target_queries { - // Accessing a root path requires us to pre-populate all fields. - if target_path == &OwnedTargetPath::event_root() { - if !set_name { - map.insert("name".into(), metric.name().to_owned().into()); + impl MetricProperty { + fn new(property: &'static str, getter: fn(&Metric) -> Option) -> Self { + Self { + property, + getter, + set: false, } + } - if !set_kind { - map.insert("kind".into(), metric.kind().into()); + fn insert(&mut self, metric: &Metric, map: &mut ObjectMap) { + if self.set { + return; } - - if !set_type { - map.insert("type".into(), metric.value().clone().into()); + if let Some(value) = (self.getter)(metric) { + map.insert(self.property.into(), value); + self.set = true; } + } + } - if !set_namespace { - if let Some(namespace) = metric.namespace() { - map.insert("namespace".into(), namespace.to_owned().into()); - } - } + fn get_single_value_tags(metric: &Metric) -> Option { + metric.tags().cloned().map(|tags| { + tags.into_iter_single() + .map(|(tag, value)| (tag.into(), value.into())) + .collect::() + .into() + }) + } - if !set_timestamp { - if let Some(timestamp) = metric.timestamp() { - map.insert("timestamp".into(), timestamp.into()); - } - } + fn get_multi_value_tags(metric: &Metric) -> Option { + metric.tags().cloned().map(|tags| { + tags.iter_sets() + .map(|(tag, tag_set)| { + let array_values: Vec = tag_set + .iter() + .map(|v| match v { + Some(s) => Value::Bytes(s.as_bytes().to_vec().into()), + None => Value::Null, + }) + .collect(); + (tag.into(), Value::Array(array_values)) + }) + .collect::() + .into() + }) + } - if !set_tags { - if let Some(tags) = metric.tags().cloned() { - map.insert( - "tags".into(), - tags.into_iter_single() - .map(|(tag, value)| (tag.into(), value.into())) - .collect::() - .into(), - ); - } - } + let mut name = MetricProperty::new("name", |metric| Some(metric.name().to_owned().into())); + let mut kind = MetricProperty::new("kind", |metric| Some(metric.kind().into())); + let mut type_ = MetricProperty::new("type", |metric| Some(metric.value().clone().into())); + let mut namespace = MetricProperty::new("namespace", |metric| { + metric.namespace().map(String::from).map(Into::into) + }); + let mut interval_ms = + MetricProperty::new("interval_ms", |metric| metric.interval_ms().map(Into::into)); + let mut timestamp = + MetricProperty::new("timestamp", |metric| metric.timestamp().map(Into::into)); + let mut tags = MetricProperty::new( + "tags", + if multi_value_tags { + get_multi_value_tags + } else { + get_single_value_tags + }, + ); + let mut map = ObjectMap::default(); + + for target_path in &info.target_queries { + // Accessing a root path requires us to pre-populate all fields. + if target_path == &OwnedTargetPath::event_root() { + let mut properties = [ + &mut name, + &mut kind, + &mut type_, + &mut namespace, + &mut interval_ms, + &mut timestamp, + &mut tags, + ]; + for property in &mut properties { + property.insert(metric, &mut map); + } break; } // For non-root paths, we continuously populate the value with the // relevant data. if let Some(OwnedSegment::Field(field)) = target_path.path.segments.first() { - match field.as_ref() { - "name" if !set_name => { - set_name = true; - map.insert("name".into(), metric.name().to_owned().into()); - } - "kind" if !set_kind => { - set_kind = true; - map.insert("kind".into(), metric.kind().into()); - } - "type" if !set_type => { - set_type = true; - map.insert("type".into(), metric.value().clone().into()); - } - "namespace" if !set_namespace && metric.namespace().is_some() => { - set_namespace = true; - map.insert( - "namespace".into(), - metric.namespace().unwrap().to_owned().into(), - ); - } - "timestamp" if !set_timestamp && metric.timestamp().is_some() => { - set_timestamp = true; - map.insert("timestamp".into(), metric.timestamp().unwrap().into()); - } - "tags" if !set_tags && metric.tags().is_some() => { - set_tags = true; - map.insert( - "tags".into(), - metric - .tags() - .cloned() - .unwrap() - .into_iter_single() - .map(|(tag, value)| (tag.into(), value.into())) - .collect::() - .into(), - ); - } - _ => {} + let property = match field.as_ref() { + "name" => Some(&mut name), + "kind" => Some(&mut kind), + "type" => Some(&mut type_), + "namespace" => Some(&mut namespace), + "timestamp" => Some(&mut timestamp), + "interval_ms" => Some(&mut interval_ms), + "tags" => Some(&mut tags), + _ => None, + }; + if let Some(property) = property { + property.insert(metric, &mut map); } } } @@ -650,7 +678,7 @@ enum MetricPathError<'a> { #[cfg(test)] mod test { - use chrono::{offset::TimeZone, Utc}; + use chrono::{Utc, offset::TimeZone}; use lookup::owned_value_path; use similar_asserts::assert_eq; use vrl::btreemap; @@ -1088,7 +1116,8 @@ mod test { Utc.with_ymd_and_hms(2020, 12, 10, 12, 0, 0) .single() .expect("invalid timestamp"), - )); + )) + .with_interval_ms(Some(NonZero::::new(507).unwrap())); let info = ProgramInfo { fallible: false, @@ -1096,6 +1125,7 @@ mod test { target_queries: vec![ OwnedTargetPath::event(owned_value_path!("name")), OwnedTargetPath::event(owned_value_path!("namespace")), + OwnedTargetPath::event(owned_value_path!("interval_ms")), OwnedTargetPath::event(owned_value_path!("timestamp")), OwnedTargetPath::event(owned_value_path!("kind")), OwnedTargetPath::event(owned_value_path!("type")), @@ -1110,6 +1140,7 @@ mod test { btreemap! { "name" => "zub", "namespace" => "zoob", + "interval_ms" => 507, "timestamp" => Utc.with_ymd_and_hms(2020, 12, 10, 12, 0, 0).single().expect("invalid timestamp"), "tags" => btreemap! { "tig" => "tog" }, "kind" => "absolute", @@ -1125,6 +1156,13 @@ mod test { #[test] fn metric_fields() { + struct Case { + path: OwnedValuePath, + current: Option, + new: Value, + delete: bool, + } + let metric = Metric::new( "name", MetricKind::Absolute, @@ -1133,39 +1171,46 @@ mod test { .with_tags(Some(metric_tags!("tig" => "tog"))); let cases = vec![ - ( - owned_value_path!("name"), // Path - Some(Value::from("name")), // Current value - Value::from("namefoo"), // New value - false, // Test deletion - ), - ( - owned_value_path!("namespace"), - None, - "namespacefoo".into(), - true, - ), - ( - owned_value_path!("timestamp"), - None, - Utc.with_ymd_and_hms(2020, 12, 8, 12, 0, 0) + Case { + path: owned_value_path!("name"), + current: Some(Value::from("name")), + new: Value::from("namefoo"), + delete: false, + }, + Case { + path: owned_value_path!("namespace"), + current: None, + new: "namespacefoo".into(), + delete: true, + }, + Case { + path: owned_value_path!("timestamp"), + current: None, + new: Utc + .with_ymd_and_hms(2020, 12, 8, 12, 0, 0) .single() .expect("invalid timestamp") .into(), - true, - ), - ( - owned_value_path!("kind"), - Some(Value::from("absolute")), - "incremental".into(), - false, - ), - ( - owned_value_path!("tags", "thing"), - None, - "footag".into(), - true, - ), + delete: true, + }, + Case { + path: owned_value_path!("interval_ms"), + current: None, + new: 123_456.into(), + delete: true, + }, + Case { + path: owned_value_path!("kind"), + current: Some(Value::from("absolute")), + new: "incremental".into(), + delete: false, + }, + Case { + path: owned_value_path!("tags", "thing"), + current: None, + new: "footag".into(), + delete: true, + }, ]; let info = ProgramInfo { @@ -1175,13 +1220,20 @@ mod test { OwnedTargetPath::event(owned_value_path!("name")), OwnedTargetPath::event(owned_value_path!("namespace")), OwnedTargetPath::event(owned_value_path!("timestamp")), + OwnedTargetPath::event(owned_value_path!("interval_ms")), OwnedTargetPath::event(owned_value_path!("kind")), ], target_assignments: vec![], }; let mut target = VrlTarget::new(Event::Metric(metric), &info, false); - for (path, current, new, delete) in cases { + for Case { + path, + current, + new, + delete, + } in cases + { let path = OwnedTargetPath::event(path); assert_eq!( @@ -1249,13 +1301,21 @@ mod test { let validpaths_get = [ ".name", ".namespace", + ".interval_ms", ".timestamp", ".kind", ".tags", ".type", ]; - let validpaths_set = [".name", ".namespace", ".timestamp", ".kind", ".tags"]; + let validpaths_set = [ + ".name", + ".namespace", + ".interval_ms", + ".timestamp", + ".kind", + ".tags", + ]; let info = ProgramInfo { fallible: false, diff --git a/lib/vector-core/src/fanout.rs b/lib/vector-core/src/fanout.rs index 889d8b6243..746f47ae71 100644 --- a/lib/vector-core/src/fanout.rs +++ b/lib/vector-core/src/fanout.rs @@ -468,11 +468,11 @@ mod tests { use tokio_test::{assert_pending, assert_ready, task::spawn}; use tracing::Span; use vector_buffers::{ + WhenFull, topology::{ builder::TopologyBuilder, channel::{BufferReceiver, BufferSender}, }, - WhenFull, }; use vrl::value::Value; @@ -622,7 +622,10 @@ mod tests { fanout.send(clones, None).await.expect("should not fail"); for receiver in receivers { - assert_eq!(collect_ready(receiver.into_stream()), &[events.clone()]); + assert_eq!( + collect_ready(receiver.into_stream()), + std::slice::from_ref(&events) + ); } } diff --git a/lib/vector-core/src/ipallowlist.rs b/lib/vector-core/src/ipallowlist.rs index e1e5980383..add55aff31 100644 --- a/lib/vector-core/src/ipallowlist.rs +++ b/lib/vector-core/src/ipallowlist.rs @@ -3,7 +3,7 @@ use std::cell::RefCell; use vector_config::GenerateError; use ipnet::IpNet; -use vector_config::{configurable_component, Configurable, Metadata, ToValue}; +use vector_config::{Configurable, Metadata, ToValue, configurable_component}; use vector_config_common::schema::{InstanceType, SchemaGenerator, SchemaObject}; /// List of allowed origin IP networks. IP addresses must be in CIDR notation. diff --git a/lib/vector-core/src/metrics/ddsketch.rs b/lib/vector-core/src/metrics/ddsketch.rs index 3d0f80bb50..b66f25ceb1 100644 --- a/lib/vector-core/src/metrics/ddsketch.rs +++ b/lib/vector-core/src/metrics/ddsketch.rs @@ -4,14 +4,14 @@ use std::{ }; use ordered_float::OrderedFloat; -use serde::{de::Error, Deserialize, Deserializer, Serialize, Serializer}; -use serde_with::{serde_as, DeserializeAs, SerializeAs}; +use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error}; +use serde_with::{DeserializeAs, SerializeAs, serde_as}; use snafu::Snafu; use vector_common::byte_size_of::ByteSizeOf; use vector_config::configurable_component; use crate::{ - event::{metric::Bucket, Metric, MetricValue}, + event::{Metric, MetricValue, metric::Bucket}, float_eq, }; @@ -456,9 +456,7 @@ impl AgentDDSketch { // something horribly broken. assert!( keys.len() - <= u32::MAX - .try_into() - .expect("we don't support 16-bit systems") + <= TryInto::::try_into(u32::MAX).expect("we don't support 16-bit systems") ); keys.sort_unstable(); @@ -1098,7 +1096,7 @@ fn round_to_even(v: f64) -> f64 { #[cfg(test)] mod tests { - use super::{round_to_even, AgentDDSketch, Config, AGENT_DEFAULT_EPS, MAX_KEY}; + use super::{AGENT_DEFAULT_EPS, AgentDDSketch, Config, MAX_KEY, round_to_even}; use crate::event::metric::Bucket; const FLOATING_POINT_ACCEPTABLE_ERROR: f64 = 1.0e-10; @@ -1106,7 +1104,7 @@ mod tests { #[cfg(ddsketch_extended)] fn generate_pareto_distribution() -> Vec> { use ordered_float::OrderedFloat; - use rand::thread_rng; + use rand::rng; use rand_distr::{Distribution, Pareto}; // Generate a set of samples that roughly correspond to the latency of a typical web @@ -1116,7 +1114,7 @@ mod tests { //let distribution = Gamma::new(1.2, 100.0).unwrap(); let distribution = Pareto::new(1.0, 1.0).expect("pareto distribution should be valid"); let mut samples = distribution - .sample_iter(thread_rng()) + .sample_iter(rng()) // Scale by 10,000 to get microseconds. .map(|n| n * 10_000.0) .filter(|n| *n > 15_000.0 && *n < 10_000_000.0) @@ -1315,7 +1313,7 @@ mod tests { #[cfg(ddsketch_extended)] fn test_ddsketch_pareto_distribution() { use ndarray::{Array, Axis}; - use ndarray_stats::{interpolate::Midpoint, QuantileExt}; + use ndarray_stats::{QuantileExt, interpolate::Midpoint}; use noisy_float::prelude::N64; // NOTE: This test unexpectedly fails to meet the relative accuracy guarantees when checking @@ -1360,9 +1358,16 @@ mod tests { .into_inner(); let _err = (estimated - actual).abs() / actual; - assert!(err <= relative_accuracy, - "relative accuracy out of bounds: q={}, estimate={}, actual={}, target-rel-acc={}, actual-rel-acc={}, bin-count={}", - q, estimated, actual, relative_accuracy, err, sketch.bin_count()); + assert!( + err <= relative_accuracy, + "relative accuracy out of bounds: q={}, estimate={}, actual={}, target-rel-acc={}, actual-rel-acc={}, bin-count={}", + q, + estimated, + actual, + relative_accuracy, + err, + sketch.bin_count() + ); } } @@ -1461,17 +1466,57 @@ mod tests { }; let cases = &[ - Case { lower: 0.0, upper: 10.0, count: 2, allowed_err: 0.0, expected: "0:1 1442:1" }, - Case { lower: 10.0, upper: 20.0, count: 4, allowed_err: 0.0, expected: "1487:1 1502:1 1514:1 1524:1" }, - Case { lower: -10.0, upper: 10.0, count: 4, allowed_err: 0.0, expected: "-1487:1 -1442:1 -1067:1 1442:1"}, - Case { lower: 0.0, upper: 10.0, count: 100, allowed_err: 0.0, expected: "0:1 1190:1 1235:1 1261:1 1280:1 1295:1 1307:1 1317:1 1326:1 1334:1 1341:1 1347:1 1353:1 1358:1 1363:1 1368:1 1372:2 1376:1 1380:1 1384:1 1388:1 1391:1 1394:1 1397:2 1400:1 1403:1 1406:2 1409:1 1412:1 1415:2 1417:1 1419:1 1421:1 1423:1 1425:1 1427:1 1429:2 1431:1 1433:1 1435:2 1437:1 1439:2 1441:1 1443:2 1445:2 1447:1 1449:2 1451:2 1453:2 1455:2 1457:2 1459:1 1460:1 1461:1 1462:1 1463:1 1464:1 1465:1 1466:1 1467:2 1468:1 1469:1 1470:1 1471:1 1472:2 1473:1 1474:1 1475:1 1476:2 1477:1 1478:2 1479:1 1480:1 1481:2 1482:1 1483:2 1484:1 1485:2 1486:1" }, - Case { lower: 1_000.0, upper: 100_000.0, count: 1_000_000 - 1, allowed_err: 0.0, expected: "1784:158 1785:162 1786:164 1787:166 1788:170 1789:171 1790:175 1791:177 1792:180 1793:183 1794:185 1795:189 1796:191 1797:195 1798:197 1799:201 1800:203 1801:207 1802:210 1803:214 1804:217 1805:220 1806:223 1807:227 1808:231 1809:234 1810:238 1811:242 1812:245 1813:249 1814:253 1815:257 1816:261 1817:265 1818:270 1819:273 1820:278 1821:282 1822:287 1823:291 1824:295 1825:300 1826:305 1827:310 1828:314 1829:320 1830:324 1831:329 1832:335 1833:340 1834:345 1835:350 1836:356 1837:362 1838:367 1839:373 1840:379 1841:384 1842:391 1843:397 1844:403 1845:409 1846:416 1847:422 1848:429 1849:435 1850:442 1851:449 1852:457 1853:463 1854:470 1855:478 1856:486 1857:493 1858:500 1859:509 1860:516 1861:525 1862:532 1863:541 1864:550 1865:558 1866:567 1867:575 1868:585 1869:594 1870:603 1871:612 1872:622 1873:632 1874:642 1875:651 1876:662 1877:672 1878:683 1879:693 1880:704 1881:716 1882:726 1883:738 1884:749 1885:761 1886:773 1887:785 1888:797 1889:809 1890:823 1891:835 1892:848 1893:861 1894:875 1895:889 1896:902 1897:917 1898:931 1899:945 1900:960 1901:975 1902:991 1903:1006 1904:1021 1905:1038 1906:1053 1907:1071 1908:1087 1909:1104 1910:1121 1911:1138 1912:1157 1913:1175 1914:1192 1915:1212 1916:1231 1917:1249 1918:1269 1919:1290 1920:1309 1921:1329 1922:1351 1923:1371 1924:1393 1925:1415 1926:1437 1927:1459 1928:1482 1929:1506 1930:1529 1931:1552 1932:1577 1933:1602 1934:1626 1935:1652 1936:1678 1937:1704 1938:1731 1939:1758 1940:1785 1941:1813 1942:1841 1943:1870 1944:1900 1945:1929 1946:1959 1947:1990 1948:2021 1949:2052 1950:2085 1951:2117 1952:2150 1953:2184 1954:2218 1955:2253 1956:2287 1957:2324 1958:2360 1959:2396 1960:2435 1961:2472 1962:2511 1963:2550 1964:2589 1965:2631 1966:2671 1967:2714 1968:2755 1969:2799 1970:2842 1971:2887 1972:2932 1973:2978 1974:3024 1975:3071 1976:3120 1977:3168 1978:3218 1979:3268 1980:3319 1981:3371 1982:3423 1983:3477 1984:3532 1985:3586 1986:3643 1987:3700 1988:3757 1989:3816 1990:3876 1991:3936 1992:3998 1993:4060 1994:4124 1995:4188 1996:4253 1997:4320 1998:4388 1999:4456 2000:4526 2001:4596 2002:4668 2003:4741 2004:4816 2005:4890 2006:4967 2007:5044 2008:5124 2009:5203 2010:5285 2011:5367 2012:5451 2013:5536 2014:5623 2015:5711 2016:5800 2017:5890 2018:5983 2019:6076 2020:6171 2021:6267 2022:6365 2023:6465 2024:6566 2025:6668 2026:6773 2027:6878 2028:6986 2029:7095 2030:7206 2031:7318 2032:7433 2033:7549 2034:7667 2035:7786 2036:7909 2037:8032 2038:8157 2039:8285 2040:8414 2041:8546 2042:8679 2043:8815 2044:8953 2045:9092 2046:9235 2047:9379 2048:9525 2049:9675 2050:9825 2051:9979 2052:10135 2053:10293 2054:10454 2055:10618 2056:10783 2057:10952 2058:11123 2059:11297 2060:11473 2061:11653 2062:11834 2063:12020 2064:12207 2065:12398 2066:12592 2067:12788 2068:12989 2069:13191 2070:13397 2071:13607 2072:13819 2073:14036 2074:14254 2075:14478 2076:14703 2077:14933 2078:15167 2079:15403 2080:8942" }, - Case { lower: 1_000.0, upper: 10_000.0, count: 10_000_000 - 1, allowed_err: 0.00001, expected: "1784:17485 1785:17758 1786:18035 1787:18318 1788:18604 1789:18894 1790:19190 1791:19489 1792:19794 1793:20103 1794:20418 1795:20736 1796:21061 1797:21389 1798:21724 1799:22063 1800:22408 1801:22758 1802:23113 1803:23475 1804:23841 1805:24215 1806:24592 1807:24977 1808:25366 1809:25764 1810:26165 1811:26575 1812:26990 1813:27412 1814:27839 1815:28275 1816:28717 1817:29165 1818:29622 1819:30083 1820:30554 1821:31032 1822:31516 1823:32009 1824:32509 1825:33016 1826:33533 1827:34057 1828:34589 1829:35129 1830:35678 1831:36235 1832:36802 1833:37377 1834:37961 1835:38554 1836:39156 1837:39768 1838:40390 1839:41020 1840:41662 1841:42312 1842:42974 1843:43645 1844:44327 1845:45020 1846:45723 1847:46438 1848:47163 1849:47900 1850:48648 1851:49409 1852:50181 1853:50964 1854:51761 1855:52570 1856:53391 1857:54226 1858:55072 1859:55934 1860:56807 1861:57695 1862:58596 1863:59512 1864:60441 1865:61387 1866:62345 1867:63319 1868:64309 1869:65314 1870:799 1870:65535 1871:1835 1871:65535 1872:2889 1872:65535 1873:3957 1873:65535 1874:5043 1874:65535 1875:6146 1875:65535 1876:7266 1876:65535 1877:8404 1877:65535 1878:9559 1878:65535 1879:10732 1879:65535 1880:11923 1880:65535 1881:13135 1881:65535 1882:14363 1882:65535 1883:15612 1883:65535 1884:16879 1884:65535 1885:18168 1885:65535 1886:19475 1886:65535 1887:20803 1887:65535 1888:22153 1888:65535 1889:23523 1889:65535 1890:24914 1890:65535 1891:26327 1891:65535 1892:27763 1892:65535 1893:29221 1893:65535 1894:30701 1894:65535 1895:32205 1895:65535 1896:33732 1896:65535 1897:35283 1897:65535 1898:36858 1898:65535 1899:38458 1899:65535 1900:40084 1900:65535 1901:41733 1901:65535 1902:43409 1902:65535 1903:45112 1903:65535 1904:46841 1904:65535 1905:48596 1905:65535 1906:50380 1906:65535 1907:52191 1907:65535 1908:54030 1908:65535 1909:55899 1909:65535 1910:57796 1910:65535 1911:59723 1911:65535 1912:61680 1912:65535 1913:63668 1913:65535 1914:152 1914:65535 1914:65535 1915:2202 1915:65535 1915:65535 1916:4285 1916:65535 1916:65535 1917:6399 1917:65535 1917:65535 1918:8547 1918:65535 1918:65535 1919:10729 1919:65535 1919:65535 1920:12945 1920:65535 1920:65535 1921:15195 1921:65535 1921:65535 1922:17480 1922:65535 1922:65535 1923:19801 1923:65535 1923:65535 1924:22158 1924:65535 1924:65535 1925:24553 1925:65535 1925:65535 1926:26985 1926:65535 1926:65535 1927:29453 1927:65535 1927:65535 1928:31963 1928:65535 1928:65535 1929:34509 1929:65535 1929:65535 1930:37097 1930:65535 1930:65535 1931:39724 1931:65535 1931:65535 1932:17411"}, + Case { + lower: 0.0, + upper: 10.0, + count: 2, + allowed_err: 0.0, + expected: "0:1 1442:1", + }, + Case { + lower: 10.0, + upper: 20.0, + count: 4, + allowed_err: 0.0, + expected: "1487:1 1502:1 1514:1 1524:1", + }, + Case { + lower: -10.0, + upper: 10.0, + count: 4, + allowed_err: 0.0, + expected: "-1487:1 -1442:1 -1067:1 1442:1", + }, + Case { + lower: 0.0, + upper: 10.0, + count: 100, + allowed_err: 0.0, + expected: "0:1 1190:1 1235:1 1261:1 1280:1 1295:1 1307:1 1317:1 1326:1 1334:1 1341:1 1347:1 1353:1 1358:1 1363:1 1368:1 1372:2 1376:1 1380:1 1384:1 1388:1 1391:1 1394:1 1397:2 1400:1 1403:1 1406:2 1409:1 1412:1 1415:2 1417:1 1419:1 1421:1 1423:1 1425:1 1427:1 1429:2 1431:1 1433:1 1435:2 1437:1 1439:2 1441:1 1443:2 1445:2 1447:1 1449:2 1451:2 1453:2 1455:2 1457:2 1459:1 1460:1 1461:1 1462:1 1463:1 1464:1 1465:1 1466:1 1467:2 1468:1 1469:1 1470:1 1471:1 1472:2 1473:1 1474:1 1475:1 1476:2 1477:1 1478:2 1479:1 1480:1 1481:2 1482:1 1483:2 1484:1 1485:2 1486:1", + }, + Case { + lower: 1_000.0, + upper: 100_000.0, + count: 1_000_000 - 1, + allowed_err: 0.0, + expected: "1784:158 1785:162 1786:164 1787:166 1788:170 1789:171 1790:175 1791:177 1792:180 1793:183 1794:185 1795:189 1796:191 1797:195 1798:197 1799:201 1800:203 1801:207 1802:210 1803:214 1804:217 1805:220 1806:223 1807:227 1808:231 1809:234 1810:238 1811:242 1812:245 1813:249 1814:253 1815:257 1816:261 1817:265 1818:270 1819:273 1820:278 1821:282 1822:287 1823:291 1824:295 1825:300 1826:305 1827:310 1828:314 1829:320 1830:324 1831:329 1832:335 1833:340 1834:345 1835:350 1836:356 1837:362 1838:367 1839:373 1840:379 1841:384 1842:391 1843:397 1844:403 1845:409 1846:416 1847:422 1848:429 1849:435 1850:442 1851:449 1852:457 1853:463 1854:470 1855:478 1856:486 1857:493 1858:500 1859:509 1860:516 1861:525 1862:532 1863:541 1864:550 1865:558 1866:567 1867:575 1868:585 1869:594 1870:603 1871:612 1872:622 1873:632 1874:642 1875:651 1876:662 1877:672 1878:683 1879:693 1880:704 1881:716 1882:726 1883:738 1884:749 1885:761 1886:773 1887:785 1888:797 1889:809 1890:823 1891:835 1892:848 1893:861 1894:875 1895:889 1896:902 1897:917 1898:931 1899:945 1900:960 1901:975 1902:991 1903:1006 1904:1021 1905:1038 1906:1053 1907:1071 1908:1087 1909:1104 1910:1121 1911:1138 1912:1157 1913:1175 1914:1192 1915:1212 1916:1231 1917:1249 1918:1269 1919:1290 1920:1309 1921:1329 1922:1351 1923:1371 1924:1393 1925:1415 1926:1437 1927:1459 1928:1482 1929:1506 1930:1529 1931:1552 1932:1577 1933:1602 1934:1626 1935:1652 1936:1678 1937:1704 1938:1731 1939:1758 1940:1785 1941:1813 1942:1841 1943:1870 1944:1900 1945:1929 1946:1959 1947:1990 1948:2021 1949:2052 1950:2085 1951:2117 1952:2150 1953:2184 1954:2218 1955:2253 1956:2287 1957:2324 1958:2360 1959:2396 1960:2435 1961:2472 1962:2511 1963:2550 1964:2589 1965:2631 1966:2671 1967:2714 1968:2755 1969:2799 1970:2842 1971:2887 1972:2932 1973:2978 1974:3024 1975:3071 1976:3120 1977:3168 1978:3218 1979:3268 1980:3319 1981:3371 1982:3423 1983:3477 1984:3532 1985:3586 1986:3643 1987:3700 1988:3757 1989:3816 1990:3876 1991:3936 1992:3998 1993:4060 1994:4124 1995:4188 1996:4253 1997:4320 1998:4388 1999:4456 2000:4526 2001:4596 2002:4668 2003:4741 2004:4816 2005:4890 2006:4967 2007:5044 2008:5124 2009:5203 2010:5285 2011:5367 2012:5451 2013:5536 2014:5623 2015:5711 2016:5800 2017:5890 2018:5983 2019:6076 2020:6171 2021:6267 2022:6365 2023:6465 2024:6566 2025:6668 2026:6773 2027:6878 2028:6986 2029:7095 2030:7206 2031:7318 2032:7433 2033:7549 2034:7667 2035:7786 2036:7909 2037:8032 2038:8157 2039:8285 2040:8414 2041:8546 2042:8679 2043:8815 2044:8953 2045:9092 2046:9235 2047:9379 2048:9525 2049:9675 2050:9825 2051:9979 2052:10135 2053:10293 2054:10454 2055:10618 2056:10783 2057:10952 2058:11123 2059:11297 2060:11473 2061:11653 2062:11834 2063:12020 2064:12207 2065:12398 2066:12592 2067:12788 2068:12989 2069:13191 2070:13397 2071:13607 2072:13819 2073:14036 2074:14254 2075:14478 2076:14703 2077:14933 2078:15167 2079:15403 2080:8942", + }, + Case { + lower: 1_000.0, + upper: 10_000.0, + count: 10_000_000 - 1, + allowed_err: 0.00001, + expected: "1784:17485 1785:17758 1786:18035 1787:18318 1788:18604 1789:18894 1790:19190 1791:19489 1792:19794 1793:20103 1794:20418 1795:20736 1796:21061 1797:21389 1798:21724 1799:22063 1800:22408 1801:22758 1802:23113 1803:23475 1804:23841 1805:24215 1806:24592 1807:24977 1808:25366 1809:25764 1810:26165 1811:26575 1812:26990 1813:27412 1814:27839 1815:28275 1816:28717 1817:29165 1818:29622 1819:30083 1820:30554 1821:31032 1822:31516 1823:32009 1824:32509 1825:33016 1826:33533 1827:34057 1828:34589 1829:35129 1830:35678 1831:36235 1832:36802 1833:37377 1834:37961 1835:38554 1836:39156 1837:39768 1838:40390 1839:41020 1840:41662 1841:42312 1842:42974 1843:43645 1844:44327 1845:45020 1846:45723 1847:46438 1848:47163 1849:47900 1850:48648 1851:49409 1852:50181 1853:50964 1854:51761 1855:52570 1856:53391 1857:54226 1858:55072 1859:55934 1860:56807 1861:57695 1862:58596 1863:59512 1864:60441 1865:61387 1866:62345 1867:63319 1868:64309 1869:65314 1870:799 1870:65535 1871:1835 1871:65535 1872:2889 1872:65535 1873:3957 1873:65535 1874:5043 1874:65535 1875:6146 1875:65535 1876:7266 1876:65535 1877:8404 1877:65535 1878:9559 1878:65535 1879:10732 1879:65535 1880:11923 1880:65535 1881:13135 1881:65535 1882:14363 1882:65535 1883:15612 1883:65535 1884:16879 1884:65535 1885:18168 1885:65535 1886:19475 1886:65535 1887:20803 1887:65535 1888:22153 1888:65535 1889:23523 1889:65535 1890:24914 1890:65535 1891:26327 1891:65535 1892:27763 1892:65535 1893:29221 1893:65535 1894:30701 1894:65535 1895:32205 1895:65535 1896:33732 1896:65535 1897:35283 1897:65535 1898:36858 1898:65535 1899:38458 1899:65535 1900:40084 1900:65535 1901:41733 1901:65535 1902:43409 1902:65535 1903:45112 1903:65535 1904:46841 1904:65535 1905:48596 1905:65535 1906:50380 1906:65535 1907:52191 1907:65535 1908:54030 1908:65535 1909:55899 1909:65535 1910:57796 1910:65535 1911:59723 1911:65535 1912:61680 1912:65535 1913:63668 1913:65535 1914:152 1914:65535 1914:65535 1915:2202 1915:65535 1915:65535 1916:4285 1916:65535 1916:65535 1917:6399 1917:65535 1917:65535 1918:8547 1918:65535 1918:65535 1919:10729 1919:65535 1919:65535 1920:12945 1920:65535 1920:65535 1921:15195 1921:65535 1921:65535 1922:17480 1922:65535 1922:65535 1923:19801 1923:65535 1923:65535 1924:22158 1924:65535 1924:65535 1925:24553 1925:65535 1925:65535 1926:26985 1926:65535 1926:65535 1927:29453 1927:65535 1927:65535 1928:31963 1928:65535 1928:65535 1929:34509 1929:65535 1929:65535 1930:37097 1930:65535 1930:65535 1931:39724 1931:65535 1931:65535 1932:17411", + }, ]; - let double_insert_cases = &[ - Case { lower: 1_000.0, upper: 10_000.0, count: 10_000_000 - 1, allowed_err: 0.0002, expected: "1784:34970 1785:35516 1786:36070 1787:36636 1788:37208 1789:37788 1790:38380 1791:38978 1792:39588 1793:40206 1794:40836 1795:41472 1796:42122 1797:42778 1798:43448 1799:44126 1800:44816 1801:45516 1802:46226 1803:46950 1804:47682 1805:48430 1806:49184 1807:49954 1808:50732 1809:51528 1810:52330 1811:53150 1812:53980 1813:54824 1814:55678 1815:56550 1816:57434 1817:58330 1818:59244 1819:60166 1820:61108 1821:62064 1822:63032 1823:64018 1824:65018 1825:497 1825:65535 1826:1531 1826:65535 1827:2579 1827:65535 1828:3643 1828:65535 1829:4723 1829:65535 1830:5821 1830:65535 1831:6935 1831:65535 1832:8069 1832:65535 1833:9219 1833:65535 1834:10387 1834:65535 1835:11573 1835:65535 1836:12777 1836:65535 1837:14001 1837:65535 1838:15245 1838:65535 1839:16505 1839:65535 1840:17789 1840:65535 1841:19089 1841:65535 1842:20413 1842:65535 1843:21755 1843:65535 1844:23119 1844:65535 1845:24505 1845:65535 1846:25911 1846:65535 1847:27341 1847:65535 1848:28791 1848:65535 1849:30265 1849:65535 1850:31761 1850:65535 1851:33283 1851:65535 1852:34827 1852:65535 1853:36393 1853:65535 1854:37987 1854:65535 1855:39605 1855:65535 1856:41247 1856:65535 1857:42917 1857:65535 1858:44609 1858:65535 1859:46333 1859:65535 1860:48079 1860:65535 1861:49855 1861:65535 1862:51657 1862:65535 1863:53489 1863:65535 1864:55347 1864:65535 1865:57239 1865:65535 1866:59155 1866:65535 1867:61103 1867:65535 1868:63083 1868:65535 1869:65093 1869:65535 1870:1598 1870:65535 1870:65535 1871:3670 1871:65535 1871:65535 1872:5778 1872:65535 1872:65535 1873:7914 1873:65535 1873:65535 1874:10086 1874:65535 1874:65535 1875:12292 1875:65535 1875:65535 1876:14532 1876:65535 1876:65535 1877:16808 1877:65535 1877:65535 1878:19118 1878:65535 1878:65535 1879:21464 1879:65535 1879:65535 1880:23846 1880:65535 1880:65535 1881:26270 1881:65535 1881:65535 1882:28726 1882:65535 1882:65535 1883:31224 1883:65535 1883:65535 1884:33758 1884:65535 1884:65535 1885:36336 1885:65535 1885:65535 1886:38950 1886:65535 1886:65535 1887:41606 1887:65535 1887:65535 1888:44306 1888:65535 1888:65535 1889:47046 1889:65535 1889:65535 1890:49828 1890:65535 1890:65535 1891:52654 1891:65535 1891:65535 1892:55526 1892:65535 1892:65535 1893:58442 1893:65535 1893:65535 1894:61402 1894:65535 1894:65535 1895:64410 1895:65535 1895:65535 1896:1929 1896:65535 1896:65535 1896:65535 1897:5031 1897:65535 1897:65535 1897:65535 1898:8181 1898:65535 1898:65535 1898:65535 1899:11381 1899:65535 1899:65535 1899:65535 1900:14633 1900:65535 1900:65535 1900:65535 1901:17931 1901:65535 1901:65535 1901:65535 1902:21283 1902:65535 1902:65535 1902:65535 1903:24689 1903:65535 1903:65535 1903:65535 1904:28147 1904:65535 1904:65535 1904:65535 1905:31657 1905:65535 1905:65535 1905:65535 1906:35225 1906:65535 1906:65535 1906:65535 1907:38847 1907:65535 1907:65535 1907:65535 1908:42525 1908:65535 1908:65535 1908:65535 1909:46263 1909:65535 1909:65535 1909:65535 1910:50057 1910:65535 1910:65535 1910:65535 1911:53911 1911:65535 1911:65535 1911:65535 1912:57825 1912:65535 1912:65535 1912:65535 1913:61801 1913:65535 1913:65535 1913:65535 1914:304 1914:65535 1914:65535 1914:65535 1914:65535 1915:4404 1915:65535 1915:65535 1915:65535 1915:65535 1916:8570 1916:65535 1916:65535 1916:65535 1916:65535 1917:12798 1917:65535 1917:65535 1917:65535 1917:65535 1918:17094 1918:65535 1918:65535 1918:65535 1918:65535 1919:21458 1919:65535 1919:65535 1919:65535 1919:65535 1920:25890 1920:65535 1920:65535 1920:65535 1920:65535 1921:30390 1921:65535 1921:65535 1921:65535 1921:65535 1922:34960 1922:65535 1922:65535 1922:65535 1922:65535 1923:39602 1923:65535 1923:65535 1923:65535 1923:65535 1924:44316 1924:65535 1924:65535 1924:65535 1924:65535 1925:49106 1925:65535 1925:65535 1925:65535 1925:65535 1926:53970 1926:65535 1926:65535 1926:65535 1926:65535 1927:58906 1927:65535 1927:65535 1927:65535 1927:65535 1928:63926 1928:65535 1928:65535 1928:65535 1928:65535 1929:3483 1929:65535 1929:65535 1929:65535 1929:65535 1929:65535 1930:8659 1930:65535 1930:65535 1930:65535 1930:65535 1930:65535 1931:13913 1931:65535 1931:65535 1931:65535 1931:65535 1931:65535 1932:34822" }, - ]; + let double_insert_cases = &[Case { + lower: 1_000.0, + upper: 10_000.0, + count: 10_000_000 - 1, + allowed_err: 0.0002, + expected: "1784:34970 1785:35516 1786:36070 1787:36636 1788:37208 1789:37788 1790:38380 1791:38978 1792:39588 1793:40206 1794:40836 1795:41472 1796:42122 1797:42778 1798:43448 1799:44126 1800:44816 1801:45516 1802:46226 1803:46950 1804:47682 1805:48430 1806:49184 1807:49954 1808:50732 1809:51528 1810:52330 1811:53150 1812:53980 1813:54824 1814:55678 1815:56550 1816:57434 1817:58330 1818:59244 1819:60166 1820:61108 1821:62064 1822:63032 1823:64018 1824:65018 1825:497 1825:65535 1826:1531 1826:65535 1827:2579 1827:65535 1828:3643 1828:65535 1829:4723 1829:65535 1830:5821 1830:65535 1831:6935 1831:65535 1832:8069 1832:65535 1833:9219 1833:65535 1834:10387 1834:65535 1835:11573 1835:65535 1836:12777 1836:65535 1837:14001 1837:65535 1838:15245 1838:65535 1839:16505 1839:65535 1840:17789 1840:65535 1841:19089 1841:65535 1842:20413 1842:65535 1843:21755 1843:65535 1844:23119 1844:65535 1845:24505 1845:65535 1846:25911 1846:65535 1847:27341 1847:65535 1848:28791 1848:65535 1849:30265 1849:65535 1850:31761 1850:65535 1851:33283 1851:65535 1852:34827 1852:65535 1853:36393 1853:65535 1854:37987 1854:65535 1855:39605 1855:65535 1856:41247 1856:65535 1857:42917 1857:65535 1858:44609 1858:65535 1859:46333 1859:65535 1860:48079 1860:65535 1861:49855 1861:65535 1862:51657 1862:65535 1863:53489 1863:65535 1864:55347 1864:65535 1865:57239 1865:65535 1866:59155 1866:65535 1867:61103 1867:65535 1868:63083 1868:65535 1869:65093 1869:65535 1870:1598 1870:65535 1870:65535 1871:3670 1871:65535 1871:65535 1872:5778 1872:65535 1872:65535 1873:7914 1873:65535 1873:65535 1874:10086 1874:65535 1874:65535 1875:12292 1875:65535 1875:65535 1876:14532 1876:65535 1876:65535 1877:16808 1877:65535 1877:65535 1878:19118 1878:65535 1878:65535 1879:21464 1879:65535 1879:65535 1880:23846 1880:65535 1880:65535 1881:26270 1881:65535 1881:65535 1882:28726 1882:65535 1882:65535 1883:31224 1883:65535 1883:65535 1884:33758 1884:65535 1884:65535 1885:36336 1885:65535 1885:65535 1886:38950 1886:65535 1886:65535 1887:41606 1887:65535 1887:65535 1888:44306 1888:65535 1888:65535 1889:47046 1889:65535 1889:65535 1890:49828 1890:65535 1890:65535 1891:52654 1891:65535 1891:65535 1892:55526 1892:65535 1892:65535 1893:58442 1893:65535 1893:65535 1894:61402 1894:65535 1894:65535 1895:64410 1895:65535 1895:65535 1896:1929 1896:65535 1896:65535 1896:65535 1897:5031 1897:65535 1897:65535 1897:65535 1898:8181 1898:65535 1898:65535 1898:65535 1899:11381 1899:65535 1899:65535 1899:65535 1900:14633 1900:65535 1900:65535 1900:65535 1901:17931 1901:65535 1901:65535 1901:65535 1902:21283 1902:65535 1902:65535 1902:65535 1903:24689 1903:65535 1903:65535 1903:65535 1904:28147 1904:65535 1904:65535 1904:65535 1905:31657 1905:65535 1905:65535 1905:65535 1906:35225 1906:65535 1906:65535 1906:65535 1907:38847 1907:65535 1907:65535 1907:65535 1908:42525 1908:65535 1908:65535 1908:65535 1909:46263 1909:65535 1909:65535 1909:65535 1910:50057 1910:65535 1910:65535 1910:65535 1911:53911 1911:65535 1911:65535 1911:65535 1912:57825 1912:65535 1912:65535 1912:65535 1913:61801 1913:65535 1913:65535 1913:65535 1914:304 1914:65535 1914:65535 1914:65535 1914:65535 1915:4404 1915:65535 1915:65535 1915:65535 1915:65535 1916:8570 1916:65535 1916:65535 1916:65535 1916:65535 1917:12798 1917:65535 1917:65535 1917:65535 1917:65535 1918:17094 1918:65535 1918:65535 1918:65535 1918:65535 1919:21458 1919:65535 1919:65535 1919:65535 1919:65535 1920:25890 1920:65535 1920:65535 1920:65535 1920:65535 1921:30390 1921:65535 1921:65535 1921:65535 1921:65535 1922:34960 1922:65535 1922:65535 1922:65535 1922:65535 1923:39602 1923:65535 1923:65535 1923:65535 1923:65535 1924:44316 1924:65535 1924:65535 1924:65535 1924:65535 1925:49106 1925:65535 1925:65535 1925:65535 1925:65535 1926:53970 1926:65535 1926:65535 1926:65535 1926:65535 1927:58906 1927:65535 1927:65535 1927:65535 1927:65535 1928:63926 1928:65535 1928:65535 1928:65535 1928:65535 1929:3483 1929:65535 1929:65535 1929:65535 1929:65535 1929:65535 1930:8659 1930:65535 1930:65535 1930:65535 1930:65535 1930:65535 1931:13913 1931:65535 1931:65535 1931:65535 1931:65535 1931:65535 1932:34822", + }]; for case in cases { let mut sketch = AgentDDSketch::with_agent_defaults(); @@ -1511,11 +1556,7 @@ mod tests { if target == actual { 0.0 } else if target == 0.0 { - if actual == 0.0 { - 0.0 - } else { - f64::INFINITY - } + if actual == 0.0 { 0.0 } else { f64::INFINITY } } else if actual < target { (target - actual) / target } else { diff --git a/lib/vector-core/src/metrics/metric_matcher.rs b/lib/vector-core/src/metrics/metric_matcher.rs new file mode 100644 index 0000000000..463b5fd2a2 --- /dev/null +++ b/lib/vector-core/src/metrics/metric_matcher.rs @@ -0,0 +1,476 @@ +use std::time::Duration; + +use metrics::Key; +use regex::Regex; + +use crate::config::metrics_expiration::{ + MetricLabelMatcher, MetricLabelMatcherConfig, MetricNameMatcherConfig, PerMetricSetExpiration, +}; + +use super::recency::KeyMatcher; + +pub(super) struct MetricKeyMatcher { + name: Option, + labels: Option, +} + +impl KeyMatcher for MetricKeyMatcher { + fn matches(&self, key: &Key) -> bool { + let name_match = self.name.as_ref().is_none_or(|m| m.matches(key)); + let labels_match = self.labels.as_ref().is_none_or(|l| l.matches(key)); + name_match && labels_match + } +} + +impl TryFrom for MetricKeyMatcher { + type Error = super::Error; + + fn try_from(value: PerMetricSetExpiration) -> Result { + Ok(Self { + name: value.name.map(TryInto::try_into).transpose()?, + labels: value.labels.map(TryInto::try_into).transpose()?, + }) + } +} + +impl TryFrom for (MetricKeyMatcher, Duration) { + type Error = super::Error; + + fn try_from(value: PerMetricSetExpiration) -> Result { + if value.expire_secs <= 0.0 { + return Err(super::Error::TimeoutMustBePositive { + timeout: value.expire_secs, + }); + } + let duration = Duration::from_secs_f64(value.expire_secs); + Ok((value.try_into()?, duration)) + } +} + +enum MetricNameMatcher { + Exact(String), + Regex(Regex), +} + +impl KeyMatcher for MetricNameMatcher { + fn matches(&self, key: &Key) -> bool { + match self { + MetricNameMatcher::Exact(name) => key.name() == name, + MetricNameMatcher::Regex(regex) => regex.is_match(key.name()), + } + } +} + +impl TryFrom for MetricNameMatcher { + type Error = super::Error; + + fn try_from(value: MetricNameMatcherConfig) -> Result { + Ok(match value { + MetricNameMatcherConfig::Exact { value } => MetricNameMatcher::Exact(value), + MetricNameMatcherConfig::Regex { pattern } => MetricNameMatcher::Regex( + Regex::new(&pattern).map_err(|_| super::Error::InvalidRegexPattern { pattern })?, + ), + }) + } +} + +enum LabelsMatcher { + Any(Vec), + All(Vec), + Exact(String, String), + Regex(String, Regex), +} + +impl KeyMatcher for LabelsMatcher { + fn matches(&self, key: &Key) -> bool { + match self { + LabelsMatcher::Any(vec) => vec.iter().any(|m| m.matches(key)), + LabelsMatcher::All(vec) => vec.iter().all(|m| m.matches(key)), + LabelsMatcher::Exact(label_key, label_value) => key + .labels() + .any(|l| l.key() == label_key && l.value() == label_value), + LabelsMatcher::Regex(label_key, regex) => key + .labels() + .any(|l| l.key() == label_key && regex.is_match(l.value())), + } + } +} + +impl TryFrom for LabelsMatcher { + type Error = super::Error; + + fn try_from(value: MetricLabelMatcher) -> Result { + Ok(match value { + MetricLabelMatcher::Exact { key, value } => Self::Exact(key, value), + MetricLabelMatcher::Regex { key, value_pattern } => Self::Regex( + key, + Regex::new(&value_pattern).map_err(|_| super::Error::InvalidRegexPattern { + pattern: value_pattern, + })?, + ), + }) + } +} + +impl TryFrom for LabelsMatcher { + type Error = super::Error; + + fn try_from(value: MetricLabelMatcherConfig) -> Result { + Ok(match value { + MetricLabelMatcherConfig::Any { matchers } => Self::Any( + matchers + .into_iter() + .map(TryInto::::try_into) + .collect::, _>>()?, + ), + MetricLabelMatcherConfig::All { matchers } => Self::All( + matchers + .into_iter() + .map(TryInto::::try_into) + .collect::, _>>()?, + ), + }) + } +} + +#[cfg(test)] +mod tests { + use metrics::Label; + use vrl::prelude::indoc; + + use super::*; + + const EMPTY: MetricKeyMatcher = MetricKeyMatcher { + name: None, + labels: None, + }; + + #[test] + fn empty_matcher_should_match_all() { + assert!(EMPTY.matches(&Key::from_name("test_name"))); + assert!(EMPTY.matches(&Key::from_parts( + "another", + [Label::new("test_key", "test_value")].iter() + ))); + } + + #[test] + fn name_matcher_should_ignore_labels() { + let matcher = MetricKeyMatcher { + name: Some(MetricNameMatcher::Exact("test_metric".to_string())), + labels: None, + }; + + assert!(matcher.matches(&Key::from_name("test_metric"))); + assert!(matcher.matches(&Key::from_parts( + "test_metric", + [Label::new("test_key", "test_value")].iter() + ))); + assert!(!matcher.matches(&Key::from_name("different_name"))); + assert!(!matcher.matches(&Key::from_parts( + "different_name", + [Label::new("test_key", "test_value")].iter() + ))); + } + + #[test] + fn exact_name_matcher_should_check_name() { + let matcher = MetricKeyMatcher { + name: Some(MetricNameMatcher::Exact("test_metric".to_string())), + labels: None, + }; + + assert!(matcher.matches(&Key::from_name("test_metric"))); + assert!(!matcher.matches(&Key::from_name("different_name"))); + assert!(!matcher.matches(&Key::from_name("_test_metric"))); + assert!(!matcher.matches(&Key::from_name("test_metric123"))); + } + + #[test] + fn regex_name_matcher_should_try_matching_the_name() { + let matcher = MetricKeyMatcher { + name: Some(MetricNameMatcher::Regex( + Regex::new(r".*test_?metric.*").unwrap(), + )), + labels: None, + }; + + assert!(matcher.matches(&Key::from_name("test_metric"))); + assert!(!matcher.matches(&Key::from_name("different_name"))); + assert!(matcher.matches(&Key::from_name("_test_metric"))); + assert!(matcher.matches(&Key::from_name("test_metric123"))); + assert!(matcher.matches(&Key::from_name("__testmetric123"))); + } + + #[test] + fn exact_label_matcher_should_look_for_exact_label_match() { + let matcher = MetricKeyMatcher { + name: None, + labels: Some(LabelsMatcher::Exact( + "test_key".to_string(), + "test_value".to_string(), + )), + }; + + assert!(!matcher.matches(&Key::from_name("test_metric"))); + assert!(matcher.matches(&Key::from_parts( + "test_metric", + [Label::new("test_key", "test_value")].iter() + ))); + assert!(!matcher.matches(&Key::from_name("different_name"))); + assert!(matcher.matches(&Key::from_parts( + "different_name", + [Label::new("test_key", "test_value")].iter() + ))); + } + + #[test] + fn regex_label_matcher_should_look_for_exact_label_match() { + let matcher = MetricKeyMatcher { + name: None, + labels: Some(LabelsMatcher::Regex( + "test_key".to_string(), + Regex::new(r"metric_val.*").unwrap(), + )), + }; + + assert!(!matcher.matches(&Key::from_name("test_metric"))); + assert!(matcher.matches(&Key::from_parts( + "test_metric", + [Label::new("test_key", "metric_value123")].iter() + ))); + assert!(!matcher.matches(&Key::from_parts( + "test_metric", + [Label::new("test_key", "test_value123")].iter() + ))); + assert!(matcher.matches(&Key::from_parts( + "different_name", + [Label::new("test_key", "metric_val0")].iter() + ))); + } + + #[test] + fn any_label_matcher_should_look_for_at_least_one_match() { + let matcher = MetricKeyMatcher { + name: None, + labels: Some(LabelsMatcher::Any(vec![ + LabelsMatcher::Regex("test_key".to_string(), Regex::new(r"metric_val.*").unwrap()), + LabelsMatcher::Exact("test_key".to_string(), "test_value".to_string()), + ])), + }; + + assert!(!matcher.matches(&Key::from_name("test_metric"))); + assert!(matcher.matches(&Key::from_parts( + "test_metric", + [Label::new("test_key", "metric_value123")].iter() + ))); + assert!(matcher.matches(&Key::from_parts( + "test_metric", + [Label::new("test_key", "test_value")].iter() + ))); + assert!(matcher.matches(&Key::from_parts( + "different_name", + [Label::new("test_key", "metric_val0")].iter() + ))); + assert!(!matcher.matches(&Key::from_parts( + "different_name", + [Label::new("test_key", "different_value")].iter() + ))); + } + + #[test] + fn all_label_matcher_should_expect_all_matches() { + let matcher = MetricKeyMatcher { + name: None, + labels: Some(LabelsMatcher::All(vec![ + LabelsMatcher::Regex("key_one".to_string(), Regex::new(r"metric_val.*").unwrap()), + LabelsMatcher::Exact("key_two".to_string(), "test_value".to_string()), + ])), + }; + + assert!(!matcher.matches(&Key::from_name("test_metric"))); + assert!(!matcher.matches(&Key::from_parts( + "test_metric", + [Label::new("key_one", "metric_value123")].iter() + ))); + assert!(!matcher.matches(&Key::from_parts( + "test_metric", + [Label::new("key_two", "test_value")].iter() + ))); + assert!( + matcher.matches(&Key::from_parts( + "different_name", + [ + Label::new("key_one", "metric_value_1234"), + Label::new("key_two", "test_value") + ] + .iter() + )) + ); + } + + #[test] + fn matcher_with_both_name_and_label_should_expect_both_to_match() { + let matcher = MetricKeyMatcher { + name: Some(MetricNameMatcher::Exact("test_metric".to_string())), + labels: Some(LabelsMatcher::Exact( + "test_key".to_string(), + "test_value".to_string(), + )), + }; + + assert!(!matcher.matches(&Key::from_name("test_metric"))); + assert!(!matcher.matches(&Key::from_name("different_name"))); + assert!(!matcher.matches(&Key::from_parts( + "different_name", + [Label::new("test_key", "test_value")].iter() + ))); + assert!(matcher.matches(&Key::from_parts( + "test_metric", + [Label::new("test_key", "test_value")].iter() + ))); + } + + #[test] + fn complex_matcher_rules() { + let matcher = MetricKeyMatcher { + name: Some(MetricNameMatcher::Regex(Regex::new(r"custom_.*").unwrap())), + labels: Some(LabelsMatcher::All(vec![ + // Let's match just sink metrics + LabelsMatcher::Exact("component_kind".to_string(), "sink".to_string()), + // And only AWS components + LabelsMatcher::Regex("component_type".to_string(), Regex::new(r"aws_.*").unwrap()), + // And some more rules + LabelsMatcher::Any(vec![ + LabelsMatcher::Exact("region".to_string(), "some_aws_region_name".to_string()), + LabelsMatcher::Regex( + "endpoint".to_string(), + Regex::new(r"test.com.*").unwrap(), + ), + ]), + ])), + }; + + assert!(!matcher.matches(&Key::from_name("test_metric"))); + assert!(!matcher.matches(&Key::from_name("custom_metric_a"))); + assert!(!matcher.matches(&Key::from_parts( + "custom_metric_with_missing_component_type", + [Label::new("component_kind", "sink")].iter() + ))); + assert!( + !matcher.matches(&Key::from_parts( + "custom_metric_with_missing_extra_labels", + [ + Label::new("component_kind", "sink"), + Label::new("component_type", "aws_cloudwatch_metrics") + ] + .iter() + )) + ); + assert!( + !matcher.matches(&Key::from_parts( + "custom_metric_with_wrong_region", + [ + Label::new("component_kind", "sink"), + Label::new("component_type", "aws_cloudwatch_metrics"), + Label::new("region", "some_other_region") + ] + .iter() + )) + ); + assert!( + !matcher.matches(&Key::from_parts( + "custom_metric_with_wrong_region_and_endpoint", + [ + Label::new("component_kind", "sink"), + Label::new("component_type", "aws_cloudwatch_metrics"), + Label::new("region", "some_other_region"), + Label::new("endpoint", "wrong_endpoint.com/metrics") + ] + .iter() + )) + ); + assert!( + matcher.matches(&Key::from_parts( + "custom_metric_with_wrong_endpoint_but_correct_region", + [ + Label::new("component_kind", "sink"), + Label::new("component_type", "aws_cloudwatch_metrics"), + Label::new("region", "some_aws_region_name"), + Label::new("endpoint", "wrong_endpoint.com/metrics") + ] + .iter() + )) + ); + assert!( + matcher.matches(&Key::from_parts( + "custom_metric_with_wrong_region_but_correct_endpoint", + [ + Label::new("component_kind", "sink"), + Label::new("component_type", "aws_cloudwatch_metrics"), + Label::new("region", "some_other_region"), + Label::new("endpoint", "test.com/metrics") + ] + .iter() + )) + ); + assert!( + !matcher.matches(&Key::from_parts( + "custom_metric_with_wrong_component_kind", + [ + Label::new("component_kind", "source"), + Label::new("component_type", "aws_cloudwatch_metrics"), + Label::new("region", "some_other_region"), + Label::new("endpoint", "test.com/metrics") + ] + .iter() + )) + ); + } + + #[test] + fn parse_simple_config_into_matcher() { + let config = serde_yaml::from_str::(indoc! {r#" + name: + type: "exact" + value: "test_metric" + labels: + type: "all" + matchers: + - type: "exact" + key: "component_kind" + value: "sink" + - type: "regex" + key: "component_type" + value_pattern: "aws_.*" + expire_secs: 1.0 + "#}) + .unwrap(); + + let matcher: MetricKeyMatcher = config.try_into().unwrap(); + + if let Some(MetricNameMatcher::Exact(value)) = matcher.name { + assert_eq!("test_metric", value); + } else { + panic!("Expected exact name matcher"); + } + + let Some(LabelsMatcher::All(all_matchers)) = matcher.labels else { + panic!("Expected main label matcher to be an all matcher"); + }; + + assert_eq!(2, all_matchers.len()); + if let LabelsMatcher::Exact(key, value) = &all_matchers[0] { + assert_eq!("component_kind", key); + assert_eq!("sink", value); + } else { + panic!("Expected first label matcher to be an exact matcher"); + } + if let LabelsMatcher::Regex(key, regex) = &all_matchers[1] { + assert_eq!("component_type", key); + assert_eq!("aws_.*", regex.as_str()); + } else { + panic!("Expected second label matcher to be a regex matcher"); + } + } +} diff --git a/lib/vector-core/src/metrics/mod.rs b/lib/vector-core/src/metrics/mod.rs index cb853f17f1..0a8ea11871 100644 --- a/lib/vector-core/src/metrics/mod.rs +++ b/lib/vector-core/src/metrics/mod.rs @@ -1,5 +1,6 @@ mod ddsketch; mod label_filter; +mod metric_matcher; mod recency; mod recorder; mod storage; @@ -7,6 +8,7 @@ mod storage; use std::{sync::OnceLock, time::Duration}; use chrono::Utc; +use metric_matcher::MetricKeyMatcher; use metrics::Key; use metrics_tracing_context::TracingContextLayer; use metrics_util::layers::Layer; @@ -14,11 +16,14 @@ use snafu::Snafu; pub use self::ddsketch::{AgentDDSketch, BinMap, Config}; use self::{label_filter::VectorLabelFilter, recorder::Registry, recorder::VectorRecorder}; -use crate::event::{Metric, MetricValue}; +use crate::{ + config::metrics_expiration::PerMetricSetExpiration, + event::{Metric, MetricValue}, +}; type Result = std::result::Result; -#[derive(Clone, Copy, Debug, PartialEq, Snafu)] +#[derive(Clone, Debug, PartialEq, Snafu)] pub enum Error { #[snafu(display("Recorder already initialized."))] AlreadyInitialized, @@ -26,6 +31,8 @@ pub enum Error { NotInitialized, #[snafu(display("Timeout value of {} must be positive.", timeout))] TimeoutMustBePositive { timeout: f64 }, + #[snafu(display("Invalid regex pattern: {}.", pattern))] + InvalidRegexPattern { pattern: String }, } static CONTROLLER: OnceLock = OnceLock::new(); @@ -142,14 +149,27 @@ impl Controller { /// # Errors /// /// The contained timeout value must be positive. - pub fn set_expiry(&self, timeout: Option) -> Result<()> { - if let Some(timeout) = timeout { - if timeout <= 0.0 { - return Err(Error::TimeoutMustBePositive { timeout }); - } + pub fn set_expiry( + &self, + global_timeout: Option, + expire_metrics_per_metric_set: Vec, + ) -> Result<()> { + if let Some(timeout) = global_timeout + && timeout <= 0.0 + { + return Err(Error::TimeoutMustBePositive { timeout }); } - self.recorder - .with_registry(|registry| registry.set_expiry(timeout.map(Duration::from_secs_f64))); + let per_metric_expiration = expire_metrics_per_metric_set + .into_iter() + .map(TryInto::try_into) + .collect::>>()?; + + self.recorder.with_registry(|registry| { + registry.set_expiry( + global_timeout.map(Duration::from_secs_f64), + per_metric_expiration, + ); + }); Ok(()) } @@ -225,7 +245,12 @@ macro_rules! update_counter { mod tests { use super::*; - use crate::event::MetricKind; + use crate::{ + config::metrics_expiration::{ + MetricLabelMatcher, MetricLabelMatcherConfig, MetricNameMatcherConfig, + }, + event::MetricKind, + }; const IDLE_TIMEOUT: f64 = 0.5; @@ -283,7 +308,9 @@ mod tests { #[test] fn expires_metrics() { let controller = init_metrics(); - controller.set_expiry(Some(IDLE_TIMEOUT)).unwrap(); + controller + .set_expiry(Some(IDLE_TIMEOUT), Vec::new()) + .unwrap(); metrics::counter!("test2").increment(1); metrics::counter!("test3").increment(2); @@ -297,7 +324,9 @@ mod tests { #[test] fn expires_metrics_tags() { let controller = init_metrics(); - controller.set_expiry(Some(IDLE_TIMEOUT)).unwrap(); + controller + .set_expiry(Some(IDLE_TIMEOUT), Vec::new()) + .unwrap(); metrics::counter!("test4", "tag" => "value1").increment(1); metrics::counter!("test4", "tag" => "value2").increment(2); @@ -311,7 +340,9 @@ mod tests { #[test] fn skips_expiring_registered() { let controller = init_metrics(); - controller.set_expiry(Some(IDLE_TIMEOUT)).unwrap(); + controller + .set_expiry(Some(IDLE_TIMEOUT), Vec::new()) + .unwrap(); let a = metrics::counter!("test5"); metrics::counter!("test6").increment(5); @@ -334,4 +365,76 @@ mod tests { value => panic!("Invalid metric value {value:?}"), } } + + #[test] + fn expires_metrics_per_set() { + let controller = init_metrics(); + controller + .set_expiry( + None, + vec![PerMetricSetExpiration { + name: Some(MetricNameMatcherConfig::Exact { + value: "test3".to_string(), + }), + labels: None, + expire_secs: IDLE_TIMEOUT, + }], + ) + .unwrap(); + + metrics::counter!("test2").increment(1); + metrics::counter!("test3").increment(2); + assert_eq!(controller.capture_metrics().len(), 4); + + std::thread::sleep(Duration::from_secs_f64(IDLE_TIMEOUT * 2.0)); + metrics::counter!("test2").increment(3); + assert_eq!(controller.capture_metrics().len(), 3); + } + + #[test] + fn expires_metrics_multiple_different_sets() { + let controller = init_metrics(); + controller + .set_expiry( + Some(IDLE_TIMEOUT * 3.0), + vec![ + PerMetricSetExpiration { + name: Some(MetricNameMatcherConfig::Exact { + value: "test3".to_string(), + }), + labels: None, + expire_secs: IDLE_TIMEOUT, + }, + PerMetricSetExpiration { + name: None, + labels: Some(MetricLabelMatcherConfig::All { + matchers: vec![MetricLabelMatcher::Exact { + key: "tag".to_string(), + value: "value1".to_string(), + }], + }), + expire_secs: IDLE_TIMEOUT * 2.0, + }, + ], + ) + .unwrap(); + + metrics::counter!("test1").increment(1); + metrics::counter!("test2").increment(1); + metrics::counter!("test3").increment(2); + metrics::counter!("test4", "tag" => "value1").increment(3); + assert_eq!(controller.capture_metrics().len(), 6); + + std::thread::sleep(Duration::from_secs_f64(IDLE_TIMEOUT * 1.5)); + metrics::counter!("test2").increment(3); + assert_eq!(controller.capture_metrics().len(), 5); + + std::thread::sleep(Duration::from_secs_f64(IDLE_TIMEOUT)); + metrics::counter!("test2").increment(3); + assert_eq!(controller.capture_metrics().len(), 4); + + std::thread::sleep(Duration::from_secs_f64(IDLE_TIMEOUT)); + metrics::counter!("test2").increment(3); + assert_eq!(controller.capture_metrics().len(), 3); + } } diff --git a/lib/vector-core/src/metrics/recency.rs b/lib/vector-core/src/metrics/recency.rs index d6f5fb9249..2207149fd8 100644 --- a/lib/vector-core/src/metrics/recency.rs +++ b/lib/vector-core/src/metrics/recency.rs @@ -48,14 +48,14 @@ //! not, and thus whether it should actually be deleted. use std::collections::HashMap; -use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::Duration; -use metrics::{atomics::AtomicU64, Counter, CounterFn, Gauge, GaugeFn, HistogramFn}; +use metrics::{Counter, CounterFn, Gauge, GaugeFn, HistogramFn, atomics::AtomicU64}; use metrics_util::{ - registry::{Registry, Storage}, Hashable, MetricKind, MetricKindMask, + registry::{Registry, Storage}, }; use parking_lot::Mutex; use quanta::{Clock, Instant}; @@ -82,7 +82,7 @@ pub(super) struct Generation(usize); #[derive(Clone)] pub(super) struct Generational { inner: T, - gen: Arc, + generation: Arc, } impl Generational { @@ -90,7 +90,7 @@ impl Generational { fn new(inner: T) -> Generational { Generational { inner, - gen: Arc::new(AtomicUsize::new(0)), + generation: Arc::new(AtomicUsize::new(0)), } } @@ -101,7 +101,7 @@ impl Generational { /// Gets the current generation. pub(super) fn get_generation(&self) -> Generation { - Generation(self.gen.load(Ordering::Acquire)) + Generation(self.generation.load(Ordering::Acquire)) } /// Acquires a reference to the inner value, and increments the generation. @@ -110,7 +110,7 @@ impl Generational { F: Fn(&T) -> V, { let result = f(&self.inner); - _ = self.gen.fetch_add(1, Ordering::AcqRel); + _ = self.generation.fetch_add(1, Ordering::AcqRel); result } } @@ -216,6 +216,39 @@ impl> Storage for GenerationalStorage { } } +pub(super) trait KeyMatcher { + fn matches(&self, key: &K) -> bool; +} + +struct PerSetTimeout> { + configuration: Vec<(M, Duration)>, + per_key_timeouts: HashMap>, +} + +impl PerSetTimeout +where + K: Clone + Eq + Hashable, + M: KeyMatcher, +{ + fn new(configuration: Vec<(M, Duration)>) -> Self { + Self { + configuration, + per_key_timeouts: HashMap::new(), + } + } + + fn get_timeout_for_key(&mut self, key: &K, default: Option) -> Option { + *self.per_key_timeouts.entry(key.clone()).or_insert_with(|| { + for (matcher, duration) in &self.configuration { + if matcher.matches(key) { + return Some(*duration); + } + } + default + }) + } +} + /// Tracks recency of metric updates by their registry generation and time. /// /// In many cases, a user may have a long-running process where metrics are stored over time using @@ -229,20 +262,22 @@ impl> Storage for GenerationalStorage { /// /// [`Recency`] is separate from [`Registry`] specifically to avoid imposing any slowdowns when /// tracking recency does not matter, despite their otherwise tight coupling. -pub(super) struct Recency { +pub(super) struct Recency> { mask: MetricKindMask, inner: Mutex<(Clock, HashMap)>, - idle_timeout: Option, + global_idle_timeout: Option, + per_set_timeouts: Mutex>, } -impl Recency +impl Recency where - K: Clone + Eq + Hashable, + K: Clone + Eq + Hashable + std::fmt::Debug, + M: KeyMatcher, { /// Creates a new [`Recency`]. /// - /// If `idle_timeout` is `None`, no recency checking will occur. Otherwise, any metric that has - /// not been updated for longer than `idle_timeout` will be subject for deletion the next time + /// If `global_idle_timeout` is `None`, no recency checking will occur. Otherwise, any metric that has + /// not been updated for longer than `global_idle_timeout` will be subject for deletion the next time /// the metric is checked. /// /// The provided `clock` is used for tracking time, while `mask` controls which metrics @@ -251,11 +286,17 @@ where /// /// Refer to the documentation for [`MetricKindMask`](crate::MetricKindMask) for more /// information on defining a metric kind mask. - pub(super) fn new(clock: Clock, mask: MetricKindMask, idle_timeout: Option) -> Self { + pub(super) fn new( + clock: Clock, + mask: MetricKindMask, + global_idle_timeout: Option, + per_set_timeouts: Vec<(M, Duration)>, + ) -> Self { Recency { mask, inner: Mutex::new((clock, HashMap::new())), - idle_timeout, + global_idle_timeout, + per_set_timeouts: Mutex::new(PerSetTimeout::new(per_set_timeouts)), } } @@ -343,47 +384,50 @@ where F: Fn(&Registry, &K) -> bool, S: Storage, { - let gen = value.get_generation(); - if let Some(idle_timeout) = self.idle_timeout { - if self.mask.matches(kind) { - let mut guard = self.inner.lock(); - let (clock, entries) = &mut *guard; - - let now = clock.now(); - let deleted = if let Some((last_gen, last_update)) = entries.get_mut(key) { - // If the value is the same as the latest value we have internally, and - // we're over the idle timeout period, then remove it and continue. - if *last_gen == gen { - // We don't want to delete the metric if there is an outstanding handle that - // could later update the shared value. So, here we look up the count of - // references to the inner value to see if there are more than expected. - // - // The magic value for `strong_count` below comes from: - // 1. The reference in the registry - // 2. The reference held by the value passed in here - // If there is another reference, then there is handle elsewhere. - let referenced = Arc::strong_count(&value.inner) > 2; - // If the delete returns false, that means that our generation counter is - // out-of-date, and that the metric has been updated since, so we don't - // actually want to delete it yet. - !referenced - && (now - *last_update) > idle_timeout - && delete_op(registry, key) - } else { - // Value has changed, so mark it such. - *last_update = now; - *last_gen = gen; - false - } + let key_timeout = self + .per_set_timeouts + .lock() + .get_timeout_for_key(key, self.global_idle_timeout); + + let generation = value.get_generation(); + if let Some(timeout) = key_timeout + && self.mask.matches(kind) + { + let mut guard = self.inner.lock(); + let (clock, entries) = &mut *guard; + + let now = clock.now(); + let deleted = if let Some((last_gen, last_update)) = entries.get_mut(key) { + // If the value is the same as the latest value we have internally, and + // we're over the idle timeout period, then remove it and continue. + if *last_gen == generation { + // We don't want to delete the metric if there is an outstanding handle that + // could later update the shared value. So, here we look up the count of + // references to the inner value to see if there are more than expected. + // + // The magic value for `strong_count` below comes from: + // 1. The reference in the registry + // 2. The reference held by the value passed in here + // If there is another reference, then there is handle elsewhere. + let referenced = Arc::strong_count(&value.inner) > 2; + // If the delete returns false, that means that our generation counter is + // out-of-date, and that the metric has been updated since, so we don't + // actually want to delete it yet. + !referenced && (now - *last_update) > timeout && delete_op(registry, key) } else { - entries.insert(key.clone(), (gen, now)); + // Value has changed, so mark it such. + *last_update = now; + *last_gen = generation; false - }; - - if deleted { - entries.remove(key); - return false; } + } else { + entries.insert(key.clone(), (generation, now)); + false + }; + + if deleted { + entries.remove(key); + return false; } } diff --git a/lib/vector-core/src/metrics/recorder.rs b/lib/vector-core/src/metrics/recorder.rs index 291e129043..b53dcc9faa 100644 --- a/lib/vector-core/src/metrics/recorder.rs +++ b/lib/vector-core/src/metrics/recorder.rs @@ -1,11 +1,12 @@ -use std::sync::{atomic::Ordering, Arc, RwLock}; +use std::sync::{Arc, RwLock, atomic::Ordering}; use std::{cell::OnceCell, time::Duration}; use chrono::Utc; use metrics::{Counter, Gauge, Histogram, Key, KeyName, Metadata, Recorder, SharedString, Unit}; -use metrics_util::{registry::Registry as MetricsRegistry, MetricKindMask}; +use metrics_util::{MetricKindMask, registry::Registry as MetricsRegistry}; use quanta::Clock; +use super::metric_matcher::MetricKeyMatcher; use super::recency::{GenerationalStorage, Recency}; use super::storage::VectorStorage; use crate::event::{Metric, MetricValue}; @@ -15,7 +16,7 @@ thread_local!(static LOCAL_REGISTRY: OnceCell = const { OnceCell::new( #[allow(dead_code)] pub(super) struct Registry { registry: MetricsRegistry>, - recency: RwLock>>, + recency: RwLock>>, } impl Registry { @@ -30,8 +31,21 @@ impl Registry { self.registry.clear(); } - pub(super) fn set_expiry(&self, timeout: Option) { - let recency = timeout.map(|_| Recency::new(Clock::new(), MetricKindMask::ALL, timeout)); + pub(super) fn set_expiry( + &self, + global_timeout: Option, + expire_metrics_per_metric_set: Vec<(MetricKeyMatcher, Duration)>, + ) { + let recency = if global_timeout.is_none() && expire_metrics_per_metric_set.is_empty() { + None + } else { + Some(Recency::new( + Clock::new(), + MetricKindMask::ALL, + global_timeout, + expire_metrics_per_metric_set, + )) + }; *(self.recency.write()).expect("Failed to acquire write lock on recency map") = recency; } @@ -46,9 +60,9 @@ impl Registry { let recency = recency.as_ref(); for (key, counter) in self.registry.get_counter_handles() { - if recency.map_or(true, |recency| { - recency.should_store_counter(&key, &counter, &self.registry) - }) { + if recency + .is_none_or(|recency| recency.should_store_counter(&key, &counter, &self.registry)) + { // NOTE this will truncate if the value is greater than 2**52. #[allow(clippy::cast_precision_loss)] let value = counter.get_inner().load(Ordering::Relaxed) as f64; @@ -57,16 +71,16 @@ impl Registry { } } for (key, gauge) in self.registry.get_gauge_handles() { - if recency.map_or(true, |recency| { - recency.should_store_gauge(&key, &gauge, &self.registry) - }) { + if recency + .is_none_or(|recency| recency.should_store_gauge(&key, &gauge, &self.registry)) + { let value = gauge.get_inner().load(Ordering::Relaxed); let value = MetricValue::Gauge { value }; metrics.push(Metric::from_metric_kv(&key, value, timestamp)); } } for (key, histogram) in self.registry.get_histogram_handles() { - if recency.map_or(true, |recency| { + if recency.is_none_or(|recency| { recency.should_store_histogram(&key, &histogram, &self.registry) }) { let value = histogram.get_inner().make_metric(); diff --git a/lib/vector-core/src/metrics/storage.rs b/lib/vector-core/src/metrics/storage.rs index d8955c11d3..b102849a6f 100644 --- a/lib/vector-core/src/metrics/storage.rs +++ b/lib/vector-core/src/metrics/storage.rs @@ -1,12 +1,12 @@ use std::sync::{ - atomic::{AtomicU32, Ordering}, Arc, + atomic::{AtomicU32, Ordering}, }; -use metrics::{atomics::AtomicU64, GaugeFn, HistogramFn}; +use metrics::{GaugeFn, HistogramFn, atomics::AtomicU64}; use metrics_util::registry::Storage; -use crate::event::{metric::Bucket, MetricValue}; +use crate::event::{MetricValue, metric::Bucket}; pub(super) struct VectorStorage; diff --git a/lib/vector-core/src/schema/definition.rs b/lib/vector-core/src/schema/definition.rs index aff0522606..904b00cb63 100644 --- a/lib/vector-core/src/schema/definition.rs +++ b/lib/vector-core/src/schema/definition.rs @@ -1,10 +1,10 @@ use std::collections::{BTreeMap, BTreeSet}; use lookup::lookup_v2::TargetPath; -use lookup::{owned_value_path, OwnedTargetPath, OwnedValuePath, PathPrefix}; -use vrl::value::{kind::Collection, Kind}; +use lookup::{OwnedTargetPath, OwnedValuePath, PathPrefix, owned_value_path}; +use vrl::value::{Kind, kind::Collection}; -use crate::config::{log_schema, LegacyKey, LogNamespace}; +use crate::config::{LegacyKey, LogNamespace, log_schema}; /// The definition of a schema. /// @@ -552,7 +552,8 @@ mod test_utils { let actual_kind = Kind::from(log.value()); if let Err(path) = self.event_kind.is_superset(&actual_kind) { - return Result::Err(format!("Event value doesn't match at path: {}\n\nEvent type at path = {:?}\n\nDefinition at path = {:?}", + return Result::Err(format!( + "Event value doesn't match at path: {}\n\nEvent type at path = {:?}\n\nDefinition at path = {:?}", path, actual_kind.at_path(&path).debug_info(), self.event_kind.at_path(&path).debug_info() diff --git a/lib/vector-core/src/serde.rs b/lib/vector-core/src/serde.rs index b32b22f862..0f3b1b7efd 100644 --- a/lib/vector-core/src/serde.rs +++ b/lib/vector-core/src/serde.rs @@ -1,6 +1,6 @@ use std::{fmt, marker::PhantomData}; -use serde::{de, Deserialize, Deserializer}; +use serde::{Deserialize, Deserializer, de}; /// Answers "Is this value in it's default state?" which can be used to skip serializing the value. #[inline] @@ -66,7 +66,7 @@ where /// } /// ``` pub mod ascii_char { - use serde::{de, Deserialize, Deserializer, Serializer}; + use serde::{Deserialize, Deserializer, Serializer, de}; /// Deserialize an ASCII character as `u8`. /// diff --git a/lib/vector-core/src/sink.rs b/lib/vector-core/src/sink.rs index 436ecc2cf7..8170ef46f8 100644 --- a/lib/vector-core/src/sink.rs +++ b/lib/vector-core/src/sink.rs @@ -1,8 +1,8 @@ use std::{fmt, iter::IntoIterator, pin::Pin}; -use futures::{stream, task::Context, task::Poll, Sink, SinkExt, Stream, StreamExt}; +use futures::{Sink, SinkExt, Stream, StreamExt, stream, task::Context, task::Poll}; -use crate::event::{into_event_stream, Event, EventArray, EventContainer}; +use crate::event::{Event, EventArray, EventContainer, into_event_stream}; pub enum VectorSink { Sink(Box + Send + Unpin>), diff --git a/lib/vector-core/src/test_util.rs b/lib/vector-core/src/test_util.rs index c404726ba1..a3d2b2949c 100644 --- a/lib/vector-core/src/test_util.rs +++ b/lib/vector-core/src/test_util.rs @@ -4,7 +4,7 @@ use std::{ task::{Context, Poll}, }; -use futures::{task::noop_waker_ref, Stream, StreamExt}; +use futures::{Stream, StreamExt, task::noop_waker_ref}; use crate::event::{Event, EventArray, EventContainer}; diff --git a/lib/vector-core/src/tls/incoming.rs b/lib/vector-core/src/tls/incoming.rs index d13c10717d..592a155a61 100644 --- a/lib/vector-core/src/tls/incoming.rs +++ b/lib/vector-core/src/tls/incoming.rs @@ -8,7 +8,7 @@ use std::{ task::{Context, Poll}, }; -use futures::{future::BoxFuture, stream, FutureExt, Stream}; +use futures::{FutureExt, Stream, future::BoxFuture, stream}; use openssl::ssl::{Ssl, SslAcceptor, SslMethod}; use openssl::x509::X509; use snafu::ResultExt; @@ -18,7 +18,7 @@ use tokio::{ net::{TcpListener, TcpStream}, }; use tokio_openssl::SslStream; -use tonic::transport::{server::Connected, Certificate}; +use tonic::transport::{Certificate, server::Connected}; use super::{ CreateAcceptorSnafu, HandshakeSnafu, IncomingListenerSnafu, MaybeTlsSettings, MaybeTlsStream, @@ -234,8 +234,8 @@ impl MaybeTlsIncomingStream { use super::MaybeTls; match &mut self.state { - StreamState::Accepted(ref mut stream) => Some(match stream { - MaybeTls::Raw(ref mut s) => s, + StreamState::Accepted(stream) => Some(match stream { + MaybeTls::Raw(s) => s, MaybeTls::Tls(s) => s.get_mut(), }), StreamState::Accepting(_) | StreamState::AcceptError(_) | StreamState::Closed => None, @@ -320,13 +320,13 @@ impl MaybeTlsIncomingStream { continue; } Err(error) => { - let error = io::Error::new(io::ErrorKind::Other, error); + let error = io::Error::other(error); this.state = StreamState::AcceptError(error.to_string()); Poll::Ready(Err(error)) } }, StreamState::AcceptError(error) => { - Poll::Ready(Err(io::Error::new(io::ErrorKind::Other, error.clone()))) + Poll::Ready(Err(io::Error::other(error.clone()))) } StreamState::Closed => Poll::Ready(Err(io::ErrorKind::BrokenPipe.into())), }; @@ -369,14 +369,12 @@ impl AsyncWrite for MaybeTlsIncomingStream { Poll::Pending } Err(error) => { - let error = io::Error::new(io::ErrorKind::Other, error); + let error = io::Error::other(error); this.state = StreamState::AcceptError(error.to_string()); Poll::Ready(Err(error)) } }, - StreamState::AcceptError(error) => { - Poll::Ready(Err(io::Error::new(io::ErrorKind::Other, error.clone()))) - } + StreamState::AcceptError(error) => Poll::Ready(Err(io::Error::other(error.clone()))), StreamState::Closed => Poll::Ready(Ok(())), } } diff --git a/lib/vector-core/src/tls/mod.rs b/lib/vector-core/src/tls/mod.rs index 9e065cb212..6cc029e44b 100644 --- a/lib/vector-core/src/tls/mod.rs +++ b/lib/vector-core/src/tls/mod.rs @@ -21,9 +21,9 @@ mod settings; pub use incoming::{CertificateMetadata, MaybeTlsIncomingStream, MaybeTlsListener}; pub use maybe_tls::MaybeTls; pub use settings::{ - MaybeTlsSettings, TlsConfig, TlsEnableableConfig, TlsSettings, TlsSourceConfig, - PEM_START_MARKER, TEST_PEM_CA_PATH, TEST_PEM_CLIENT_CRT_PATH, TEST_PEM_CLIENT_KEY_PATH, - TEST_PEM_CRT_PATH, TEST_PEM_INTERMEDIATE_CA_PATH, TEST_PEM_KEY_PATH, + MaybeTlsSettings, PEM_START_MARKER, TEST_PEM_CA_PATH, TEST_PEM_CLIENT_CRT_PATH, + TEST_PEM_CLIENT_KEY_PATH, TEST_PEM_CRT_PATH, TEST_PEM_INTERMEDIATE_CA_PATH, TEST_PEM_KEY_PATH, + TlsConfig, TlsEnableableConfig, TlsSettings, TlsSourceConfig, }; pub type Result = std::result::Result; diff --git a/lib/vector-core/src/tls/outgoing.rs b/lib/vector-core/src/tls/outgoing.rs index 295b4f5643..37e02f22a0 100644 --- a/lib/vector-core/src/tls/outgoing.rs +++ b/lib/vector-core/src/tls/outgoing.rs @@ -5,7 +5,7 @@ use tokio::net::TcpStream; use tokio_openssl::SslStream; use super::{ - tls_connector, ConnectSnafu, HandshakeSnafu, MaybeTlsSettings, MaybeTlsStream, SslBuildSnafu, + ConnectSnafu, HandshakeSnafu, MaybeTlsSettings, MaybeTlsStream, SslBuildSnafu, tls_connector, }; impl MaybeTlsSettings { diff --git a/lib/vector-core/src/tls/settings.rs b/lib/vector-core/src/tls/settings.rs index 0ec1a18e93..f5e193d8d7 100644 --- a/lib/vector-core/src/tls/settings.rs +++ b/lib/vector-core/src/tls/settings.rs @@ -1,19 +1,19 @@ -use std::{ - fmt, - fs::File, - io::Read, - path::{Path, PathBuf}, -}; - +use cfg_if::cfg_if; use lookup::lookup_v2::OptionalValuePath; use openssl::{ pkcs12::{ParsedPkcs12_2, Pkcs12}, pkey::{PKey, Private}, - ssl::{select_next_proto, AlpnError, ConnectConfiguration, SslContextBuilder, SslVerifyMode}, + ssl::{AlpnError, ConnectConfiguration, SslContextBuilder, SslVerifyMode, select_next_proto}, stack::Stack, - x509::{store::X509StoreBuilder, X509}, + x509::{X509, store::X509StoreBuilder}, }; use snafu::ResultExt; +use std::{ + fmt, + fs::File, + io::Read, + path::{Path, PathBuf}, +}; use vector_config::configurable_component; use super::{ @@ -41,8 +41,9 @@ pub const TEST_PEM_CLIENT_KEY_PATH: &str = #[configurable_component] #[configurable(metadata(docs::advanced))] #[derive(Clone, Debug, Default)] +#[serde(deny_unknown_fields)] pub struct TlsEnableableConfig { - /// Whether or not to require TLS for incoming or outgoing connections. + /// Whether to require TLS for incoming or outgoing connections. /// /// When enabled and used for incoming connections, an identity certificate is also required. See `tls.crt_file` for /// more information. @@ -68,7 +69,7 @@ impl TlsEnableableConfig { } } -/// TlsEnableableConfig for `sources`, adding metadata from the client certificate. +/// `TlsEnableableConfig` for `sources`, adding metadata from the client certificate. #[configurable_component] #[derive(Clone, Debug, Default)] pub struct TlsSourceConfig { @@ -92,7 +93,7 @@ pub struct TlsConfig { /// If enabled, certificates must not be expired and must be issued by a trusted /// issuer. This verification operates in a hierarchical manner, checking that the leaf certificate (the /// certificate presented by the client/server) is not only valid, but that the issuer of that certificate is also valid, and - /// so on until the verification process reaches a root certificate. + /// so on, until the verification process reaches a root certificate. /// /// Do NOT set this to `false` unless you understand the risks of not verifying the validity of certificates. pub verify_certificate: Option, @@ -109,7 +110,7 @@ pub struct TlsConfig { /// Sets the list of supported ALPN protocols. /// - /// Declare the supported ALPN protocols, which are used during negotiation with peer. They are prioritized in the order + /// Declare the supported ALPN protocols, which are used during negotiation with a peer. They are prioritized in the order /// that they are defined. #[configurable(metadata(docs::examples = "h2"))] pub alpn_protocols: Option>, @@ -127,7 +128,7 @@ pub struct TlsConfig { /// The certificate must be in DER, PEM (X.509), or PKCS#12 format. Additionally, the certificate can be provided as /// an inline string in PEM format. /// - /// If this is set, and is not a PKCS#12 archive, `key_file` must also be set. + /// If this is set _and_ is not a PKCS#12 archive, `key_file` must also be set. #[serde(alias = "crt_path")] #[configurable(metadata(docs::examples = "/path/to/host_certificate.crt"))] #[configurable(metadata(docs::human_name = "Certificate File Path"))] @@ -202,7 +203,9 @@ impl TlsSettings { ); } if options.verify_hostname == Some(false) { - warn!("The `verify_hostname` option is DISABLED, this may lead to security vulnerabilities."); + warn!( + "The `verify_hostname` option is DISABLED, this may lead to security vulnerabilities." + ); } } @@ -310,11 +313,21 @@ impl TlsSettings { if self.authorities.is_empty() { debug!("Fetching system root certs."); - #[cfg(windows)] - load_windows_certs(context).unwrap(); - - #[cfg(target_os = "macos")] - load_mac_certs(context).unwrap(); + cfg_if! { + if #[cfg(windows)] { + load_windows_certs(context).unwrap(); + } else if #[cfg(target_os = "macos")] { + cfg_if! { // Panic in release builds, warn in debug builds. + if #[cfg(debug_assertions)] { + if let Err(error) = load_mac_certs(context) { + warn!("Failed to load macOS certs: {error}"); + } + } else { + load_mac_certs(context).unwrap(); + } + } + } + } } else { let mut store = X509StoreBuilder::new().context(NewStoreBuilderSnafu)?; for authority in &self.authorities { @@ -330,8 +343,10 @@ impl TlsSettings { if let Some(alpn) = &self.alpn_protocols { if for_server { let server_proto = alpn.clone(); + // See https://github.com/sfackler/rust-openssl/pull/2360. + let server_proto_ref: &'static [u8] = Box::leak(server_proto.into_boxed_slice()); context.set_alpn_select_callback(move |_, client_proto| { - select_next_proto(server_proto.as_slice(), client_proto).ok_or(AlpnError::NOACK) + select_next_proto(server_proto_ref, client_proto).ok_or(AlpnError::NOACK) }); } else { context @@ -368,7 +383,7 @@ impl TlsConfig { |der| X509::from_der(&der).map(|x509| vec![x509]), |pem| { pem.match_indices(PEM_START_MARKER) - .map(|(start, _)| X509::from_pem(pem[start..].as_bytes())) + .map(|(start, _)| X509::from_pem(&pem.as_bytes()[start..])) .collect() }, ) @@ -639,10 +654,10 @@ fn der_or_pem(data: Vec, der_fn: impl Fn(Vec) -> T, pem_fn: impl Fn(S /// file "name" contains a PEM start marker, it is assumed to contain /// inline data and is used directly instead of opening a file. fn open_read(filename: &Path, note: &'static str) -> Result<(Vec, PathBuf)> { - if let Some(filename) = filename.to_str() { - if filename.contains(PEM_START_MARKER) { - return Ok((Vec::from(filename), "inline text".into())); - } + if let Some(filename) = filename.to_str() + && filename.contains(PEM_START_MARKER) + { + return Ok((Vec::from(filename), "inline text".into())); } let mut text = Vec::::new(); diff --git a/lib/vector-core/src/transform/mod.rs b/lib/vector-core/src/transform/mod.rs index 37d694e6c1..f8f31e7eb1 100644 --- a/lib/vector-core/src/transform/mod.rs +++ b/lib/vector-core/src/transform/mod.rs @@ -2,9 +2,9 @@ use std::{collections::HashMap, error, pin::Pin, sync::Arc, time::Instant}; use futures::{Stream, StreamExt}; use vector_common::internal_event::{ - self, register, CountByteSize, EventsSent, InternalEventHandle as _, Registered, DEFAULT_OUTPUT, + self, CountByteSize, DEFAULT_OUTPUT, EventsSent, InternalEventHandle as _, Registered, register, }; -use vector_common::{byte_size_of::ByteSizeOf, json_size::JsonSize, EventDataEq}; +use vector_common::{EventDataEq, byte_size_of::ByteSizeOf, json_size::JsonSize}; use crate::config::{ComponentKey, OutputId}; use crate::event::EventMutRef; @@ -12,7 +12,7 @@ use crate::schema::Definition; use crate::{ config, event::{ - into_event_stream, EstimatedJsonEncodedSizeOf, Event, EventArray, EventContainer, EventRef, + EstimatedJsonEncodedSizeOf, Event, EventArray, EventContainer, EventRef, into_event_stream, }, fanout::{self, Fanout}, schema, @@ -376,7 +376,6 @@ impl TransformOutputsBuf { /// # Panics /// /// Panics if there is no default output. - #[cfg(any(feature = "test", test))] pub fn drain(&mut self) -> impl Iterator + '_ { self.primary_buffer .as_mut() @@ -389,7 +388,6 @@ impl TransformOutputsBuf { /// # Panics /// /// Panics if there is no output with the given name. - #[cfg(any(feature = "test", test))] pub fn drain_named(&mut self, name: &str) -> impl Iterator + '_ { self.named_buffers .get_mut(name) @@ -402,12 +400,10 @@ impl TransformOutputsBuf { /// # Panics /// /// Panics if there is no default output. - #[cfg(any(feature = "test", test))] pub fn take_primary(&mut self) -> OutputBuffer { std::mem::take(self.primary_buffer.as_mut().expect("no default output")) } - #[cfg(any(feature = "test", test))] pub fn take_all_named(&mut self) -> HashMap { std::mem::take(&mut self.named_buffers) } @@ -474,7 +470,7 @@ impl OutputBuffer { self.0.capacity() } - pub fn first(&self) -> Option { + pub fn first(&self) -> Option> { self.0.first().and_then(|first| match first { EventArray::Logs(l) => l.first().map(Into::into), EventArray::Metrics(m) => m.first().map(Into::into), @@ -482,7 +478,6 @@ impl OutputBuffer { }) } - #[cfg(any(feature = "test", test))] pub fn drain(&mut self) -> impl Iterator + '_ { self.0.drain(..).flat_map(EventArray::into_events) } @@ -499,11 +494,11 @@ impl OutputBuffer { Ok(()) } - fn iter_events(&self) -> impl Iterator { + fn iter_events(&self) -> impl Iterator> { self.0.iter().flat_map(EventArray::iter_events) } - fn events_mut(&mut self) -> impl Iterator { + fn events_mut(&mut self) -> impl Iterator> { self.0.iter_mut().flat_map(EventArray::iter_events_mut) } diff --git a/lib/vector-core/src/transform/runtime_transform/mod.rs b/lib/vector-core/src/transform/runtime_transform/mod.rs index 750ea0d1d2..e29d0b5e45 100644 --- a/lib/vector-core/src/transform/runtime_transform/mod.rs +++ b/lib/vector-core/src/transform/runtime_transform/mod.rs @@ -3,8 +3,8 @@ mod vec_stream; use std::{future::ready, pin::Pin, time::Duration}; use futures::{ - stream::{self, BoxStream}, FutureExt, Stream, StreamExt, + stream::{self, BoxStream}, }; use tokio::time; use tokio_stream::wrappers::IntervalStream; @@ -82,9 +82,9 @@ where { let timers = self.timers(); let mut is_shutdown: bool = false; // TODO: consider using an enum describing the state instead of a - // a single boolean variable. - // It is used to prevent timers to emit messages after the source - // stream stopped. + // a single boolean variable. + // It is used to prevent timers to emit messages after the source + // stream stopped. Box::pin( input_rx diff --git a/lib/vector-core/src/transform/runtime_transform/vec_stream.rs b/lib/vector-core/src/transform/runtime_transform/vec_stream.rs index da07fcd1e8..b0237ce3f2 100644 --- a/lib/vector-core/src/transform/runtime_transform/vec_stream.rs +++ b/lib/vector-core/src/transform/runtime_transform/vec_stream.rs @@ -27,7 +27,7 @@ use std::{ task::{Context, Poll}, }; -use futures::{stream::Fuse, Stream, StreamExt}; +use futures::{Stream, StreamExt, stream::Fuse}; use pin_project::pin_project; impl VecStreamExt for T {} diff --git a/lib/vector-core/src/vrl.rs b/lib/vector-core/src/vrl.rs index 22f649929c..f22e34e605 100644 --- a/lib/vector-core/src/vrl.rs +++ b/lib/vector-core/src/vrl.rs @@ -1,5 +1,5 @@ -use lookup::{owned_value_path, OwnedTargetPath}; -use vrl::compiler::{compile_with_state, CompilationResult, CompileConfig, Function, TypeState}; +use lookup::{OwnedTargetPath, owned_value_path}; +use vrl::compiler::{CompilationResult, CompileConfig, Function, TypeState, compile_with_state}; use vrl::diagnostic::DiagnosticList; /// Compiles a VRL program diff --git a/lib/vector-lib/Cargo.toml b/lib/vector-lib/Cargo.toml index b951c8d169..89ae3bed75 100644 --- a/lib/vector-lib/Cargo.toml +++ b/lib/vector-lib/Cargo.toml @@ -2,13 +2,14 @@ name = "vector-lib" version = "0.1.0" authors = ["Vector Contributors "] -edition = "2021" +edition = "2024" publish = false [dependencies] codecs = { path = "../codecs", default-features = false } enrichment = { path = "../enrichment" } file-source = { path = "../file-source", optional = true } +file-source-common = { path = "../file-source-common", optional = true } opentelemetry-proto = { path = "../opentelemetry-proto", optional = true } prometheus-parser = { path = "../prometheus-parser", optional = true } vector-api-client = { path = "../vector-api-client", optional = true } @@ -25,7 +26,7 @@ vrl = { workspace = true, optional = true } api = ["vector-tap/api"] api-client = ["dep:vector-api-client"] lua = ["vector-core/lua"] -file-source = ["dep:file-source"] +file-source = ["dep:file-source", "dep:file-source-common"] opentelemetry = ["dep:opentelemetry-proto"] prometheus = ["dep:prometheus-parser"] proptest = ["vector-lookup/proptest", "vrl/proptest"] diff --git a/lib/vector-lib/src/lib.rs b/lib/vector-lib/src/lib.rs index 471c8737b1..125460cc3a 100644 --- a/lib/vector-lib/src/lib.rs +++ b/lib/vector-lib/src/lib.rs @@ -2,25 +2,27 @@ pub use codecs; pub use enrichment; #[cfg(feature = "file-source")] pub use file_source; +#[cfg(feature = "file-source")] +pub use file_source_common; #[cfg(feature = "api-client")] pub use vector_api_client as api_client; pub use vector_buffers as buffers; #[cfg(feature = "test")] pub use vector_common::event_test_util; pub use vector_common::{ - assert_event_data_eq, btreemap, byte_size_of, byte_size_of::ByteSizeOf, conversion, - encode_logfmt, finalization, finalizer, id, impl_event_data_eq, internal_event, json_size, - registered_event, request_metadata, sensitive_string, shutdown, trigger, Error, Result, - TimeZone, + Error, Result, TimeZone, assert_event_data_eq, btreemap, byte_size_of, + byte_size_of::ByteSizeOf, conversion, encode_logfmt, finalization, finalizer, id, + impl_event_data_eq, internal_event, json_size, registered_event, request_metadata, + sensitive_string, shutdown, trigger, }; pub use vector_config as configurable; pub use vector_config::impl_generate_config_from_default; #[cfg(feature = "vrl")] pub use vector_core::compile_vrl; pub use vector_core::{ - buckets, default_data_dir, emit, event, fanout, ipallowlist, metric_tags, metrics, partition, - quantiles, register, samples, schema, serde, sink, source, tcp, tls, transform, - EstimatedJsonEncodedSizeOf, + EstimatedJsonEncodedSizeOf, buckets, default_data_dir, emit, event, fanout, ipallowlist, + metric_tags, metrics, partition, quantiles, register, samples, schema, serde, sink, source, + tcp, tls, transform, }; pub use vector_lookup as lookup; pub use vector_stream as stream; @@ -31,16 +33,16 @@ pub use vrl; pub mod config { pub use vector_common::config::ComponentKey; pub use vector_core::config::{ - clone_input_definitions, init_log_schema, init_telemetry, log_schema, proxy, telemetry, AcknowledgementsConfig, DataType, GlobalOptions, Input, LegacyKey, LogNamespace, LogSchema, - OutputId, SourceAcknowledgementsConfig, SourceOutput, Tags, Telemetry, TransformOutput, - MEMORY_BUFFER_DEFAULT_MAX_EVENTS, + MEMORY_BUFFER_DEFAULT_MAX_EVENTS, OutputId, SourceAcknowledgementsConfig, SourceOutput, + Tags, Telemetry, TransformOutput, WildcardMatching, clone_input_definitions, + init_log_schema, init_telemetry, log_schema, proxy, telemetry, }; } #[cfg(feature = "opentelemetry")] pub mod opentelemetry { - pub use opentelemetry_proto::{convert, proto}; + pub use opentelemetry_proto::{common, logs, metrics, proto, spans}; } #[cfg(feature = "prometheus")] diff --git a/lib/vector-lookup/Cargo.toml b/lib/vector-lookup/Cargo.toml index ac2bf2ccd1..e588f6f108 100644 --- a/lib/vector-lookup/Cargo.toml +++ b/lib/vector-lookup/Cargo.toml @@ -2,7 +2,7 @@ name = "vector-lookup" version = "0.1.0" authors = ["Vector Contributors "] -edition = "2021" +edition = "2024" publish = false license = "MPL-2.0" diff --git a/lib/vector-lookup/src/lookup_v2/mod.rs b/lib/vector-lookup/src/lookup_v2/mod.rs index a6d5be15b3..00cbc0a1ea 100644 --- a/lib/vector-lookup/src/lookup_v2/mod.rs +++ b/lib/vector-lookup/src/lookup_v2/mod.rs @@ -5,8 +5,8 @@ use std::fmt; use vector_config_macros::configurable_component; pub use vrl::path::{ - parse_target_path, parse_value_path, BorrowedSegment, OwnedSegment, OwnedTargetPath, - OwnedValuePath, PathConcat, PathParseError, PathPrefix, TargetPath, ValuePath, + BorrowedSegment, OwnedSegment, OwnedTargetPath, OwnedValuePath, PathConcat, PathParseError, + PathPrefix, TargetPath, ValuePath, parse_target_path, parse_value_path, }; use vrl::value::KeyString; diff --git a/lib/vector-stream/Cargo.toml b/lib/vector-stream/Cargo.toml index 45b1e77aaa..587cfd3b37 100644 --- a/lib/vector-stream/Cargo.toml +++ b/lib/vector-stream/Cargo.toml @@ -2,23 +2,24 @@ name = "vector-stream" version = "0.1.0" authors = ["Vector Contributors "] -edition = "2021" +edition = "2024" publish = false [dependencies] -async-stream = { version = "0.3.6", default-features = false } +async-stream.workspace = true futures.workspace = true -futures-util = { version = "0.3.29", default-features = false, features = ["std"] } +futures-util.workspace = true pin-project.workspace = true -tokio = { version = "1.43.0", default-features = false, features = ["net"] } +tokio = { workspace = true, features = ["full"] } tokio-util = { version = "0.7.0", default-features = false, features = ["time"] } -tower = { version = "0.4", default-features = false, features = ["util"] } -tracing = { version = "0.1.34", default-features = false } -twox-hash = { version = "2.1.0", default-features = false, features = ["xxhash64"] } +tower = { version = "0.5.2", default-features = false, features = ["util"] } +tracing.workspace = true +twox-hash = { version = "2.1.2", default-features = false, features = ["xxhash64"] } vector-common = { path = "../vector-common" } vector-core = { path = "../vector-core" } [dev-dependencies] -proptest = "1.5" -rand = "0.8.5" -rand_distr = "0.4.3" +proptest = "1.7" +rand.workspace = true +rand_distr.workspace = true +tokio = { workspace = true, features = ["test-util"] } diff --git a/lib/vector-stream/src/batcher/mod.rs b/lib/vector-stream/src/batcher/mod.rs index cd2cb6aea1..58cade00e8 100644 --- a/lib/vector-stream/src/batcher/mod.rs +++ b/lib/vector-stream/src/batcher/mod.rs @@ -4,13 +4,13 @@ pub mod limiter; use std::{ pin::Pin, - task::{ready, Context, Poll}, + task::{Context, Poll, ready}, }; pub use config::BatchConfig; use futures::{ - stream::{Fuse, Stream}, Future, StreamExt, + stream::{Fuse, Stream}, }; use pin_project::pin_project; use tokio::time::Sleep; @@ -66,7 +66,7 @@ where } else { Poll::Ready(Some(this.state.take_batch())) } - } + }; } Poll::Ready(Some(item)) => { let (item_fits, item_metadata) = this.state.item_fits_in_batch(&item); @@ -100,7 +100,7 @@ where } else { Poll::Pending } - } + }; } } } diff --git a/lib/vector-stream/src/concurrent_map.rs b/lib/vector-stream/src/concurrent_map.rs index 57ce9b8bc4..96af68f3bf 100644 --- a/lib/vector-stream/src/concurrent_map.rs +++ b/lib/vector-stream/src/concurrent_map.rs @@ -3,12 +3,12 @@ use std::{ num::NonZeroUsize, panic, pin::Pin, - task::{ready, Context, Poll}, + task::{Context, Poll, ready}, }; use futures_util::{ - stream::{Fuse, FuturesOrdered}, Stream, StreamExt, + stream::{Fuse, FuturesOrdered}, }; use pin_project::pin_project; use tokio::task::JoinHandle; @@ -83,7 +83,11 @@ where } match ready!(this.in_flight.poll_next_unpin(cx)) { - // Either nothing is in-flight, or nothing is ready. + // If the stream is done and there is no futures managed by FuturesOrdered, + // we must end the stream by returning Poll::Ready(None). + None if this.stream.is_done() => Poll::Ready(None), + // If there are no in-flight futures managed by FuturesOrdered but the underlying + // stream is not done, then we must keep polling that stream. None => Poll::Pending, Some(result) => match result { Ok(item) => Poll::Ready(Some(item)), @@ -101,3 +105,21 @@ where } } } + +#[cfg(test)] +mod tests { + use super::*; + use futures_util::stream::StreamExt; + + #[tokio::test] + async fn test_concurrent_map_on_empty_stream() { + let stream = futures_util::stream::empty::<()>(); + let limit = Some(NonZeroUsize::new(2).unwrap()); + // The `as _` is required to construct a `dyn Future` + let f = |()| Box::pin(async move {}) as _; + let mut concurrent_map = ConcurrentMap::new(stream, limit, f); + + // Assert that the stream does not hang + assert_eq!(concurrent_map.next().await, None); + } +} diff --git a/lib/vector-stream/src/driver.rs b/lib/vector-stream/src/driver.rs index d8e60ffdca..5b190d62a3 100644 --- a/lib/vector-stream/src/driver.rs +++ b/lib/vector-stream/src/driver.rs @@ -1,13 +1,13 @@ use std::{collections::VecDeque, fmt, future::poll_fn, task::Poll}; -use futures::{poll, FutureExt, Stream, StreamExt, TryFutureExt}; +use futures::{FutureExt, Stream, StreamExt, TryFutureExt, poll}; use tokio::{pin, select}; use tower::Service; use tracing::Instrument; use vector_common::internal_event::emit; use vector_common::internal_event::{ - register, ByteSize, BytesSent, CallError, InternalEventHandle as _, PollReadyError, Registered, - RegisteredEventCache, SharedString, TaggedEventsSent, + ByteSize, BytesSent, CallError, InternalEventHandle as _, PollReadyError, Registered, + RegisteredEventCache, SharedString, TaggedEventsSent, register, }; use vector_common::request_metadata::{GroupedCountByteSize, MetaDescriptive}; use vector_core::event::{EventFinalizers, EventStatus, Finalizable}; @@ -212,10 +212,10 @@ where trace!(message = "Service call succeeded.", request_id); finalizers.update_status(response.event_status()); if response.event_status() == EventStatus::Delivered { - if let Some(bytes_sent) = bytes_sent { - if let Some(byte_size) = response.bytes_sent() { - bytes_sent.emit(ByteSize(byte_size)); - } + if let Some(bytes_sent) = bytes_sent + && let Some(byte_size) = response.bytes_sent() + { + bytes_sent.emit(ByteSize(byte_size)); } response.events_sent().emit_event(events_sent); @@ -226,7 +226,7 @@ where finalizers.update_status(EventStatus::Rejected); } } - }; + } drop(finalizers); // suppress "argument not consumed" warning } @@ -246,13 +246,13 @@ mod tests { use std::{ future::Future, pin::Pin, - sync::{atomic::AtomicUsize, atomic::Ordering, Arc}, - task::{ready, Context, Poll}, + sync::{Arc, atomic::AtomicUsize, atomic::Ordering}, + task::{Context, Poll, ready}, time::Duration, }; use futures_util::stream; - use rand::{prelude::StdRng, SeedableRng}; + use rand::{SeedableRng, prelude::StdRng}; use rand_distr::{Distribution, Pareto}; use tokio::{ sync::{OwnedSemaphorePermit, Semaphore}, @@ -360,9 +360,8 @@ mod tests { pub(crate) fn get_sleep_dur(&mut self) -> Duration { let lower = self.lower_bound_us; let upper = self.upper_bound_us; - - // Generate a value between 10ms and 500ms, with a long tail shape to the distribution. - #[allow(clippy::cast_sign_loss)] // Value will be positive anyways + // Generate a value between `lower` and `upper`, with a long tail shape to the distribution. + #[allow(clippy::cast_sign_loss)] // Value will be positive anyway self.jitter .sample_iter(&mut self.jitter_gen) .map(|n| n * lower as f64) diff --git a/lib/vector-stream/src/futures_unordered_count.rs b/lib/vector-stream/src/futures_unordered_count.rs index 95c2c3ff2b..2d78457ecf 100644 --- a/lib/vector-stream/src/futures_unordered_count.rs +++ b/lib/vector-stream/src/futures_unordered_count.rs @@ -5,7 +5,7 @@ use std::{ task::{Context, Poll}, }; -use futures_util::{stream::FuturesUnordered, Stream}; +use futures_util::{Stream, stream::FuturesUnordered}; use pin_project::pin_project; /// A set of futures which may complete in any order, and results are returned as a count of ready @@ -76,7 +76,7 @@ impl Stream for FuturesUnorderedCount { Poll::Pending } else { Poll::Ready(Some(mem::take(this.items))) - } + }; } // We got a future result, so bump the counter. diff --git a/lib/vector-stream/src/partitioned_batcher.rs b/lib/vector-stream/src/partitioned_batcher.rs index b0ba86b85b..42541b381c 100644 --- a/lib/vector-stream/src/partitioned_batcher.rs +++ b/lib/vector-stream/src/partitioned_batcher.rs @@ -3,22 +3,22 @@ use std::{ hash::{BuildHasherDefault, Hash}, num::NonZeroUsize, pin::Pin, - task::{ready, Context, Poll}, + task::{Context, Poll, ready}, time::Duration, }; use futures::stream::{Fuse, Stream, StreamExt}; use pin_project::pin_project; -use tokio_util::time::{delay_queue::Key, DelayQueue}; +use tokio_util::time::{DelayQueue, delay_queue::Key}; use twox_hash::XxHash64; use vector_common::byte_size_of::ByteSizeOf; use vector_core::{partition::Partitioner, time::KeyedTimer}; use crate::batcher::{ + BatchConfig, config::BatchConfigParts, data::BatchData, limiter::{ByteSizeOfItemSize, ItemBatchSize, SizeLimit}, - BatchConfig, }; /// A `KeyedTimer` based on `DelayQueue`. @@ -75,10 +75,11 @@ where // This is a yet-unseen item key, so create a new expiration // entry. let expiration_key = self.expirations.insert(item_key.clone(), self.timeout); - assert!(self - .expiration_map - .insert(item_key, expiration_key) - .is_none()); + assert!( + self.expiration_map + .insert(item_key, expiration_key) + .is_none() + ); } } @@ -355,15 +356,15 @@ mod test { time::Duration, }; - use futures::{stream, Stream}; + use futures::{Stream, stream}; use pin_project::pin_project; use proptest::prelude::*; use tokio::{pin, time::advance}; use vector_core::{partition::Partitioner, time::KeyedTimer}; use crate::{ - partitioned_batcher::{ExpirationQueue, PartitionedBatcher}, BatcherSettings, + partitioned_batcher::{ExpirationQueue, PartitionedBatcher}, }; #[derive(Debug)] @@ -645,7 +646,7 @@ mod test { // Asserts that ExpirationQueue properly implements KeyedTimer. We are // primarily concerned with whether expiration is properly observed. let timeout = Duration::from_millis(100); // 1/10 of a second, an - // eternity + // eternity let mut expiration_queue: ExpirationQueue = ExpirationQueue::new(timeout); diff --git a/lib/vector-tap/Cargo.toml b/lib/vector-tap/Cargo.toml index 04e0daa0a2..325520eac8 100644 --- a/lib/vector-tap/Cargo.toml +++ b/lib/vector-tap/Cargo.toml @@ -2,7 +2,7 @@ name = "vector-tap" version = "0.1.0" authors = ["Vector Contributors "] -edition = "2021" +edition = "2024" publish = false license = "MPL-2.0" @@ -10,25 +10,21 @@ license = "MPL-2.0" api = ["dep:async-graphql"] [dependencies] -async-graphql = { version = "7.0.7", default-features = false, features = ["playground"], optional = true} -colored = { version = "3.0.0", default-features = false } +async-graphql = { version = "7.0.17", default-features = false, features = ["playground"], optional = true } +colored.workspace = true futures.workspace = true glob.workspace = true -serde_yaml = { version = "0.9.34", default-features = false } -tokio = { version = "1.43.0", default-features = false, features = ["time"] } +serde_yaml.workspace = true +tokio = { workspace = true, features = ["time"] } tokio-stream = { version = "0.1.17", default-features = false, features = ["sync"] } tokio-tungstenite = { version = "0.20.1", default-features = false } -tracing = { version = "0.1.34", default-features = false } +tracing.workspace = true url = { version = "2.5.4", default-features = false } uuid.workspace = true vector-api-client = { path = "../vector-api-client" } vector-common = { path = "../vector-common" } vector-core = { path = "../vector-core" } vector-buffers = { path = "../vector-buffers" } -futures-util = "0.3.30" [dev-dependencies] -chrono = { workspace = true } -portpicker = { path = "../portpicker" } -serde_json = { workspace = true } -tokio = { version = "1.43.0", default-features = false, features = ["test-util"] } +tokio = { workspace = true, features = ["test-util"] } diff --git a/lib/vector-tap/src/controller.rs b/lib/vector-tap/src/controller.rs index 656f77c108..92036b6d5e 100644 --- a/lib/vector-tap/src/controller.rs +++ b/lib/vector-tap/src/controller.rs @@ -3,7 +3,7 @@ use std::{ num::NonZeroUsize, }; -use futures::{future::try_join_all, FutureExt}; +use futures::{FutureExt, future::try_join_all}; use tokio::sync::{ mpsc as tokio_mpsc, mpsc::error::{SendError, TrySendError}, @@ -11,7 +11,7 @@ use tokio::sync::{ }; use tracing::{Instrument, Span}; use uuid::Uuid; -use vector_buffers::{topology::builder::TopologyBuilder, WhenFull}; +use vector_buffers::{WhenFull, topology::builder::TopologyBuilder}; use vector_common::config::ComponentKey; use vector_core::event::{EventArray, LogArray, MetricArray, TraceArray}; use vector_core::fanout; @@ -26,7 +26,7 @@ type TapSender = tokio_mpsc::Sender; type ShutdownTx = oneshot::Sender<()>; type ShutdownRx = oneshot::Receiver<()>; -const TAP_BUFFER_SIZE: NonZeroUsize = unsafe { NonZeroUsize::new_unchecked(100) }; +const TAP_BUFFER_SIZE: NonZeroUsize = NonZeroUsize::new(100).unwrap(); /// Clients can supply glob patterns to find matched topology components. trait GlobMatcher { @@ -123,7 +123,9 @@ impl TapPayload { invalid_matches: Vec, ) -> Self { let pattern = pattern.into(); - let message = format!("[tap] Warning: source inputs cannot be tapped. Input pattern '{}' matches sources {:?}", pattern, invalid_matches); + let message = format!( + "[tap] Warning: source inputs cannot be tapped. Input pattern '{pattern}' matches sources {invalid_matches:?}" + ); Self::Notification(Notification::InvalidMatch(InvalidMatch::new( message, pattern, @@ -138,8 +140,7 @@ impl TapPayload { ) -> Self { let pattern = pattern.into(); let message = format!( - "[tap] Warning: sink outputs cannot be tapped. Output pattern '{}' matches sinks {:?}", - pattern, invalid_matches + "[tap] Warning: sink outputs cannot be tapped. Output pattern '{pattern}' matches sinks {invalid_matches:?}" ); Self::Notification(Notification::InvalidMatch(InvalidMatch::new( message, diff --git a/lib/vector-tap/src/lib.rs b/lib/vector-tap/src/lib.rs index fa52e8a7bb..721d15ac61 100644 --- a/lib/vector-tap/src/lib.rs +++ b/lib/vector-tap/src/lib.rs @@ -19,8 +19,8 @@ use url::Url; use vector_api_client::{ connect_subscription_client, gql::{ - output_events_by_component_id_patterns_subscription::OutputEventsByComponentIdPatternsSubscriptionOutputEventsByComponentIdPatterns as GraphQLTapOutputEvent, TapEncodingFormat, TapSubscriptionExt, + output_events_by_component_id_patterns_subscription::OutputEventsByComponentIdPatternsSubscriptionOutputEventsByComponentIdPatterns as GraphQLTapOutputEvent, }, }; @@ -203,7 +203,7 @@ impl<'a> TapRunner<'a> { // If the stream times out, that indicates the duration specified by the user // has elapsed. We should exit gracefully. { - return Ok(()) + return Ok(()); } Ok(_) => return Err(TapExecutorError::GraphQLError), } @@ -259,104 +259,3 @@ impl<'a> TapRunner<'a> { } } } - -#[cfg(test)] -mod tests { - use std::net::{IpAddr, Ipv4Addr}; - - use chrono::Utc; - use futures_util::sink::SinkExt; - use futures_util::stream::StreamExt; - use serde_json::Value; - use tokio::net::TcpListener; - use tokio::time::sleep; - use tokio_tungstenite::accept_async; - use tokio_tungstenite::tungstenite::Message; - - use portpicker::pick_unused_port; - - use super::*; - - #[tokio::test(start_paused = true)] - async fn test_async_output_channel() { - let component_id = "test-component-id"; - let component_type = "test-component-type"; - let message = "test-message"; - let timestamp = Utc::now(); - let string_encoding = "test-str"; - - // Start a local WebSocket server to mimic Vector GraphQL API - let ip_addr = IpAddr::V4(Ipv4Addr::LOCALHOST); - let port = pick_unused_port(ip_addr); - let addr = format!("{ip_addr}:{port}"); - - let listener = TcpListener::bind(&addr).await.unwrap(); - let server = tokio::spawn(async move { - let (stream, _) = listener.accept().await.unwrap(); - let mut ws_stream = accept_async(stream).await.unwrap(); - if let Some(Ok(Message::Text(msg))) = ws_stream.next().await { - let client_init_msg: Value = - serde_json::from_str(&msg).expect("Init message should be in JSON format"); - let subscription_id = &client_init_msg["id"]; - - let message_to_send = format!( - "{{\ - \"type\":\"data\",\ - \"id\":{subscription_id},\ - \"payload\":{{\ - \"data\":{{\ - \"outputEventsByComponentIdPatterns\":[{{\ - \"__typename\":\"Log\",\ - \"componentId\":\"{component_id}\",\ - \"componentType\":\"{component_type}\",\ - \"componentKind\":\"source\",\ - \"message\":\"{message}\",\ - \"timestamp\":\"{timestamp}\",\ - \"string\":\"{string_encoding}\"\ - }}]\ - }}\ - }}\ - }}", - ); - - // Send 2 messages to client, mimicking 3 second interval - loop { - ws_stream - .send(Message::Text(message_to_send.clone())) - .await - .unwrap(); - sleep(Duration::from_secs(3)).await; - } - } - }); - - let (output_tx, mut output_rx) = tokio_mpsc::channel(10); - let url = Url::parse(&format!("ws://{addr}")).unwrap(); - let output_channel = OutputChannel::AsyncChannel(output_tx); - - let tap_runner = TapRunner::new( - &url, - vec![], - vec![], - &output_channel, - TapEncodingFormat::Json, - ); - assert!(tap_runner.run_tap(0, 0, Some(5000), false).await.is_ok()); - - let mut num_recv = 0; - while let Ok(events) = output_rx.try_recv() { - assert_eq!(events.len(), 1); - if let GraphQLTapOutputEvent::Log(ev) = &events[0] { - num_recv += 1; - assert_eq!(ev.component_id, component_id); - assert_eq!(ev.component_type, component_type); - assert_eq!(ev.message, Some(message.to_string())); - assert_eq!(ev.timestamp, Some(timestamp)); - assert_eq!(ev.string, string_encoding); - } - } - assert_eq!(num_recv, 2); - - server.abort(); - } -} diff --git a/lib/vector-tap/src/notification.rs b/lib/vector-tap/src/notification.rs index 58fd81766a..04ba715718 100644 --- a/lib/vector-tap/src/notification.rs +++ b/lib/vector-tap/src/notification.rs @@ -14,7 +14,7 @@ pub struct Matched { impl Matched { pub fn new(pattern: String) -> Self { Self { - message: format!("[tap] Pattern '{}' successfully matched.", pattern), + message: format!("[tap] Pattern '{pattern}' successfully matched."), pattern, } } @@ -34,8 +34,7 @@ impl NotMatched { pub fn new(pattern: String) -> Self { Self { message: format!( - "[tap] Pattern '{}' failed to match: will retry on configuration reload.", - pattern + "[tap] Pattern '{pattern}' failed to match: will retry on configuration reload." ), pattern, } diff --git a/lib/vector-vrl/cli/Cargo.toml b/lib/vector-vrl/cli/Cargo.toml index aec059418b..d97acae8c3 100644 --- a/lib/vector-vrl/cli/Cargo.toml +++ b/lib/vector-vrl/cli/Cargo.toml @@ -2,7 +2,7 @@ name = "vector-vrl-cli" version = "0.1.0" authors = ["Vector Contributors "] -edition = "2021" +edition = "2024" publish = false license = "MPL-2.0" diff --git a/lib/vector-vrl/cli/src/main.rs b/lib/vector-vrl/cli/src/main.rs index 5f08c6f09b..9aa4b09fdb 100644 --- a/lib/vector-vrl/cli/src/main.rs +++ b/lib/vector-vrl/cli/src/main.rs @@ -1,5 +1,5 @@ use clap::Parser; -use vrl::cli::{cmd::cmd, Opts}; +use vrl::cli::{Opts, cmd::cmd}; fn main() { let mut functions = vrl::stdlib::all(); diff --git a/lib/vector-vrl/functions/Cargo.toml b/lib/vector-vrl/functions/Cargo.toml index 432cac075c..d6319b079c 100644 --- a/lib/vector-vrl/functions/Cargo.toml +++ b/lib/vector-vrl/functions/Cargo.toml @@ -2,7 +2,7 @@ name = "vector-vrl-functions" version = "0.1.0" authors = ["Vector Contributors "] -edition = "2021" +edition = "2024" publish = false license = "MPL-2.0" diff --git a/lib/vector-vrl/functions/src/get_secret.rs b/lib/vector-vrl/functions/src/get_secret.rs index f559925daf..28381e64ce 100644 --- a/lib/vector-vrl/functions/src/get_secret.rs +++ b/lib/vector-vrl/functions/src/get_secret.rs @@ -1,8 +1,7 @@ use vrl::prelude::*; fn get_secret(ctx: &mut Context, key: Value) -> std::result::Result { - let key_bytes = key.as_bytes().expect("argument must be a string"); - let key_str = String::from_utf8_lossy(key_bytes); + let key_str = key.as_str().expect("argument must be a string"); let value = match ctx.target().get_secret(key_str.as_ref()) { Some(secret) => secret.into(), None => Value::Null, diff --git a/lib/vector-vrl/functions/src/remove_secret.rs b/lib/vector-vrl/functions/src/remove_secret.rs index fec124cf5e..bd3a9319d4 100644 --- a/lib/vector-vrl/functions/src/remove_secret.rs +++ b/lib/vector-vrl/functions/src/remove_secret.rs @@ -1,8 +1,7 @@ use vrl::prelude::*; fn remove_secret(ctx: &mut Context, key: Value) -> std::result::Result { - let key_bytes = key.as_bytes().expect("argument must be a string"); - let key_str = String::from_utf8_lossy(key_bytes); + let key_str = key.as_str().expect("argument must be a string"); ctx.target_mut().remove_secret(key_str.as_ref()); Ok(Value::Null) } diff --git a/lib/vector-vrl/functions/src/set_secret.rs b/lib/vector-vrl/functions/src/set_secret.rs index 09ed6cd065..5b0c56f705 100644 --- a/lib/vector-vrl/functions/src/set_secret.rs +++ b/lib/vector-vrl/functions/src/set_secret.rs @@ -5,8 +5,8 @@ fn set_secret( key: Value, secret: Value, ) -> std::result::Result { - let key_str = String::from_utf8_lossy(key.as_bytes().expect("key must be a string")); - let secret_str = String::from_utf8_lossy(secret.as_bytes().expect("secret must be a string")); + let key_str = key.as_str().expect("key must be a string"); + let secret_str = secret.as_str().expect("secret must be a string"); ctx.target_mut() .insert_secret(key_str.as_ref(), secret_str.as_ref()); diff --git a/lib/vector-vrl/tests/Cargo.toml b/lib/vector-vrl/tests/Cargo.toml index 37b9010d5b..ddcb78084a 100644 --- a/lib/vector-vrl/tests/Cargo.toml +++ b/lib/vector-vrl/tests/Cargo.toml @@ -2,7 +2,7 @@ name = "vector-vrl-tests" version = "0.1.0" authors = ["Vector Contributors "] -edition = "2021" +edition = "2024" publish = false [dependencies] @@ -16,7 +16,7 @@ clap.workspace = true glob.workspace = true serde.workspace = true serde_json.workspace = true -tracing-subscriber = { version = "0.3.19", default-features = false, features = ["fmt"] } +tracing-subscriber = { workspace = true, features = ["fmt"] } [target.'cfg(not(target_env = "msvc"))'.dependencies] tikv-jemallocator = { version = "0.6.0" } diff --git a/lib/vector-vrl/tests/resources/json-schema_definition.json b/lib/vector-vrl/tests/resources/json-schema_definition.json new file mode 100644 index 0000000000..9c4c8a00d0 --- /dev/null +++ b/lib/vector-vrl/tests/resources/json-schema_definition.json @@ -0,0 +1 @@ +{"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"https://example.com/product.schema.json","title":"Product","description":"A product from Acme's catalog","type":"object","properties":{"productUser":{"description":"The unique identifier for a product user","type":"string","format":"email"}}} diff --git a/lib/vector-vrl/tests/src/docs.rs b/lib/vector-vrl/tests/src/docs.rs index ec694ce316..0dacafb10e 100644 --- a/lib/vector-vrl/tests/src/docs.rs +++ b/lib/vector-vrl/tests/src/docs.rs @@ -183,7 +183,7 @@ fn test_from_cue_example(category: &'static str, name: String, example: Example) Test { name: title, - category: format!("docs/{}/{}", category, name), + category: format!("docs/{category}/{name}"), error: None, source, object, diff --git a/lib/vector-vrl/tests/src/main.rs b/lib/vector-vrl/tests/src/main.rs index 4fd7517969..451f03502c 100644 --- a/lib/vector-vrl/tests/src/main.rs +++ b/lib/vector-vrl/tests/src/main.rs @@ -6,7 +6,7 @@ mod test_enrichment; use std::env; use std::path::PathBuf; -use vrl::test::{get_tests_from_functions, run_tests, Test, TestConfig}; +use vrl::test::{Test, TestConfig, get_tests_from_functions, run_tests}; use chrono_tz::Tz; use clap::Parser; @@ -56,7 +56,7 @@ pub struct Cmd { impl Cmd { fn timezone(&self) -> TimeZone { if let Some(ref tz) = self.timezone { - TimeZone::parse(tz).unwrap_or_else(|| panic!("couldn't parse timezone: {}", tz)) + TimeZone::parse(tz).unwrap_or_else(|| panic!("couldn't parse timezone: {tz}")) } else { TimeZone::Named(Tz::UTC) } @@ -68,10 +68,10 @@ fn should_run(name: &str, pat: &Option, _runtime: VrlRuntime) -> bool { return false; } - if let Some(pat) = pat { - if !name.contains(pat) { - return false; - } + if let Some(pat) = pat + && !name.contains(pat) + { + return false; } true diff --git a/lib/vector-vrl/tests/src/test_enrichment.rs b/lib/vector-vrl/tests/src/test_enrichment.rs index 725c4cd4a2..ff803ea090 100644 --- a/lib/vector-vrl/tests/src/test_enrichment.rs +++ b/lib/vector-vrl/tests/src/test_enrichment.rs @@ -10,6 +10,7 @@ impl enrichment::Table for TestEnrichmentTable { _case: enrichment::Case, _condition: &'a [enrichment::Condition<'a>], _select: Option<&[String]>, + _wildcard: Option<&Value>, _index: Option, ) -> Result { let mut result = ObjectMap::new(); @@ -25,6 +26,7 @@ impl enrichment::Table for TestEnrichmentTable { _case: enrichment::Case, _condition: &'a [enrichment::Condition<'a>], _select: Option<&[String]>, + _wildcard: Option<&Value>, _index: Option, ) -> Result, String> { let mut result1 = ObjectMap::new(); diff --git a/lib/vector-vrl/web-playground/Cargo.toml b/lib/vector-vrl/web-playground/Cargo.toml index 6d5cf3a9dd..3aaa7fda74 100644 --- a/lib/vector-vrl/web-playground/Cargo.toml +++ b/lib/vector-vrl/web-playground/Cargo.toml @@ -1,7 +1,10 @@ [package] name = "vector-vrl-web-playground" version = "0.1.0" -edition = "2021" +edition = "2024" +repository = "https://github.com/vectordotdev/vector" +description = "A playground for experimenting with VRL" +license = "MPL-2.0" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -12,10 +15,12 @@ crate-type = ["cdylib"] wasm-bindgen = "0.2" vrl.workspace = true serde.workspace = true -serde-wasm-bindgen = "0.6" +web-sys = { version = "0.3", features = ["Window", "Performance"] } gloo-utils = { version = "0.2", features = ["serde"] } vector-vrl-functions = { path = "../functions" } enrichment = { path = "../../enrichment" } +# Required per https://docs.rs/getrandom/latest/getrandom/#webassembly-support +getrandom = { version = "0.2.15", features = ["js"] } [build-dependencies] -cargo-lock = "10.0.1" +cargo-lock = "10.1.0" diff --git a/lib/vector-vrl/web-playground/README.md b/lib/vector-vrl/web-playground/README.md index 9246101e4d..0105aa2231 100644 --- a/lib/vector-vrl/web-playground/README.md +++ b/lib/vector-vrl/web-playground/README.md @@ -181,5 +181,5 @@ init().then(() => { [vrl-playground]: https://github.com/vectordotdev/vector/issues/14653 [mozilla-wasm-rust-docs]: https://developer.mozilla.org/en-US/docs/WebAssembly/Rust_to_wasm [rust-book-wasm]: https://rustwasm.github.io/docs/book/ -[vrl-repl]: https://github.com/vectordotdev/vector/tree/master/lib/vrl/cli +[vrl-repl]: https://github.com/vectordotdev/vector/tree/master/lib/vector-vrl/cli [vrl-wasm-unsupported-filter]: https://github.com/vectordotdev/vector/issues?q=is%3Aopen+is%3Aissue+label%3A%22vrl%3A+playground%22+wasm+compatible diff --git a/lib/vector-vrl/web-playground/build.rs b/lib/vector-vrl/web-playground/build.rs index b6bc46a33c..f9b6c61e05 100644 --- a/lib/vector-vrl/web-playground/build.rs +++ b/lib/vector-vrl/web-playground/build.rs @@ -4,7 +4,7 @@ use std::path::{Path, PathBuf}; use std::process::Command; use std::{env, fs, io}; -use cargo_lock::{package::SourceKind, Lockfile}; +use cargo_lock::{Lockfile, package::SourceKind}; fn get_vector_lock_path() -> PathBuf { let path = fs::canonicalize(env::var("CARGO_MANIFEST_DIR").unwrap()).unwrap(); @@ -55,40 +55,39 @@ fn write_vrl_constants(lockfile: &Lockfile, output_file: &mut File) { .find(|&package| package.name.as_str() == "vrl") .expect("missing VRL dependency"); - let vrl_source = vrl_dep.source.clone().expect("missing VRL source id"); - - let (version, link) = match vrl_source.kind() { - SourceKind::Git(_) => { - let precise = vrl_source - .precise() - .expect("git reference should have precise") - .to_string(); - ( - precise.clone(), - format!("{}/tree/{precise}", vrl_source.url()), - ) - } - SourceKind::Registry if vrl_source.is_default_registry() => { - let version = vrl_dep.version.to_string(); - ( - version.to_string(), - format!("https://crates.io/crates/vrl/{version}"), - ) + let (version, link) = if let Some(vrl_source) = vrl_dep.source.clone() { + match vrl_source.kind() { + SourceKind::Git(_) => { + let precise = vrl_source + .precise() + .expect("git reference should have precise") + .to_string(); + ( + precise.clone(), + Some(format!("{}/tree/{precise}", vrl_source.url())), + ) + } + SourceKind::Registry if vrl_source.is_default_registry() => { + let version = vrl_dep.version.to_string(); + ( + version.clone(), + Some(format!("https://crates.io/crates/vrl/{version}")), + ) + } + SourceKind::Path => (vrl_dep.version.to_string(), None), + kind => unimplemented!("unhandled source kind: {:?}", kind), } - SourceKind::Path - | SourceKind::Registry - | SourceKind::SparseRegistry - | SourceKind::LocalRegistry - | SourceKind::Directory => unimplemented!("unhandled source kind: {:?}", vrl_source.kind()), - _ => unimplemented!("unknown source kind: {:?}", vrl_source.kind()), + } else { + (vrl_dep.version.to_string(), None) }; output_file .write_all(create_const_statement("VRL_VERSION", &version).as_bytes()) .expect("Failed to write VRL version constant"); + let link_str = link.as_deref().unwrap_or(""); output_file - .write_all(create_const_statement("VRL_LINK", &link).as_bytes()) + .write_all(create_const_statement("VRL_LINK", link_str).as_bytes()) .expect("Failed to write VRL_LINK constant"); } diff --git a/lib/vector-vrl/web-playground/public/index.css b/lib/vector-vrl/web-playground/public/index.css index caaacc2cd0..364ac714fe 100644 --- a/lib/vector-vrl/web-playground/public/index.css +++ b/lib/vector-vrl/web-playground/public/index.css @@ -176,7 +176,7 @@ div#output-section { border-bottom: 1px solid var(--datadog-gray-light); } -/* BUTTONS */ +/* BUTTONS */ .btn { display: inline-block; outline: 0; @@ -373,3 +373,50 @@ div#output-section { font-size: 11px; } } + +#toolbar-section { + display: flex; + align-items: center; + gap: 8px; +} + +.timezone-container { + margin-left: auto; + display: flex; + align-items: center; +} + +#timezone-label { + margin-right: 8px; + font-weight: bold; + font-family: "Open Sans", -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Oxygen, Ubuntu, + Cantarell, "Helvetica Neue", sans-serif; + font-size: 14px; +} + +#timezone-input { + padding: 0px 10px; + border: 1px solid #ccc; + border-radius: 4px; + background-color: #f8f9fa; + border-width: 1px; + border-style: solid; + font-size: 14px; + font-family: "Open Sans", -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Oxygen, Ubuntu, + Cantarell, "Helvetica Neue", sans-serif; + line-height: 1.5; + color: #212529; + height: 36px; + box-sizing: border-box; +} + +#output-cell-title .cell-title { + display: flex; + justify-content: space-between; + align-items: center; +} + +#elapsed-time { + font-weight: normal; + font-size: 12px; +} diff --git a/lib/vector-vrl/web-playground/public/index.html b/lib/vector-vrl/web-playground/public/index.html index 9799dcd4a4..7064dc6f4c 100644 --- a/lib/vector-vrl/web-playground/public/index.html +++ b/lib/vector-vrl/web-playground/public/index.html @@ -2,7 +2,6 @@ - VRL playground @@ -53,8 +52,30 @@

VRL Playground

-
- +
+ + + + + + + + + + + + + + + +
+
@@ -74,7 +95,10 @@

VRL Playground

-

Output

+

+ Output + +

diff --git a/lib/vector-vrl/web-playground/public/index.js b/lib/vector-vrl/web-playground/public/index.js index aaf0c87007..1f08115e8b 100644 --- a/lib/vector-vrl/web-playground/public/index.js +++ b/lib/vector-vrl/web-playground/public/index.js @@ -39,7 +39,6 @@ export class VrlWebPlayground { constructor() { let temp = init().then(() => { this.run_vrl = run_vrl; - this.vector_version = vector_version(); this.vector_link = vector_link(); @@ -210,10 +209,18 @@ export class VrlWebPlayground { return input; } - let res = this.run_vrl(input); + const tz_input = document.getElementById('timezone-input'); + // set default tz if nothing is set. this is going to use default timezone + let timezone = tz_input.value ? tz_input.value : (tz_input.value = "Default"); + let res = this.run_vrl(input, timezone); console.log("[DEBUG::handleRunCode()] Printing out res: ", res); if (res.result) { this.outputEditor.setValue(JSON.stringify(res.result, null, "\t")); + if (res.elapsed_time !== null) { + document.getElementById("elapsed-time").textContent = `Duration: ${res.elapsed_time} milliseconds`; + }else { + document.getElementById("elapsed-time").textContent = ""; + } } else if (res.msg) { // disable json linting for error msgs // since error msgs are not valid json diff --git a/lib/vector-vrl/web-playground/src/lib.rs b/lib/vector-vrl/web-playground/src/lib.rs index a434050aef..93487a0236 100644 --- a/lib/vector-vrl/web-playground/src/lib.rs +++ b/lib/vector-vrl/web-playground/src/lib.rs @@ -1,9 +1,9 @@ use gloo_utils::format::JsValueSerdeExt; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; -use vrl::compiler::runtime::{Runtime, Terminate}; use vrl::compiler::TimeZone; -use vrl::compiler::{compile_with_state, CompileConfig, TargetValue, TypeState}; +use vrl::compiler::runtime::{Runtime, Terminate}; +use vrl::compiler::{CompileConfig, TargetValue, TypeState, compile_with_state}; use vrl::diagnostic::DiagnosticList; use vrl::diagnostic::Formatter; use vrl::value::Secrets; @@ -29,17 +29,22 @@ impl Input { } } -// The module returns the result of the last expression and the event that results from the -// applied program +// The module returns the result of the last expression, the resulting event, +// and the execution time. #[derive(Deserialize, Serialize)] pub struct VrlCompileResult { pub output: Value, pub result: Value, + pub elapsed_time: Option, } impl VrlCompileResult { - fn new(output: Value, result: Value) -> Self { - Self { output, result } + fn new(output: Value, result: Value, elapsed_time: Option) -> Self { + Self { + output, + result, + elapsed_time, + } } } @@ -76,7 +81,10 @@ impl VrlDiagnosticResult { } } -fn compile(mut input: Input) -> Result { +fn compile( + mut input: Input, + tz_str: Option, +) -> Result { let mut functions = vrl::stdlib::all(); functions.extend(vector_vrl_functions::all()); functions.extend(enrichment::vrl_functions()); @@ -85,7 +93,24 @@ fn compile(mut input: Input) -> Result { let state = TypeState::default(); let mut runtime = Runtime::default(); let config = CompileConfig::default(); - let timezone = TimeZone::default(); + + let timezone = match tz_str.as_deref() { + // Empty or "Default" tz string will default to tz default + None | Some("") | Some("Default") => TimeZone::default(), + Some(other) => match other.parse() { + Ok(tz) => TimeZone::Named(tz), + Err(_) => { + // Returns error message if tz parsing has failed. + // This avoids head scratching, instead of it silently using the default timezone. + let error_message = format!("Invalid timezone identifier: '{other}'"); + return Err(VrlDiagnosticResult { + list: vec![error_message.clone()], + msg: error_message.clone(), + msg_colorized: error_message, + }); + } + }, + }; let mut target_value = TargetValue { value: event.clone(), @@ -98,18 +123,32 @@ fn compile(mut input: Input) -> Result { Err(diagnostics) => return Err(VrlDiagnosticResult::new(&input.program, diagnostics)), }; - match runtime.resolve(&mut target_value, &program.program, &timezone) { - Ok(result) => Ok(VrlCompileResult::new(result, target_value.value)), + let (result, elapsed_time) = + if let Some(performance) = web_sys::window().and_then(|w| w.performance()) { + let start_time = performance.now(); + let result = runtime.resolve(&mut target_value, &program.program, &timezone); + let end_time = performance.now(); + (result, Some(end_time - start_time)) + } else { + // If performance API is not available, run the program without timing. + let result = runtime.resolve(&mut target_value, &program.program, &timezone); + (result, None) + }; + + match result { + // The final event is in `target_value.value`. + // The value of the last expression is in `res`. + Ok(res) => Ok(VrlCompileResult::new(res, target_value.value, elapsed_time)), Err(err) => Err(VrlDiagnosticResult::new_runtime_error(&input.program, err)), } } // The user-facing function #[wasm_bindgen] -pub fn run_vrl(incoming: &JsValue) -> JsValue { +pub fn run_vrl(incoming: &JsValue, tz_str: &str) -> JsValue { let input: Input = incoming.into_serde().unwrap(); - match compile(input) { + match compile(input, Some(tz_str.to_string())) { Ok(res) => JsValue::from_serde(&res).unwrap(), Err(err) => JsValue::from_serde(&err).unwrap(), } @@ -129,6 +168,7 @@ pub fn vector_link() -> String { pub fn vrl_version() -> String { built_info::VRL_VERSION.to_string() } + #[wasm_bindgen] pub fn vrl_link() -> String { built_info::VRL_LINK.to_string() diff --git a/regression/cases/datadog_agent_remap_blackhole/experiment.yaml b/regression/cases/datadog_agent_remap_blackhole/experiment.yaml index 3ab8416137..3650e2c115 100644 --- a/regression/cases/datadog_agent_remap_blackhole/experiment.yaml +++ b/regression/cases/datadog_agent_remap_blackhole/experiment.yaml @@ -3,7 +3,7 @@ optimization_goal: ingress_throughput target: name: vector command: /usr/bin/vector - cpu_allotment: 7 + cpu_allotment: 6 memory_allotment: 8GiB environment: diff --git a/regression/cases/datadog_agent_remap_blackhole_acks/experiment.yaml b/regression/cases/datadog_agent_remap_blackhole_acks/experiment.yaml index 3ab8416137..3650e2c115 100644 --- a/regression/cases/datadog_agent_remap_blackhole_acks/experiment.yaml +++ b/regression/cases/datadog_agent_remap_blackhole_acks/experiment.yaml @@ -3,7 +3,7 @@ optimization_goal: ingress_throughput target: name: vector command: /usr/bin/vector - cpu_allotment: 7 + cpu_allotment: 6 memory_allotment: 8GiB environment: diff --git a/regression/cases/datadog_agent_remap_datadog_logs/experiment.yaml b/regression/cases/datadog_agent_remap_datadog_logs/experiment.yaml index 3ab8416137..3650e2c115 100644 --- a/regression/cases/datadog_agent_remap_datadog_logs/experiment.yaml +++ b/regression/cases/datadog_agent_remap_datadog_logs/experiment.yaml @@ -3,7 +3,7 @@ optimization_goal: ingress_throughput target: name: vector command: /usr/bin/vector - cpu_allotment: 7 + cpu_allotment: 6 memory_allotment: 8GiB environment: diff --git a/regression/cases/datadog_agent_remap_datadog_logs_acks/experiment.yaml b/regression/cases/datadog_agent_remap_datadog_logs_acks/experiment.yaml index 3ab8416137..3650e2c115 100644 --- a/regression/cases/datadog_agent_remap_datadog_logs_acks/experiment.yaml +++ b/regression/cases/datadog_agent_remap_datadog_logs_acks/experiment.yaml @@ -3,7 +3,7 @@ optimization_goal: ingress_throughput target: name: vector command: /usr/bin/vector - cpu_allotment: 7 + cpu_allotment: 6 memory_allotment: 8GiB environment: diff --git a/regression/cases/file_to_blackhole/experiment.yaml b/regression/cases/file_to_blackhole/experiment.yaml index 7675acfd30..d479f10750 100644 --- a/regression/cases/file_to_blackhole/experiment.yaml +++ b/regression/cases/file_to_blackhole/experiment.yaml @@ -4,7 +4,7 @@ erratic: true target: name: vector command: /usr/bin/vector - cpu_allotment: 7 + cpu_allotment: 6 memory_allotment: 8GiB environment: diff --git a/regression/cases/file_to_blackhole/lading/lading.yaml b/regression/cases/file_to_blackhole/lading/lading.yaml index 2b718c6b55..5017287363 100644 --- a/regression/cases/file_to_blackhole/lading/lading.yaml +++ b/regression/cases/file_to_blackhole/lading/lading.yaml @@ -1,14 +1,17 @@ generator: - file_gen: - traditional: + logrotate_fs: seed: [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131] - path_template: "/tmp/file-gen-%NNN%.log" - duplicates: 4 + load_profile: + constant: 100MiB + concurrent_logs: 4 + maximum_bytes_per_log: 100MiB + total_rotations: 5 + max_depth: 0 variant: "ascii" - bytes_per_second: "100Mb" - maximum_bytes_per_file: "100Mb" - maximum_prebuild_cache_size_bytes: "400Mb" + maximum_prebuild_cache_size_bytes: 250MiB + mount_point: /smp-shared blackhole: - tcp: diff --git a/regression/cases/file_to_blackhole/vector/vector.yaml b/regression/cases/file_to_blackhole/vector/vector.yaml index 48edf30e0e..377ae72154 100644 --- a/regression/cases/file_to_blackhole/vector/vector.yaml +++ b/regression/cases/file_to_blackhole/vector/vector.yaml @@ -11,7 +11,7 @@ sources: file: type: "file" include: - - "/tmp/file-gen-*.log" + - "/smp-shared/*.log" ## ## Sinks diff --git a/regression/cases/fluent_elasticsearch/experiment.yaml b/regression/cases/fluent_elasticsearch/experiment.yaml index 3ab8416137..3650e2c115 100644 --- a/regression/cases/fluent_elasticsearch/experiment.yaml +++ b/regression/cases/fluent_elasticsearch/experiment.yaml @@ -3,7 +3,7 @@ optimization_goal: ingress_throughput target: name: vector command: /usr/bin/vector - cpu_allotment: 7 + cpu_allotment: 6 memory_allotment: 8GiB environment: diff --git a/regression/cases/http_elasticsearch/experiment.yaml b/regression/cases/http_elasticsearch/experiment.yaml index 3ab8416137..3650e2c115 100644 --- a/regression/cases/http_elasticsearch/experiment.yaml +++ b/regression/cases/http_elasticsearch/experiment.yaml @@ -3,7 +3,7 @@ optimization_goal: ingress_throughput target: name: vector command: /usr/bin/vector - cpu_allotment: 7 + cpu_allotment: 6 memory_allotment: 8GiB environment: diff --git a/regression/cases/http_text_to_http_json/experiment.yaml b/regression/cases/http_text_to_http_json/experiment.yaml index 3ab8416137..3650e2c115 100644 --- a/regression/cases/http_text_to_http_json/experiment.yaml +++ b/regression/cases/http_text_to_http_json/experiment.yaml @@ -3,7 +3,7 @@ optimization_goal: ingress_throughput target: name: vector command: /usr/bin/vector - cpu_allotment: 7 + cpu_allotment: 6 memory_allotment: 8GiB environment: diff --git a/regression/cases/http_to_http_acks/experiment.yaml b/regression/cases/http_to_http_acks/experiment.yaml index 3ab8416137..3650e2c115 100644 --- a/regression/cases/http_to_http_acks/experiment.yaml +++ b/regression/cases/http_to_http_acks/experiment.yaml @@ -3,7 +3,7 @@ optimization_goal: ingress_throughput target: name: vector command: /usr/bin/vector - cpu_allotment: 7 + cpu_allotment: 6 memory_allotment: 8GiB environment: diff --git a/regression/cases/http_to_http_json/experiment.yaml b/regression/cases/http_to_http_json/experiment.yaml index 3ab8416137..3650e2c115 100644 --- a/regression/cases/http_to_http_json/experiment.yaml +++ b/regression/cases/http_to_http_json/experiment.yaml @@ -3,7 +3,7 @@ optimization_goal: ingress_throughput target: name: vector command: /usr/bin/vector - cpu_allotment: 7 + cpu_allotment: 6 memory_allotment: 8GiB environment: diff --git a/regression/cases/http_to_http_noack/experiment.yaml b/regression/cases/http_to_http_noack/experiment.yaml index 3ab8416137..3650e2c115 100644 --- a/regression/cases/http_to_http_noack/experiment.yaml +++ b/regression/cases/http_to_http_noack/experiment.yaml @@ -3,7 +3,7 @@ optimization_goal: ingress_throughput target: name: vector command: /usr/bin/vector - cpu_allotment: 7 + cpu_allotment: 6 memory_allotment: 8GiB environment: diff --git a/regression/cases/http_to_s3/experiment.yaml b/regression/cases/http_to_s3/experiment.yaml index 3ab8416137..3650e2c115 100644 --- a/regression/cases/http_to_s3/experiment.yaml +++ b/regression/cases/http_to_s3/experiment.yaml @@ -3,7 +3,7 @@ optimization_goal: ingress_throughput target: name: vector command: /usr/bin/vector - cpu_allotment: 7 + cpu_allotment: 6 memory_allotment: 8GiB environment: diff --git a/regression/cases/otlp_grpc_to_blackhole/experiment.yaml b/regression/cases/otlp_grpc_to_blackhole/experiment.yaml index 3ab8416137..3650e2c115 100644 --- a/regression/cases/otlp_grpc_to_blackhole/experiment.yaml +++ b/regression/cases/otlp_grpc_to_blackhole/experiment.yaml @@ -3,7 +3,7 @@ optimization_goal: ingress_throughput target: name: vector command: /usr/bin/vector - cpu_allotment: 7 + cpu_allotment: 6 memory_allotment: 8GiB environment: diff --git a/regression/cases/otlp_http_to_blackhole/experiment.yaml b/regression/cases/otlp_http_to_blackhole/experiment.yaml index 3ab8416137..3650e2c115 100644 --- a/regression/cases/otlp_http_to_blackhole/experiment.yaml +++ b/regression/cases/otlp_http_to_blackhole/experiment.yaml @@ -3,7 +3,7 @@ optimization_goal: ingress_throughput target: name: vector command: /usr/bin/vector - cpu_allotment: 7 + cpu_allotment: 6 memory_allotment: 8GiB environment: diff --git a/regression/cases/socket_to_socket_blackhole/experiment.yaml b/regression/cases/socket_to_socket_blackhole/experiment.yaml index 3ab8416137..3650e2c115 100644 --- a/regression/cases/socket_to_socket_blackhole/experiment.yaml +++ b/regression/cases/socket_to_socket_blackhole/experiment.yaml @@ -3,7 +3,7 @@ optimization_goal: ingress_throughput target: name: vector command: /usr/bin/vector - cpu_allotment: 7 + cpu_allotment: 6 memory_allotment: 8GiB environment: diff --git a/regression/cases/splunk_hec_indexer_ack_blackhole/experiment.yaml b/regression/cases/splunk_hec_indexer_ack_blackhole/experiment.yaml index 3ab8416137..3650e2c115 100644 --- a/regression/cases/splunk_hec_indexer_ack_blackhole/experiment.yaml +++ b/regression/cases/splunk_hec_indexer_ack_blackhole/experiment.yaml @@ -3,7 +3,7 @@ optimization_goal: ingress_throughput target: name: vector command: /usr/bin/vector - cpu_allotment: 7 + cpu_allotment: 6 memory_allotment: 8GiB environment: diff --git a/regression/cases/splunk_hec_route_s3/experiment.yaml b/regression/cases/splunk_hec_route_s3/experiment.yaml index 3ab8416137..3650e2c115 100644 --- a/regression/cases/splunk_hec_route_s3/experiment.yaml +++ b/regression/cases/splunk_hec_route_s3/experiment.yaml @@ -3,7 +3,7 @@ optimization_goal: ingress_throughput target: name: vector command: /usr/bin/vector - cpu_allotment: 7 + cpu_allotment: 6 memory_allotment: 8GiB environment: diff --git a/regression/cases/splunk_hec_to_splunk_hec_logs_acks/experiment.yaml b/regression/cases/splunk_hec_to_splunk_hec_logs_acks/experiment.yaml index 3ab8416137..3650e2c115 100644 --- a/regression/cases/splunk_hec_to_splunk_hec_logs_acks/experiment.yaml +++ b/regression/cases/splunk_hec_to_splunk_hec_logs_acks/experiment.yaml @@ -3,7 +3,7 @@ optimization_goal: ingress_throughput target: name: vector command: /usr/bin/vector - cpu_allotment: 7 + cpu_allotment: 6 memory_allotment: 8GiB environment: diff --git a/regression/cases/splunk_hec_to_splunk_hec_logs_noack/experiment.yaml b/regression/cases/splunk_hec_to_splunk_hec_logs_noack/experiment.yaml index 3ab8416137..3650e2c115 100644 --- a/regression/cases/splunk_hec_to_splunk_hec_logs_noack/experiment.yaml +++ b/regression/cases/splunk_hec_to_splunk_hec_logs_noack/experiment.yaml @@ -3,7 +3,7 @@ optimization_goal: ingress_throughput target: name: vector command: /usr/bin/vector - cpu_allotment: 7 + cpu_allotment: 6 memory_allotment: 8GiB environment: diff --git a/regression/cases/syslog_humio_logs/experiment.yaml b/regression/cases/syslog_humio_logs/experiment.yaml index 3ab8416137..3650e2c115 100644 --- a/regression/cases/syslog_humio_logs/experiment.yaml +++ b/regression/cases/syslog_humio_logs/experiment.yaml @@ -3,7 +3,7 @@ optimization_goal: ingress_throughput target: name: vector command: /usr/bin/vector - cpu_allotment: 7 + cpu_allotment: 6 memory_allotment: 8GiB environment: diff --git a/regression/cases/syslog_log2metric_humio_metrics/experiment.yaml b/regression/cases/syslog_log2metric_humio_metrics/experiment.yaml index 3ab8416137..3650e2c115 100644 --- a/regression/cases/syslog_log2metric_humio_metrics/experiment.yaml +++ b/regression/cases/syslog_log2metric_humio_metrics/experiment.yaml @@ -3,7 +3,7 @@ optimization_goal: ingress_throughput target: name: vector command: /usr/bin/vector - cpu_allotment: 7 + cpu_allotment: 6 memory_allotment: 8GiB environment: diff --git a/regression/cases/syslog_log2metric_splunk_hec_metrics/experiment.yaml b/regression/cases/syslog_log2metric_splunk_hec_metrics/experiment.yaml index 3ab8416137..3650e2c115 100644 --- a/regression/cases/syslog_log2metric_splunk_hec_metrics/experiment.yaml +++ b/regression/cases/syslog_log2metric_splunk_hec_metrics/experiment.yaml @@ -3,7 +3,7 @@ optimization_goal: ingress_throughput target: name: vector command: /usr/bin/vector - cpu_allotment: 7 + cpu_allotment: 6 memory_allotment: 8GiB environment: diff --git a/regression/cases/syslog_log2metric_tag_cardinality_limit_blackhole/experiment.yaml b/regression/cases/syslog_log2metric_tag_cardinality_limit_blackhole/experiment.yaml index 3ab8416137..3650e2c115 100644 --- a/regression/cases/syslog_log2metric_tag_cardinality_limit_blackhole/experiment.yaml +++ b/regression/cases/syslog_log2metric_tag_cardinality_limit_blackhole/experiment.yaml @@ -3,7 +3,7 @@ optimization_goal: ingress_throughput target: name: vector command: /usr/bin/vector - cpu_allotment: 7 + cpu_allotment: 6 memory_allotment: 8GiB environment: diff --git a/regression/cases/syslog_loki/experiment.yaml b/regression/cases/syslog_loki/experiment.yaml index 3ab8416137..3650e2c115 100644 --- a/regression/cases/syslog_loki/experiment.yaml +++ b/regression/cases/syslog_loki/experiment.yaml @@ -3,7 +3,7 @@ optimization_goal: ingress_throughput target: name: vector command: /usr/bin/vector - cpu_allotment: 7 + cpu_allotment: 6 memory_allotment: 8GiB environment: diff --git a/regression/cases/syslog_regex_logs2metric_ddmetrics/experiment.yaml b/regression/cases/syslog_regex_logs2metric_ddmetrics/experiment.yaml index 3ab8416137..3650e2c115 100644 --- a/regression/cases/syslog_regex_logs2metric_ddmetrics/experiment.yaml +++ b/regression/cases/syslog_regex_logs2metric_ddmetrics/experiment.yaml @@ -3,7 +3,7 @@ optimization_goal: ingress_throughput target: name: vector command: /usr/bin/vector - cpu_allotment: 7 + cpu_allotment: 6 memory_allotment: 8GiB environment: diff --git a/regression/cases/syslog_splunk_hec_logs/experiment.yaml b/regression/cases/syslog_splunk_hec_logs/experiment.yaml index 3ab8416137..3650e2c115 100644 --- a/regression/cases/syslog_splunk_hec_logs/experiment.yaml +++ b/regression/cases/syslog_splunk_hec_logs/experiment.yaml @@ -3,7 +3,7 @@ optimization_goal: ingress_throughput target: name: vector command: /usr/bin/vector - cpu_allotment: 7 + cpu_allotment: 6 memory_allotment: 8GiB environment: diff --git a/regression/config.yaml b/regression/config.yaml index cc37662a0a..b9b54de25b 100644 --- a/regression/config.yaml +++ b/regression/config.yaml @@ -1,5 +1,5 @@ lading: - version: 0.25.3 + version: 0.25.4 target: diff --git a/rfcs/2020-03-06-1999-api-extensions-for-lua-transform.md b/rfcs/2020-03-06-1999-api-extensions-for-lua-transform.md index d2bcb02004..4b8b75c032 100644 --- a/rfcs/2020-03-06-1999-api-extensions-for-lua-transform.md +++ b/rfcs/2020-03-06-1999-api-extensions-for-lua-transform.md @@ -488,7 +488,7 @@ Events produced by the transforms through calling an emitting function can have Both log and metrics events are encoded using [external tagging](https://serde.rs/enum-representations.html#externally-tagged). -* [Log events](https://vector.dev/docs/about/data-model/log/) could be seen as tables created using +* [Log events](https://vector.dev/docs/architecture/data-model/log/) could be seen as tables created using ```lua { @@ -498,7 +498,7 @@ Both log and metrics events are encoded using [external tagging](https://serde.r } ``` - The content of the `log` field corresponds to the usual [log event](https://vector.dev/docs/about/data-model/log/#examples) structure, with possible nesting of the fields. + The content of the `log` field corresponds to the usual [log event](https://vector.dev/docs/architecture/data-model/log/#examples) structure, with possible nesting of the fields. If a log event is created by the user inside the transform is a table, then, if default fields named according to the [global schema](https://vector.dev/docs/reference/global-options/#log_schema) are not present in such a table, then they are automatically added to the event. This rule does not apply to events having `userdata` type. @@ -532,7 +532,7 @@ Both log and metrics events are encoded using [external tagging](https://serde.r > > And then emits the event. In that case Vector would not automatically insert the `timestamp` field. -* [Metric events](https://vector.dev/docs/about/data-model/metric/) could be seen as tables created using +* [Metric events](https://vector.dev/docs/architecture/data-model/metric/) could be seen as tables created using ```lua { @@ -542,7 +542,7 @@ Both log and metrics events are encoded using [external tagging](https://serde.r } ``` - The content of the `metric` field matches the [metric data model](https://vector.dev/docs/about/data-model/metric). The values use [external tagging](https://serde.rs/enum-representations.html#externally-tagged) with respect to the metric type, see the examples. + The content of the `metric` field matches the [metric data model](https://vector.dev/docs/architecture/data-model/metric). The values use [external tagging](https://serde.rs/enum-representations.html#externally-tagged) with respect to the metric type, see the examples. In case when the metric events are created as tables in user-defined code, the following default values are assumed if they are not provided: @@ -552,7 +552,7 @@ Both log and metrics events are encoded using [external tagging](https://serde.r | `kind` | `absolute` | | `tags` | empty map | - Furthermore, for [`aggregated_histogram`](https://vector.dev/docs/about/data-model/metric/#aggregated_histogram) the `count` field inside the `value` map can be omitted. + Furthermore, for [`aggregated_histogram`](https://vector.dev/docs/architecture/data-model/metric/#aggregated_histogram) the `count` field inside the `value` map can be omitted. **Example: `counter`** @@ -621,7 +621,7 @@ Both log and metrics events are encoded using [external tagging](https://serde.r > } > } > } - > Note that the field [`count`](https://vector.dev/docs/about/data-model/metric/#count) is not required because it can be inferred by Vector automatically by summing up the values from `counts`. + > Note that the field [`count`](https://vector.dev/docs/architecture/data-model/metric/#count) is not required because it can be inferred by Vector automatically by summing up the values from `counts`. **Example: `aggregated_summary`** > The minimal Lua code required to create an aggregated summary metric is the following: @@ -645,14 +645,14 @@ The mapping between Vector data types and Lua data types is the following: | Vector Type | Lua Type | Comment | | :----------- | :-------- | :------- | -| [`String`](https://vector.dev/docs/about/data-model/log/#strings) | [`string`](https://www.lua.org/pil/2.4.html) || -| [`Integer`](https://vector.dev/docs/about/data-model/log/#ints) | [`integer`](https://docs.rs/mlua/0.6.0/mlua/type.Integer.html) || -| [`Float`](https://vector.dev/docs/about/data-model/log/#floats) | [`number`](https://docs.rs/mlua/0.6.0/mlua/type.Number.html) || -| [`Boolean`](https://vector.dev/docs/about/data-model/log/#booleans) | [`boolean`](https://www.lua.org/pil/2.2.html) || -| [`Timestamp`](https://vector.dev/docs/about/data-model/log/#timestamps) | [`userdata`](https://www.lua.org/pil/28.1.html) | There is no dedicated timestamp type in Lua. However, there is a standard library function [`os.date`](https://www.lua.org/manual/5.1/manual.html#pdf-os.date) which returns a table with fields `year`, `month`, `day`, `hour`, `min`, `sec`, and some others. Other standard library functions, such as [`os.time`](https://www.lua.org/manual/5.1/manual.html#pdf-os.time), support tables with these fields as arguments. Because of that, Vector timestamps passed to the transform are represented as `userdata` with the same set of accessible fields. In order to have one-to-one correspondence between Vector timestamps and Lua timestamps, `os.date` function from the standard library is patched to return not a table, but `userdata` with the same set of fields as it usually would return instead. This approach makes it possible to have both compatibility with the standard library functions and a dedicated data type for timestamps. | -| [`Null`](https://vector.dev/docs/about/data-model/log/#null-values) | empty string | In Lua setting a table field to `nil` means deletion of this field. Furthermore, setting an array element to `nil` leads to deletion of this element. In order to avoid inconsistencies, already present `Null` values are visible represented as empty strings from Lua code, and it is impossible to create a new `Null` value in the user-defined code. | -| [`Map`](https://vector.dev/docs/about/data-model/log/#maps) | [`userdata`](https://www.lua.org/pil/28.1.html) or [`table`](https://www.lua.org/pil/2.5.html) | Maps which are parts of events passed to the transform from Vector have `userdata` type. User-created maps have `table` type. Both types are converted to Vector's `Map` type when they are emitted from the transform. | -| [`Array`](https://vector.dev/docs/about/data-model/log/#arrays) | [`sequence`](https://www.lua.org/pil/11.1.html) | Sequences in Lua are a special case of tables. Because of that fact, the indexes can in principle start from any number. However, the convention in Lua is to start indexes from 1 instead of 0, so Vector should adhere it. | +| [`String`](https://vector.dev/docs/architecture/data-model/log/#strings) | [`string`](https://www.lua.org/pil/2.4.html) || +| [`Integer`](https://vector.dev/docs/architecture/data-model/log/#ints) | [`integer`](https://docs.rs/mlua/0.6.0/mlua/type.Integer.html) || +| [`Float`](https://vector.dev/docs/architecture/data-model/log/#floats) | [`number`](https://docs.rs/mlua/0.6.0/mlua/type.Number.html) || +| [`Boolean`](https://vector.dev/docs/architecture/data-model/log/#booleans) | [`boolean`](https://www.lua.org/pil/2.2.html) || +| [`Timestamp`](https://vector.dev/docs/architecture/data-model/log/#timestamps) | [`userdata`](https://www.lua.org/pil/28.1.html) | There is no dedicated timestamp type in Lua. However, there is a standard library function [`os.date`](https://www.lua.org/manual/5.1/manual.html#pdf-os.date) which returns a table with fields `year`, `month`, `day`, `hour`, `min`, `sec`, and some others. Other standard library functions, such as [`os.time`](https://www.lua.org/manual/5.1/manual.html#pdf-os.time), support tables with these fields as arguments. Because of that, Vector timestamps passed to the transform are represented as `userdata` with the same set of accessible fields. In order to have one-to-one correspondence between Vector timestamps and Lua timestamps, `os.date` function from the standard library is patched to return not a table, but `userdata` with the same set of fields as it usually would return instead. This approach makes it possible to have both compatibility with the standard library functions and a dedicated data type for timestamps. | +| [`Null`](https://vector.dev/docs/architecture/data-model/log/#null-values) | empty string | In Lua setting a table field to `nil` means deletion of this field. Furthermore, setting an array element to `nil` leads to deletion of this element. In order to avoid inconsistencies, already present `Null` values are visible represented as empty strings from Lua code, and it is impossible to create a new `Null` value in the user-defined code. | +| [`Map`](https://vector.dev/docs/architecture/data-model/log/#maps) | [`userdata`](https://www.lua.org/pil/28.1.html) or [`table`](https://www.lua.org/pil/2.5.html) | Maps which are parts of events passed to the transform from Vector have `userdata` type. User-created maps have `table` type. Both types are converted to Vector's `Map` type when they are emitted from the transform. | +| [`Array`](https://vector.dev/docs/architecture/data-model/log/#arrays) | [`sequence`](https://www.lua.org/pil/11.1.html) | Sequences in Lua are a special case of tables. Because of that fact, the indexes can in principle start from any number. However, the convention in Lua is to start indexes from 1 instead of 0, so Vector should adhere it. | ### Configuration @@ -679,7 +679,7 @@ The implementation of `lua` transform supports only log events. Processing of lo Events have type [`userdata`](https://www.lua.org/pil/28.1.html) with custom [metamethods](https://www.lua.org/pil/13.html), so they are views to Vector's events. Thus passing an event to Lua has zero cost, so only when fields are actually accessed the data is copied to Lua. -The fields are accessed through string indexes using [Vector's field path notation](https://vector.dev/docs/about/data-model/log/). +The fields are accessed through string indexes using [Vector's field path notation](https://vector.dev/docs/architecture/data-model/log/). ## Sales Pitch diff --git a/rfcs/2020-05-25-2692-more-usable-logevents.md b/rfcs/2020-05-25-2692-more-usable-logevents.md index 73e50f5231..09ab89b35d 100644 --- a/rfcs/2020-05-25-2692-more-usable-logevents.md +++ b/rfcs/2020-05-25-2692-more-usable-logevents.md @@ -162,7 +162,7 @@ There is no guide accompanying this RFC, it only minimally touches user facing s ## Doc Level Proposal -> **Placement:** Insert into [Log Event](https://vector.dev/docs/about/data-model/log/#types)'s [Types](https://vector.dev/docs/about/data-model/log/#types) section +> **Placement:** Insert into [Log Event](https://vector.dev/docs/architecture/data-model/log/#types)'s [Types](https://vector.dev/docs/architecture/data-model/log/#types) section ### Bytes diff --git a/rfcs/2020-07-28-3642-jmx_rfc.md b/rfcs/2020-07-28-3642-jmx_rfc.md index fb3a98d32c..b872b27657 100644 --- a/rfcs/2020-07-28-3642-jmx_rfc.md +++ b/rfcs/2020-07-28-3642-jmx_rfc.md @@ -223,7 +223,7 @@ principles of Vector: > One Tool. All Data. - One simple tool gets your logs, metrics, and traces > (coming soon) from A to B. -[Vector principles](https://vector.dev/docs/about/what-is-vector/#who-should-use-vector) +[Vector principles](https://vector.dev/docs/) If users are already running Prometheus though, they could opt for the Prometheus path. diff --git a/rfcs/2020-08-21-3092-apache-metrics-source.md b/rfcs/2020-08-21-3092-apache-metrics-source.md index c7ab4a9162..4b285632f1 100644 --- a/rfcs/2020-08-21-3092-apache-metrics-source.md +++ b/rfcs/2020-08-21-3092-apache-metrics-source.md @@ -166,8 +166,7 @@ principles of Vector: > One Tool. All Data. - One simple tool gets your logs, metrics, and traces > (coming soon) from A to B. -[Vector -principles](https://vector.dev/docs/about/what-is-vector/#who-should-use-vector) +[Vector principles](https://vector.dev/docs/) On the same page, it is mentioned that Vector should be a replacement for Telegraf. diff --git a/rfcs/2020-08-26-3191-host-metrics.md b/rfcs/2020-08-26-3191-host-metrics.md index f494030000..58243ddf1e 100644 --- a/rfcs/2020-08-26-3191-host-metrics.md +++ b/rfcs/2020-08-26-3191-host-metrics.md @@ -180,7 +180,7 @@ principles of Vector: > (coming soon) from A to B. [Vector -principles](https://vector.dev/docs/about/what-is-vector/#who-should-use-vector) +principles](https://vector.dev/docs/) On the same page, it is mentioned that Vector should be a replacement for Telegraf. diff --git a/rfcs/2020-08-27-3603-postgres-metrics.md b/rfcs/2020-08-27-3603-postgres-metrics.md index cbc7facbdc..2b47b4e07e 100644 --- a/rfcs/2020-08-27-3603-postgres-metrics.md +++ b/rfcs/2020-08-27-3603-postgres-metrics.md @@ -136,7 +136,7 @@ principles of Vector: > (coming soon) from A to B. [Vector -principles](https://vector.dev/docs/about/what-is-vector/#who-should-use-vector) +principles](https://vector.dev/docs/) On the same page, it is mentioned that Vector should be a replacement for Telegraf. diff --git a/rfcs/2020-08-31-3640-nginx-metrics-source.md b/rfcs/2020-08-31-3640-nginx-metrics-source.md index 5351aa1351..93551a5149 100644 --- a/rfcs/2020-08-31-3640-nginx-metrics-source.md +++ b/rfcs/2020-08-31-3640-nginx-metrics-source.md @@ -131,7 +131,7 @@ principles of Vector: > (coming soon) from A to B. [Vector -principles](https://vector.dev/docs/about/what-is-vector/#who-should-use-vector) +principles](https://vector.dev/docs/) On the same page, it is mentioned that Vector should be a replacement for Telegraf. diff --git a/rfcs/2020-08-31-3641-mongo-metrics.md b/rfcs/2020-08-31-3641-mongo-metrics.md index ca100cf457..32949f59b1 100644 --- a/rfcs/2020-08-31-3641-mongo-metrics.md +++ b/rfcs/2020-08-31-3641-mongo-metrics.md @@ -1107,7 +1107,7 @@ principles of Vector: > (coming soon) from A to B. [Vector -principles](https://vector.dev/docs/about/what-is-vector/#who-should-use-vector) +principles](https://vector.dev/docs/) On the same page, it is mentioned that Vector should be a replacement for Telegraf. diff --git a/rfcs/2021-09-01-8547-accept-metrics-in-datadog-agent-source.md b/rfcs/2021-09-01-8547-accept-metrics-in-datadog-agent-source.md index aec7cd9db8..ac95ef6267 100644 --- a/rfcs/2021-09-01-8547-accept-metrics-in-datadog-agent-source.md +++ b/rfcs/2021-09-01-8547-accept-metrics-in-datadog-agent-source.md @@ -116,7 +116,7 @@ A few details about the Datadog Agents & [Datadog metrics](https://docs.datadogh [here](https://github.com/DataDog/agent-payload/blob/master/proto/metrics/agent_payload.proto#L47-L81). Vector has a nice description of its [metrics data -model](https://vector.dev/docs/about/under-the-hood/architecture/data-model/metric/) and a [concise enum for +model](https://vector.dev/docs/architecture/data-model/metric/) and a [concise enum for representing it](https://github.com/vectordotdev/vector/blob/master/lib/vector-core/src/event/metric.rs#L135-L169). diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 2a53366289..b6981e6398 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,3 +1,3 @@ [toolchain] -channel = "1.83" +channel = "1.89" profile = "default" diff --git a/scripts/build-docker.sh b/scripts/build-docker.sh index 14a53e4a97..b05e488029 100755 --- a/scripts/build-docker.sh +++ b/scripts/build-docker.sh @@ -15,7 +15,8 @@ VERSION="${VECTOR_VERSION:-"$(cargo vdev version)"}" DATE="${DATE:-"$(date -u +%Y-%m-%d)"}" PLATFORM="${PLATFORM:-}" PUSH="${PUSH:-"true"}" -REPO="${REPO:-"timberio/vector"}" +REPOS="${REPOS:-"timberio/vector"}" +IFS=, read -ra REPO_LIST <<< "$REPOS" IFS=, read -ra REQUESTED_PLATFORMS <<< "$PLATFORM" declare -A SUPPORTED_PLATFORMS=( @@ -50,34 +51,42 @@ evaluate_supported_platforms_for_base() { build() { local BASE="$1" local VERSION="$2" - - local TAG="$REPO:$VERSION-$BASE" local DOCKERFILE="distribution/docker/$BASE/Dockerfile" + local BUILDABLE_PLATFORMS="" + if [ -n "$PLATFORM" ]; then + BUILDABLE_PLATFORMS=$(evaluate_supported_platforms_for_base "$BASE") + fi + # Collect all tags + TAGS=() + for REPO in "${REPO_LIST[@]}"; do + TAGS+=(--tag "$REPO:$VERSION-$BASE") + done + + # Build once with all tags if [ -n "$PLATFORM" ]; then ARGS=() if [[ "$PUSH" == "true" ]]; then ARGS+=(--push) fi - local BUILDABLE_PLATFORMS - BUILDABLE_PLATFORMS=$(evaluate_supported_platforms_for_base "$BASE") - docker buildx build \ --platform="$BUILDABLE_PLATFORMS" \ - --tag "$TAG" \ + "${TAGS[@]}" \ target/artifacts \ -f "$DOCKERFILE" \ "${ARGS[@]}" else docker build \ - --tag "$TAG" \ + "${TAGS[@]}" \ target/artifacts \ -f "$DOCKERFILE" - if [[ "$PUSH" == "true" ]]; then + if [[ "$PUSH" == "true" ]]; then + for TAG in "${TAGS[@]}"; do docker push "$TAG" - fi + done + fi fi } @@ -85,7 +94,7 @@ build() { # Build # -echo "Building $REPO:* Docker images" +echo "Building Docker images for $REPOS" if [[ "$CHANNEL" == "release" ]]; then VERSION_EXACT="$VERSION" diff --git a/scripts/check-one-feature b/scripts/check-one-feature index c380ab6b36..92bba0ceed 100755 --- a/scripts/check-one-feature +++ b/scripts/check-one-feature @@ -55,6 +55,7 @@ then else exit=$? cat $log + echo "Error: Failed to run $0 $feature" fi exit $exit diff --git a/scripts/check_changelog_fragments.sh b/scripts/check_changelog_fragments.sh index 731d4ea34f..916ebe88dd 100755 --- a/scripts/check_changelog_fragments.sh +++ b/scripts/check_changelog_fragments.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # This script is intended to run during CI, however it can be run locally by # committing changelog fragments before executing the script. If the script @@ -17,16 +17,18 @@ if [ ! -d "${CHANGELOG_DIR}" ]; then fi # diff-filter=A lists only added files -FRAGMENTS=$(git diff --name-only --diff-filter=A --merge-base origin/master ${CHANGELOG_DIR}) +FRAGMENTS=$(git diff --name-only --diff-filter=A --merge-base "${MERGE_BASE:-origin/master}" ${CHANGELOG_DIR}) if [ -z "$FRAGMENTS" ]; then echo "No changelog fragments detected" - echo "If no changes necessitate user-facing explanations, add the GH label 'no-changelog'" + echo "If no changes necessitate user-facing explanations, add the GH label 'no-changelog'" echo "Otherwise, add changelog fragments to changelog.d/" echo "For details, see 'changelog.d/README.md'" exit 1 fi +[[ "$(wc -l <<< "$FRAGMENTS")" -gt "${MAX_FRAGMENTS:-1000}" ]] && exit 1 + # extract the basename from the file path FRAGMENTS=$(xargs -n1 basename <<< "${FRAGMENTS}") diff --git a/scripts/ci-int-e2e-test.sh b/scripts/ci-int-e2e-test.sh deleted file mode 100755 index 3b569db530..0000000000 --- a/scripts/ci-int-e2e-test.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -# Used in CI to run and stop an integration test and upload the results of it. -# This is useful to allow retrying the integration test at a higher level than -# the nextest and reduce code duplication in the workflow file. - -set -u - -if [[ -z "${CI:-}" ]]; then - echo "Aborted: this script is for use in CI." >&2 - exit 1 -fi - -if [ $# -ne 2 ] -then - echo "usage: $0 [int|e2e] TEST_NAME" - exit 1 -fi - -set -x - -TEST_TYPE=$1 # either "int" or "e2e" -TEST_NAME=$2 - -cargo vdev -v "${TEST_TYPE}" start -a "${TEST_NAME}" -sleep 30 -cargo vdev -v "${TEST_TYPE}" test --retries 2 -a "${TEST_NAME}" -RET=$? -cargo vdev -v "${TEST_TYPE}" stop -a "${TEST_NAME}" -./scripts/upload-test-results.sh -exit $RET diff --git a/scripts/e2e/Dockerfile b/scripts/e2e/Dockerfile index 495c69275c..3af5d83489 100644 --- a/scripts/e2e/Dockerfile +++ b/scripts/e2e/Dockerfile @@ -1,8 +1,7 @@ -ARG RUST_VERSION +ARG RUST_VERSION=1.85 ARG FEATURES -ARG DEBIAN_RELEASE=slim-bookworm -FROM docker.io/rust:${RUST_VERSION}-${DEBIAN_RELEASE} +FROM docker.io/rust:${RUST_VERSION}-slim-bookworm RUN apt-get update && apt-get -y --no-install-recommends install \ build-essential \ @@ -17,18 +16,10 @@ RUN apt-get update && apt-get -y --no-install-recommends install \ libxxhash-dev \ unzip \ zlib1g-dev \ - zlib1g + zlib1g \ + mold -RUN git clone https://github.com/rui314/mold.git \ - && mkdir mold/build \ - && cd mold/build \ - && git checkout v2.0.0 \ - && ../install-build-deps.sh \ - && cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=c++ .. \ - && cmake --build . -j $(nproc) \ - && cmake --install . - -RUN rustup run "${RUST_VERSION}" cargo install cargo-nextest --version 0.9.72 --locked +RUN cargo install cargo-nextest --version 0.9.95 --locked COPY scripts/environment/install-protoc.sh / COPY tests/data/ca/certs /certs @@ -41,6 +32,6 @@ ARG FEATURES RUN --mount=type=cache,target=/vector/target \ --mount=type=cache,target=/usr/local/cargo/registry \ --mount=type=cache,target=/usr/local/cargo/git \ - /usr/local/bin/mold -run cargo build --tests --lib --bin vector \ + /usr/bin/mold -run cargo build --tests --lib --bin vector \ --no-default-features --features $FEATURES && \ cp target/debug/vector /usr/bin/vector diff --git a/scripts/e2e/datadog-logs/compose.yaml b/scripts/e2e/datadog-logs/compose.yaml index 56fb68350c..ae004588f4 100644 --- a/scripts/e2e/datadog-logs/compose.yaml +++ b/scripts/e2e/datadog-logs/compose.yaml @@ -1,4 +1,4 @@ -version: '3' +version: '3.8' services: # Generates random log data for consumption by the custom Agent check @@ -17,7 +17,9 @@ services: - "-o" - "/var/log/a_custom.log" volumes: - - log_path:/var/log/ + - type: volume + source: log_path + target: /var/log/ # Tails a custom log created by `log_generator` and sends log data to # the `fakeintake-agent` service @@ -32,12 +34,22 @@ services: - DD_ENABLE_PAYLOADS_SERVICE_CHECKS=false - DD_CONTAINER_EXCLUDE="name:.*" volumes: - # The Agent config file - - ${PWD}/tests/data/e2e/datadog/logs/agent_only.yaml:/etc/datadog-agent/datadog.yaml - # The custom logs check - - ${PWD}/tests/data/e2e/datadog/logs/logs.conf.d:/conf.d:ro - # The custom log to tail, created by the `log_generator` service - - log_path:/var/log/ + # The Agent config file + - type: bind + source: ../../../tests/data/e2e/datadog/logs/agent_only.yaml + target: /etc/datadog-agent/datadog.yaml + read_only: true + + # The custom logs check + - type: bind + source: ../../../tests/data/e2e/datadog/logs/logs.conf.d + target: /conf.d + read_only: true + + # The custom log to tail, created by the `log_generator` service + - type: volume + source: log_path + target: /var/log/ # Tails a custom log created by `log_generator` and sends log data to # the `vector` service @@ -52,12 +64,22 @@ services: - DD_ENABLE_PAYLOADS_SERVICE_CHECKS=false - DD_CONTAINER_EXCLUDE="name:.*" volumes: - # The Agent config file - - ${PWD}/tests/data/e2e/datadog/logs/agent_vector.yaml:/etc/datadog-agent/datadog.yaml - # The custom logs check - - ${PWD}/tests/data/e2e/datadog/logs/logs.conf.d:/conf.d:ro - # The custom log to tail, created by the `log_generator` service - - log_path:/var/log/ + # The Agent config file + - type: bind + source: ../../../tests/data/e2e/datadog/logs/agent_vector.yaml + target: /etc/datadog-agent/datadog.yaml + read_only: true + + # The custom logs check + - type: bind + source: ../../../tests/data/e2e/datadog/logs/logs.conf.d + target: /conf.d + read_only: true + + # The custom log to tail, created by the `log_generator` service + - type: volume + source: log_path + target: /var/log/ # Receives log data from the `datadog-agent-vector` service and sends # to the `fakeintake-vector` service. @@ -65,7 +87,7 @@ services: depends_on: - fakeintake-vector build: - context: ${PWD} + context: ../../.. # re-using the integration test runner image since it already has # compiled vector on it. image: ${CONFIG_VECTOR_IMAGE} @@ -79,19 +101,21 @@ services: - "-c" - "/home/vector/tests/data/e2e/datadog/logs/vector.toml" volumes: - - ${PWD}:/home/vector + - type: bind + source: ../../.. + target: /home/vector # Receives log data from the `datadog-agent` service. Is queried by the test runner # which does the validation of consistency with the other fakeintake service. fakeintake-agent: # TODO: temporarily pegging the image as latest results in failures - image: docker.io/datadog/fakeintake:v77a06f2b + image: docker.io/datadog/fakeintake:ved764626 # Receives log data from the `datadog-agent-vector` service. Is queried by the test runner # which does the validation of consistency with the other fakeintake service. fakeintake-vector: # TODO: temporarily pegging the image as latest results in failures - image: docker.io/datadog/fakeintake:v77a06f2b + image: docker.io/datadog/fakeintake:ved764626 networks: default: diff --git a/scripts/e2e/datadog-metrics/compose.yaml b/scripts/e2e/datadog-metrics/compose.yaml index 5942fda101..80523ac814 100644 --- a/scripts/e2e/datadog-metrics/compose.yaml +++ b/scripts/e2e/datadog-metrics/compose.yaml @@ -27,8 +27,8 @@ services: - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} - DD_HOSTNAME=datadog-agent volumes: - # The Agent config file - - ${PWD}/tests/data/e2e/datadog/metrics/agent_only.yaml:/etc/datadog-agent/datadog.yaml + # The Agent config file + - ../../../tests/data/e2e/datadog/metrics/agent_only.yaml:/etc/datadog-agent/datadog.yaml # Sends metric data received from the Emitter to the `vector` service datadog-agent-vector: @@ -39,8 +39,8 @@ services: - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} - DD_HOSTNAME=datadog-agent-vector volumes: - # The Agent config file - - ${PWD}/tests/data/e2e/datadog/metrics/agent_vector.yaml:/etc/datadog-agent/datadog.yaml + # The Agent config file + - ../../../tests/data/e2e/datadog/metrics/agent_vector.yaml:/etc/datadog-agent/datadog.yaml # Receives metric data from the `datadog-agent-vector` service and sends # to the `fakeintake-vector` service. @@ -48,7 +48,7 @@ services: depends_on: - fakeintake-vector build: - context: ${PWD} + context: ../../.. # re-using the integration test runner image since it already has # compiled vector on it. image: ${CONFIG_VECTOR_IMAGE} @@ -62,7 +62,7 @@ services: - "-c" - "/home/vector/tests/data/e2e/datadog/metrics/vector.toml" volumes: - - ${PWD}:/home/vector + - ../../..:/home/vector # Receives metric data from the `datadog-agent` service. Is queried by the test runner # which does the validation of consistency with the other fakeintake service. diff --git a/scripts/e2e/datadog-metrics/test.yaml b/scripts/e2e/datadog-metrics/test.yaml index b9025d08eb..d1b814fd02 100644 --- a/scripts/e2e/datadog-metrics/test.yaml +++ b/scripts/e2e/datadog-metrics/test.yaml @@ -24,4 +24,5 @@ paths: - "src/sinks/datadog/metrics/**" - "src/sinks/util/**" - "scripts/integration/datadog-e2e/metrics/**" +- "tests/e2e/datadog/metrics/**" - "tests/data/e2e/datadog/metrics/**" diff --git a/scripts/e2e/opentelemetry-logs/README.md b/scripts/e2e/opentelemetry-logs/README.md new file mode 100644 index 0000000000..ce61906248 --- /dev/null +++ b/scripts/e2e/opentelemetry-logs/README.md @@ -0,0 +1,25 @@ +# OpenTelemetry Vector E2E Log Pipeline Test + +This end-to-end (E2E) test validates that log events generated in a container are correctly ingested by Vector, processed, and forwarded to an OpenTelemetry Collector sink, where they are exported to a file for verification. + +## How this test works + +- **Orchestrates all required services:** + - **Log generator**: Emits fake OTLP logs. + - **Vector**: Receives, transforms, and forwards logs to the OTEL sink and a file. + - **OTEL Collector Source**: Forwards or processes logs upstream. + - **OTEL Collector Sink**: Receives logs from Vector and writes them to a file. +- **Mounts volumes** to share configuration and output files between containers and the host. +- **Exposes ports** for OTLP HTTP ingestion and for accessing Vector/collector APIs if needed. + +## How to Run + +```shell +# from the repo root directory +./scripts/run-integration-test.sh e2e opentelemetry-logs +``` + +## Notes + +- The test ensures true end-to-end delivery and format compliance for OTLP logs through Vector and the OpenTelemetry Collector stack. +- Adjust the log generator, remap logic, or assertions as needed for your use case. diff --git a/scripts/e2e/opentelemetry-logs/compose.yaml b/scripts/e2e/opentelemetry-logs/compose.yaml new file mode 100644 index 0000000000..d721abf6e7 --- /dev/null +++ b/scripts/e2e/opentelemetry-logs/compose.yaml @@ -0,0 +1,70 @@ +name: opentelemetry-vector-e2e +services: + otel-collector-source: + container_name: otel-collector-source + image: otel/opentelemetry-collector-contrib:${CONFIG_COLLECTOR_VERSION} + init: true + volumes: + - type: bind + source: ../../../tests/data/e2e/opentelemetry/logs/collector-source.yaml + target: /etc/otelcol-contrib/config.yaml + read_only: true + ports: + - "${OTEL_COLLECTOR_SOURCE_GRPC_PORT:-4317}:4317" + - "${OTEL_COLLECTOR_SOURCE_HTTP_PORT:-4318}:4318" + command: [ "--config=/etc/otelcol-contrib/config.yaml" ] + + logs-generator: + container_name: logs-generator + build: + context: ./generator + init: true + depends_on: + - otel-collector-source + - vector + - otel-collector-sink + volumes: + - type: bind + source: ./generator + target: /generator + environment: + - PYTHONUNBUFFERED=1 + command: [ "python", "/generator/logs_generator.py", "-n", "100" ] + + otel-collector-sink: + container_name: otel-collector-sink + image: otel/opentelemetry-collector-contrib:${CONFIG_COLLECTOR_VERSION} + init: true + volumes: + - type: bind + source: ../../../tests/data/e2e/opentelemetry/logs/collector-sink.yaml + target: /etc/otelcol-contrib/config.yaml + read_only: true + - type: bind + source: ../../../tests/data/e2e/opentelemetry/logs/output + target: /output + ports: + - "${OTEL_COLLECTOR_SINK_HTTP_PORT:-5318}:5318" + + vector: + container_name: vector-otel-logs-e2e + build: + context: ../../../ + dockerfile: ./scripts/e2e/Dockerfile + args: + FEATURES: e2e-tests-opentelemetry + RUST_VERSION: ${RUST_VERSION:-1.88} + init: true + volumes: + - type: bind + source: ../../../tests/data/e2e/opentelemetry/logs/${CONFIG_VECTOR_CONFIG} + target: /etc/vector/vector.yaml + read_only: true + - type: bind + source: ../../../tests/data/e2e/opentelemetry/logs/output + target: /output + environment: + - VECTOR_LOG=${VECTOR_LOG:-info} + - FEATURES=e2e-tests-opentelemetry + - OTEL_E2E_OUTPUT_PATH + command: [ "vector", "-c", "/etc/vector/vector.yaml" ] diff --git a/scripts/e2e/opentelemetry-logs/generator/Dockerfile b/scripts/e2e/opentelemetry-logs/generator/Dockerfile new file mode 100644 index 0000000000..349b4a8a66 --- /dev/null +++ b/scripts/e2e/opentelemetry-logs/generator/Dockerfile @@ -0,0 +1,4 @@ +FROM python:3.11-alpine +WORKDIR /generator +COPY requirements.txt logs_generator.py ./ +RUN pip install --no-cache-dir -r requirements.txt diff --git a/scripts/e2e/opentelemetry-logs/generator/logs_generator.py b/scripts/e2e/opentelemetry-logs/generator/logs_generator.py new file mode 100755 index 0000000000..a5266485dc --- /dev/null +++ b/scripts/e2e/opentelemetry-logs/generator/logs_generator.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 + +import argparse +import json +import random +import time +import uuid + +import requests + +SEVERITIES = ["DEBUG", "INFO", "WARN", "ERROR"] +PATHS = ["/", "/login", "/api/data", "/metrics", "/health"] + +def generate_log(endpoint: str, count: int) -> dict: + now_nanos = time.time_ns() + timestamp = time.strftime("%Y-%m-%dT%H:%M:%S%z") + severity = random.choice(SEVERITIES) + log_id = str(uuid.uuid4())[:8] + + log_data = { + "resourceLogs": [ + { + "resource": { + "attributes": [ + {"key": "service.name", "value": {"stringValue": "opentelemetry-logs"}} + ] + }, + "scopeLogs": [ + { + "scope": {"name": "log-generator"}, + "logRecords": [ + { + "timeUnixNano": now_nanos, + "severityText": severity, + "body": {"stringValue": f"[{log_id}] {severity} log {count} at {timestamp}"}, + "attributes": [ + {"key": "count", "value": {"intValue": count}} + ] + } + ] + } + ] + } + ] + } + + try: + response = requests.post( + endpoint, + data=json.dumps(log_data), + headers={"Content-Type": "application/json"}, + timeout=2 + ) + if response.status_code == 200: + return { + "success": True, + "message": f"Log {count} sent successfully", + "log_id": log_id, + "status_code": response.status_code + } + else: + return { + "success": False, + "message": f"HTTP {response.status_code}: {response.text.strip() or '[empty]'}", + "log_id": log_id, + "status code": response.status_code, + } + + except requests.exceptions.RequestException as e: + return { + "success": False, + "message": f"RequestException: {str(e)}", + "log_id": log_id, + } + + +def non_negative_float(value): + f = float(value) + if f < 0: + raise argparse.ArgumentTypeError(f"Interval must be non-negative, got {value}") + return f + + +def main(): + parser = argparse.ArgumentParser(description="Generate OTLP logs periodically.") + parser.add_argument( + "--interval", + type=non_negative_float, + help="Seconds between log sends (non-negative, optional)" + ) + parser.add_argument("-n", type=int, default=0, help="Total logs to send (0 or negative = infinite)") + parser.add_argument("--host", type=str, default="otel-collector-source", help="Host for the OTLP collector") + parser.add_argument("--port", type=int, default=4318, help="Port for OTLP HTTP logs") + parser.add_argument("--path", type=str, default="/v1/logs", help="OTLP HTTP logs path") + + args = parser.parse_args() + endpoint = f"http://{args.host}:{args.port}{args.path}" + + print(f"Starting log generator → {endpoint}") + + count = 0 + sent = 0 + failed = 0 + + while True: + result = generate_log(endpoint, count) + count += 1 + if result["success"]: + print(f"✅ Sent log {count} (ID: {result['log_id']})") + sent += 1 + else: + print(f"❌ Failed log {count} (ID: {result['log_id']}): {result['message']}") + failed += 1 + + if 0 < args.n <= count: + break + + if args.interval is not None: + time.sleep(args.interval) + + print(f"\n📊 Finished: Sent={sent}, Failed={failed}") + + +if __name__ == "__main__": + main() diff --git a/scripts/e2e/opentelemetry-logs/generator/requirements.txt b/scripts/e2e/opentelemetry-logs/generator/requirements.txt new file mode 100644 index 0000000000..f2293605cf --- /dev/null +++ b/scripts/e2e/opentelemetry-logs/generator/requirements.txt @@ -0,0 +1 @@ +requests diff --git a/scripts/e2e/opentelemetry-logs/test.yaml b/scripts/e2e/opentelemetry-logs/test.yaml new file mode 100644 index 0000000000..4063e7cc12 --- /dev/null +++ b/scripts/e2e/opentelemetry-logs/test.yaml @@ -0,0 +1,25 @@ +features: + - e2e-tests-opentelemetry + +test: "e2e" + +test_filter: "opentelemetry::logs::" + +runner: + env: + OTEL_COLLECTOR_SOURCE_GRPC_PORT: '4317' + OTEL_COLLECTOR_SOURCE_HTTP_PORT: '4318' + OTEL_COLLECTOR_SINK_HTTP_PORT: '5318' + +matrix: + # Determines which `otel/opentelemetry-collector-contrib` version to use + collector_version: [ 'latest' ] + vector_config: [ 'vector_default.yaml', 'vector_otlp.yaml' ] + +# Only trigger this integration test if relevant OTEL source/sink files change +paths: + - "src/sources/opentelemetry/**" + - "src/sinks/opentelemetry/**" + - "src/internal_events/opentelemetry_*" + - "tests/e2e/opentelemetry/logs/**" + - "scripts/e2e/opentelemetry-logs/**" diff --git a/scripts/ensure-wasm-pack-installed.sh b/scripts/ensure-wasm-pack-installed.sh deleted file mode 100755 index 5cedc1e445..0000000000 --- a/scripts/ensure-wasm-pack-installed.sh +++ /dev/null @@ -1,13 +0,0 @@ -#! /usr/bin/env bash - -if [[ "$(wasm-pack --version)" != "wasm-pack 0.13.0" ]] ; then - echo "wasm-pack version 0.13.0 is not installed" - # We are using the version from git due to the bug: https://github.com/vectordotdev/vector/pull/16060#issuecomment-1428429602 - echo "running cargo install --git https://github.com/rustwasm/wasm-pack.git --rev e3582b7 wasm-pack" - cargo install --git https://github.com/rustwasm/wasm-pack.git --rev e3582b7 wasm-pack -else - echo "wasm-pack version 0.13.0 is installed already" -fi - -brew install llvm -export PATH="/opt/homebrew/opt/llvm/bin:$PATH" diff --git a/scripts/ensure-wasm-target-installed.sh b/scripts/ensure-wasm-target-installed.sh index e80b06c4d3..da89901163 100644 --- a/scripts/ensure-wasm-target-installed.sh +++ b/scripts/ensure-wasm-target-installed.sh @@ -1,4 +1,4 @@ -#! /usr/bin/env bash +#!/usr/bin/env bash if [[ "$(rustup target list --installed | grep wasm32-unknown-unknown)" != "wasm32-unknown-unknown" ]] ; then echo "wasm32-unknown-unknown target is not installed" diff --git a/scripts/environment/Dockerfile b/scripts/environment/Dockerfile index f62773c8e0..08c2c1692e 100644 --- a/scripts/environment/Dockerfile +++ b/scripts/environment/Dockerfile @@ -1,4 +1,4 @@ -FROM docker.io/ubuntu:20.04 +FROM docker.io/ubuntu:24.04 ENV DEBIAN_FRONTEND=noninteractive \ TZ='America/New York' \ PATH=/root/.cargo/bin:/root/.local/bin/:$PATH \ @@ -12,7 +12,7 @@ RUN echo $TZ > /etc/timezone # Setup the env COPY scripts/environment/*.sh /git/vectordotdev/vector/scripts/environment/ -RUN cd git/vectordotdev/vector && ./scripts/environment/bootstrap-ubuntu-20.04.sh +RUN cd git/vectordotdev/vector && ./scripts/environment/bootstrap-ubuntu-24.04.sh # Setup the toolchain WORKDIR /git/vectordotdev/vector diff --git a/scripts/environment/binstall.sh b/scripts/environment/binstall.sh new file mode 100755 index 0000000000..def6e9eb7b --- /dev/null +++ b/scripts/environment/binstall.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash + +set -eux +set -o pipefail + +BINSTALL_VERSION="v1.14.1" +BINSTALL_SHA256SUM_X86_64_LINUX="e1d1231720e6ed497a4b0f8881b08f5df9ce1a938fb3ae6f2444e95eb601fe99" +BINSTALL_SHA256SUM_AARCH64_LINUX="17d69bcc07a0e38c912e7f596ed71b1f5f59dc8980da59890c5bc86c07e8506a" +BINSTALL_SHA256SUM_ARMV7_LINUX="e4ba720023e02b071aa805ae62412e94741c1bb0e0a2bb2b35896fec3d140128" +BINSTALL_SHA256SUM_AARCH64_DARWIN="07d46d31fb68ac10b906c5d39d611ded7787966f4ed15c598cb6175b45a2b069" +BINSTALL_SHA256SUM_X86_64_DARWIN="3de381bdcca08c418dc790d2a283711894a0577c6e55bba0d4e6cb8b0378b36" + +pushd "$(mktemp -d)" + +base_url="https://github.com/cargo-bins/cargo-binstall/releases/download/${BINSTALL_VERSION}/cargo-binstall" + +download() { + curl --retry 3 --proto '=https' --tlsv1.2 -fsSL "$@" +} + +os="$(uname -s)" +machine="$(uname -m)" + +if [ "$os" = "Darwin" ]; then + if [ "$machine" = "arm64" ]; then + url="${base_url}-aarch64-apple-darwin.zip" + download_sha256sum="${BINSTALL_SHA256SUM_AARCH64_DARWIN}" + elif [ "$machine" = "x86_64" ]; then + url="${base_url}-x86_64-apple-darwin.zip" + download_sha256sum="${BINSTALL_SHA256SUM_X86_64_DARWIN}" + else + echo "Unsupported OS ${os} machine ${machine}" + popd + exit 1 + fi + + download -o output.zip "$url" +elif [ "$os" = "Linux" ]; then + if [ "$machine" = "armv7l" ]; then + target="armv7-unknown-linux-musleabihf" + download_sha256sum="${BINSTALL_SHA256SUM_ARMV7_LINUX}" + elif [ "$machine" = "aarch64" ]; then + target="${machine}-unknown-linux-musl" + download_sha256sum="${BINSTALL_SHA256SUM_AARCH64_LINUX}" + elif [ "$machine" = "x86_64" ]; then + target="${machine}-unknown-linux-musl" + download_sha256sum="${BINSTALL_SHA256SUM_X86_64_LINUX}" + else + echo "Unsupported OS ${os} machine ${machine}" + popd + exit 1 + fi + + url="${base_url}-${target}.tgz" + + download -o output.tgz "$url" +# elif [ "${OS-}" = "Windows_NT" ]; then +# target="${machine}-pc-windows-msvc" +# url="${base_url}-${target}.zip" +# download -o output.zip "$url" +else + echo "Unsupported OS ${os}" + popd + exit 1 +fi + +echo "${download_sha256sum} $(echo output.*)" | sha256sum --check + +case "$(echo output.*)" in + *.zip) unzip output.* ;; + *.tgz) tar -xvzf output.* ;; + *) >&2 echo "output.* not found"; exit 1 ;; +esac + +./cargo-binstall --self-install || ./cargo-binstall -y --force cargo-binstall diff --git a/scripts/environment/bootstrap-macos.sh b/scripts/environment/bootstrap-macos.sh index 32a8fd528e..5cf014b315 100755 --- a/scripts/environment/bootstrap-macos.sh +++ b/scripts/environment/bootstrap-macos.sh @@ -1,20 +1,9 @@ -#! /usr/bin/env bash +#!/usr/bin/env bash set -e -o verbose brew update - -# `brew install` attempts to upgrade python as a dependency but fails -# https://github.com/actions/setup-python/issues/577 -brew list -1 | grep python | while read -r formula; do brew unlink "$formula"; brew link --overwrite "$formula"; done - brew install ruby@3 coreutils cue-lang/tap/cue protobuf -# rustup-init (renamed to rustup) is already installed in GHA, but seems to be lacking the rustup binary -# Reinstalling seems to fix it -# TODO(jszwedko): It's possible GHA just needs to update its images and this won't be needed in the -# future -brew reinstall rustup - gem install bundler echo "export PATH=\"/usr/local/opt/ruby/bin:\$PATH\"" >> "$HOME/.bash_profile" diff --git a/scripts/environment/bootstrap-ubuntu-20.04.sh b/scripts/environment/bootstrap-ubuntu-24.04.sh similarity index 88% rename from scripts/environment/bootstrap-ubuntu-20.04.sh rename to scripts/environment/bootstrap-ubuntu-24.04.sh index 2ccb4b56d0..664efef7b6 100755 --- a/scripts/environment/bootstrap-ubuntu-20.04.sh +++ b/scripts/environment/bootstrap-ubuntu-24.04.sh @@ -1,4 +1,7 @@ -#! /usr/bin/env bash +#!/usr/bin/env bash +# Refer to https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2404-Readme.md +# for all runner information such as OS version and installed software. + set -e -o verbose if [ -n "$RUSTFLAGS" ] @@ -22,7 +25,6 @@ apt-get install --yes \ # Deps apt-get install --yes --no-install-recommends \ - awscli \ build-essential \ ca-certificates \ cmake \ @@ -58,30 +60,17 @@ tar \ cp "${TEMP}/cue" /usr/bin/cue rm -rf "$TEMP" -# Grease -# Grease is used for the `make release-github` task. -TEMP=$(mktemp -d) -curl \ - -L https://github.com/vectordotdev/grease/releases/download/v1.0.1/grease-1.0.1-linux-amd64.tar.gz \ - -o "${TEMP}/grease-1.0.1-linux-amd64.tar.gz" -tar \ - -xvf "${TEMP}/grease-1.0.1-linux-amd64.tar.gz" \ - -C "${TEMP}" -cp "${TEMP}/grease/bin/grease" /usr/bin/grease -rm -rf "$TEMP" - # Locales locale-gen en_US.UTF-8 dpkg-reconfigure locales if ! command -v rustup ; then - # Rust/Cargo should already be installed on both GH Actions-provided Ubuntu 20.04 images _and_ - # by our own Ubuntu 20.04 images + # https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2404-Readme.md#rust-tools curl https://sh.rustup.rs -sSf | sh -s -- -y --profile minimal fi -# Rust/Cargo should already be installed on both GH Actions-provided Ubuntu 20.04 images _and_ -# by our own Ubuntu 20.04 images, so this is really just make sure the path is configured. +# Rust/Cargo should already be installed on both GH Actions-provided Ubuntu 24.04 images _and_ +# by our own Ubuntu 24.04 images, so this is really just make sure the path is configured. if [ -n "${CI-}" ] ; then echo "${HOME}/.cargo/bin" >> "${GITHUB_PATH}" # we often run into OOM issues in CI due to the low memory vs. CPU ratio on c5 instances diff --git a/scripts/environment/bootstrap-windows-2022.ps1 b/scripts/environment/bootstrap-windows-2025.ps1 similarity index 99% rename from scripts/environment/bootstrap-windows-2022.ps1 rename to scripts/environment/bootstrap-windows-2025.ps1 index 27b35d9ff5..7af1123b2d 100644 --- a/scripts/environment/bootstrap-windows-2022.ps1 +++ b/scripts/environment/bootstrap-windows-2025.ps1 @@ -10,7 +10,7 @@ echo "CARGO_BUILD_JOBS=$N_JOBS" | Out-File -FilePath $env:GITHUB_ENV -Encoding u if ($env:RELEASE_BUILDER -ne "true") { # Ensure we have cargo-next test installed. - rustup run stable cargo install cargo-nextest --version 0.9.72 --locked + rustup run stable cargo install cargo-nextest --version 0.9.95 --locked } # Enable retries to avoid transient network issues. diff --git a/scripts/environment/entrypoint.sh b/scripts/environment/entrypoint.sh index ae6b55eb1c..b97484e991 100755 --- a/scripts/environment/entrypoint.sh +++ b/scripts/environment/entrypoint.sh @@ -1,7 +1,16 @@ -#! /usr/bin/env bash +#!/usr/bin/env bash # set HOSTNAME to container id for `cross` -HOSTNAME="$(head -1 /proc/self/cgroup|cut -d/ -f3)" -export HOSTNAME +if [ -f /.docker-container-id ]; then + HOSTNAME="$(cat /.docker-container-id)" + export HOSTNAME +fi + +if [ -z "$HOSTNAME" ]; then + echo "Failed to properly set HOSTNAME, cross may not work" + # Fallback if everything else fails + HOSTNAME="vector-environment" + export HOSTNAME +fi exec "$@" diff --git a/scripts/environment/install-protoc.sh b/scripts/environment/install-protoc.sh index 082c3112e3..dc0eaa0903 100755 --- a/scripts/environment/install-protoc.sh +++ b/scripts/environment/install-protoc.sh @@ -1,4 +1,4 @@ -#! /usr/bin/env bash +#!/usr/bin/env bash set -o errexit -o verbose # A parameter can be optionally passed to this script to specify an alternative diff --git a/scripts/environment/prepare.sh b/scripts/environment/prepare.sh index 7ad153d4a5..1721acd12c 100755 --- a/scripts/environment/prepare.sh +++ b/scripts/environment/prepare.sh @@ -1,42 +1,171 @@ -#! /usr/bin/env bash -set -e -o verbose +#!/usr/bin/env bash +set -euo pipefail + +ALL_MODULES=( + rustup + cargo-deb + cross + cargo-nextest + cargo-deny + cargo-msrv + dd-rust-license-tool + wasm-pack + markdownlint + datadog-ci + release-flags +) + +# By default, install everything +MODULES=( "${ALL_MODULES[@]}" ) + +# Helper to join an array by comma +join_by() { local IFS="$1"; shift; echo "$*"; } + +# If the INSTALL_MODULES env var is set, override MODULES +if [[ -n "${INSTALL_MODULES:-}" ]]; then + IFS=',' read -r -a MODULES <<< "$INSTALL_MODULES" +fi + +# Parse CLI args for --modules or -m +for arg in "$@"; do + case $arg in + --modules=*|-m=*) + val="${arg#*=}" + IFS=',' read -r -a MODULES <<< "$val" + shift + ;; + --help|-h) + cat </dev/null || ./scripts/environment/binstall.sh; then + install=(binstall -y) + else + echo "Failed to install cargo binstall, defaulting to cargo install" + fi + fi +fi +set -e -o verbose +if contains_module cargo-deb; then + if [[ "$(cargo-deb --version 2>/dev/null)" != "2.9.3" ]]; then + rustup run stable cargo "${install[@]}" cargo-deb --version 2.9.3 --force --locked + fi +fi + +if contains_module cross; then + if ! cross --version 2>/dev/null | grep -q '^cross 0.2.5'; then + rustup run stable cargo "${install[@]}" cross --version 0.2.5 --force --locked + fi fi -if [[ "$(cargo-nextest --version)" != "cargo-nextest 0.9.72" ]] ; then - rustup run stable cargo install cargo-nextest --version 0.9.72 --force --locked + +if contains_module cargo-nextest; then + if ! cargo-nextest --version 2>/dev/null | grep -q '^cargo-nextest 0.9.95'; then + rustup run stable cargo "${install[@]}" cargo-nextest --version 0.9.95 --force --locked + fi fi -if [[ "$(cargo-deny --version)" != "cargo-deny 0.16.1" ]] ; then - rustup run stable cargo install cargo-deny --version 0.16.1 --force --locked + +if contains_module cargo-deny; then + if ! cargo-deny --version 2>/dev/null | grep -q '^cargo-deny 0.16.2'; then + rustup run stable cargo "${install[@]}" cargo-deny --version 0.16.2 --force --locked + fi fi -if ! dd-rust-license-tool --help >& /dev/null ; then - rustup run stable cargo install dd-rust-license-tool --version 1.0.2 --force --locked + +if contains_module cargo-msrv; then + if ! cargo-msrv --version 2>/dev/null | grep -q '^cargo-msrv 0.18.4'; then + rustup run stable cargo "${install[@]}" cargo-msrv --version 0.18.4 --force --locked + fi fi -if [[ "$(wasm-pack --version)" != "wasm-pack 0.13.0" ]] ; then - echo "wasm-pack version 0.13.0 is not installed" - # We are using the version from git due to the bug: https://github.com/vectordotdev/vector/pull/16060#issuecomment-1428429602 - echo "running cargo install --git https://github.com/rustwasm/wasm-pack.git --rev e3582b7 wasm-pack" - cargo install --force --git https://github.com/rustwasm/wasm-pack.git --rev e3582b7 wasm-pack -else - echo "wasm-pack version 0.13.0 is installed already" +if contains_module dd-rust-license-tool; then + if ! dd-rust-license-tool --help &>/dev/null; then + rustup run stable cargo install dd-rust-license-tool --version 1.0.2 --force --locked + fi fi -# Currently fixing this to version 0.30 since version 0.31 has introduced -# a change that means it only works with versions of node > 10. -# https://github.com/igorshubovych/markdownlint-cli/issues/258 -# ubuntu 20.04 gives us version 10.19. We can revert once we update the -# ci image to install a newer version of node. -sudo npm -g install markdownlint-cli@0.30 -sudo npm -g install @datadog/datadog-ci +if contains_module wasm-pack; then + if ! wasm-pack --version | grep -q '^wasm-pack 0.13.1'; then + rustup run stable cargo "${install[@]}" --locked --version 0.13.1 wasm-pack + fi +fi -# Make sure our release build settings are present. -. scripts/environment/release-flags.sh +if contains_module markdownlint; then + if [[ "$(markdownlint --version 2>/dev/null)" != "0.45.0" ]]; then + sudo npm install -g markdownlint-cli@0.45.0 + fi +fi + +if contains_module datadog-ci; then + if [[ "$(datadog-ci version 2>/dev/null)" != "v3.16.0" ]]; then + sudo npm install -g @datadog/datadog-ci@3.16.0 + fi +fi diff --git a/scripts/environment/release-flags.sh b/scripts/environment/release-flags.sh index 5ccbc70f60..4115517c43 100755 --- a/scripts/environment/release-flags.sh +++ b/scripts/environment/release-flags.sh @@ -1,4 +1,4 @@ -#! /usr/bin/env bash +#!/usr/bin/env bash set -e -o verbose # We want to ensure we're building using "full" release capabilities when possible, which diff --git a/scripts/environment/setup-helm.sh b/scripts/environment/setup-helm.sh index 66c919f895..a16b731957 100755 --- a/scripts/environment/setup-helm.sh +++ b/scripts/environment/setup-helm.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -euo pipefail KUBERNETES_VERSION="v1.18.6" diff --git a/scripts/generate-component-docs.rb b/scripts/generate-component-docs.rb index 9972b89752..5506212d1b 100755 --- a/scripts/generate-component-docs.rb +++ b/scripts/generate-component-docs.rb @@ -515,88 +515,60 @@ def expand_schema_references(root_schema, unexpanded_schema) original_title = unexpanded_schema['title'] original_description = unexpanded_schema['description'] - loop do - expanded = false - - # If the schema has a top level reference, we expand it. - schema_ref = schema['$ref'] - if !schema_ref.nil? - expanded_schema_ref = get_cached_expanded_schema(schema_ref) - if expanded_schema_ref.nil? - @logger.debug "Expanding top-level schema ref of '#{schema_ref}'..." - - unexpanded_schema_ref = get_schema_by_name(root_schema, schema_ref) - expanded_schema_ref = expand_schema_references(root_schema, unexpanded_schema_ref) - - @expanded_schema_cache[schema_ref] = expanded_schema_ref - end - - schema.delete('$ref') - schema = nested_merge(expanded_schema_ref, schema) - - expanded = true - end - - # If the schema is an array type and has a reference for its items, we expand that. - items_ref = schema.dig('items', '$ref') - if !items_ref.nil? - expanded_items_schema_ref = expand_schema_references(root_schema, schema['items']) - - schema['items'].delete('$ref') - schema['items'] = nested_merge(expanded_items_schema_ref, schema['items']) + # If the schema has a top level reference, we expand it. + schema_ref = schema['$ref'] + if !schema_ref.nil? + expanded_schema_ref = get_cached_expanded_schema(schema_ref) + if expanded_schema_ref.nil? + @logger.debug "Expanding top-level schema ref of '#{schema_ref}'..." - expanded = true - end + unexpanded_schema_ref = get_schema_by_name(root_schema, schema_ref) + expanded_schema_ref = expand_schema_references(root_schema, unexpanded_schema_ref) - # If the schema has any object properties, we expand those. - if !schema['properties'].nil? - schema['properties'] = schema['properties'].transform_values { |property_schema| - new_property_schema = expand_schema_references(root_schema, property_schema) - if new_property_schema != property_schema - expanded = true - end - - new_property_schema - } + @expanded_schema_cache[schema_ref] = expanded_schema_ref end - # If the schema has any `allOf`/`oneOf` subschemas, we expand those, too. - if !schema['allOf'].nil? - schema['allOf'] = schema['allOf'].map { |subschema| - new_subschema = expand_schema_references(root_schema, subschema) - if new_subschema != subschema - expanded = true - end + schema.delete('$ref') + schema = nested_merge(expanded_schema_ref, schema) + end - new_subschema - } - end + # If the schema is an array type and has a reference for its items, we expand that. + items_ref = schema.dig('items', '$ref') + if !items_ref.nil? + expanded_items_schema_ref = expand_schema_references(root_schema, schema['items']) - if !schema['oneOf'].nil? - schema['oneOf'] = schema['oneOf'].map { |subschema| - new_subschema = expand_schema_references(root_schema, subschema) - if new_subschema != subschema - expanded = true - end + schema['items'].delete('$ref') + schema['items'] = nested_merge(expanded_items_schema_ref, schema['items']) + end - new_subschema - } - end + # If the schema has any object properties, we expand those. + if !schema['properties'].nil? + schema['properties'] = schema['properties'].transform_values { |property_schema| + new_property_schema = expand_schema_references(root_schema, property_schema) + new_property_schema + } + end - if !schema['anyOf'].nil? - schema['anyOf'] = schema['anyOf'].map { |subschema| - new_subschema = expand_schema_references(root_schema, subschema) - if new_subschema != subschema - expanded = true - end + # If the schema has any `allOf`/`oneOf` subschemas, we expand those, too. + if !schema['allOf'].nil? + schema['allOf'] = schema['allOf'].map { |subschema| + new_subschema = expand_schema_references(root_schema, subschema) + new_subschema + } + end - new_subschema - } - end + if !schema['oneOf'].nil? + schema['oneOf'] = schema['oneOf'].map { |subschema| + new_subschema = expand_schema_references(root_schema, subschema) + new_subschema + } + end - if !expanded - break - end + if !schema['anyOf'].nil? + schema['anyOf'] = schema['anyOf'].map { |subschema| + new_subschema = expand_schema_references(root_schema, subschema) + new_subschema + } end # If the original schema had either a title or description, we forcefully reset both of them back @@ -798,6 +770,17 @@ def resolve_schema(root_schema, schema) end end + # required for global option configuration + is_common_field = get_schema_metadata(schema, 'docs::common') + if !is_common_field.nil? + resolved['common'] = is_common_field + end + + is_required_field = get_schema_metadata(schema, 'docs::required') + if !is_required_field.nil? + resolved['required'] = is_required_field + end + # Reconcile the resolve schema, which essentially gives us a chance to, once the schema is # entirely resolved, check it for logical inconsistencies, fix up anything that we reasonably can, # and so on. @@ -945,6 +928,13 @@ def resolve_bare_schema(root_schema, schema) fix_grouped_enums_if_numeric!(grouped) grouped.transform_values! { |values| { 'enum' => values } } grouped + when nil + # Unconstrained/empty schema (e.g., Value without constraints). + # Represent it as accepting any JSON type so downstream code can render it + # and attach defaults/examples based on actual values. + @logger.debug 'Resolving unconstrained schema (any type).' + + { '*' => {} } else @logger.error "Failed to resolve the schema. Schema: #{schema}" exit 1 @@ -1659,7 +1649,7 @@ def get_rendered_description_from_schema(schema) description.strip end -def render_and_import_schema(root_schema, schema_name, friendly_name, config_map_path, cue_relative_path) +def unwrap_resolved_schema(root_schema, schema_name, friendly_name) @logger.info "[*] Resolving schema definition for #{friendly_name}..." # Try and resolve the schema, unwrapping it as an object schema which is a requirement/expectation @@ -1673,7 +1663,10 @@ def render_and_import_schema(root_schema, schema_name, friendly_name, config_map exit 1 end - unwrapped_resolved_schema = sort_hash_nested(unwrapped_resolved_schema) + return sort_hash_nested(unwrapped_resolved_schema) +end + +def render_and_import_schema(unwrapped_resolved_schema, friendly_name, config_map_path, cue_relative_path) # Set up the appropriate structure for the value based on the configuration map path. It defines # the nested levels of the map where our resolved schema should go, as well as a means to generate @@ -1691,8 +1684,7 @@ def render_and_import_schema(root_schema, schema_name, friendly_name, config_map config_map_path.prepend('config-schema-base') tmp_file_prefix = config_map_path.join('-') - final = { 'base' => { 'components' => data } } - final_json = to_pretty_json(final) + final_json = to_pretty_json(data) # Write the resolved schema as JSON, which we'll then use to import into a Cue file. json_output_file = write_to_temp_file(["config-schema-#{tmp_file_prefix}-", '.json'], final_json) @@ -1700,7 +1692,7 @@ def render_and_import_schema(root_schema, schema_name, friendly_name, config_map # Try importing it as Cue. @logger.info "[*] Importing #{friendly_name} schema as Cue file..." - cue_output_file = "website/cue/reference/components/#{cue_relative_path}" + cue_output_file = "website/cue/reference/#{cue_relative_path}" unless system(@cue_binary_path, 'import', '-f', '-o', cue_output_file, '-p', 'metadata', json_output_file) @logger.error "[!] Failed to import #{friendly_name} schema as valid Cue." exit 1 @@ -1708,23 +1700,65 @@ def render_and_import_schema(root_schema, schema_name, friendly_name, config_map @logger.info "[✓] Imported #{friendly_name} schema to '#{cue_output_file}'." end -def render_and_import_base_component_schema(root_schema, schema_name, component_type) +def render_and_import_generated_component_schema(root_schema, schema_name, component_type) + friendly_name = "generated #{component_type} configuration" + unwrapped_resolved_schema = unwrap_resolved_schema(root_schema, schema_name, friendly_name) render_and_import_schema( - root_schema, - schema_name, - "base #{component_type} configuration", - ["#{component_type}s"], - "base/#{component_type}s.cue" + unwrapped_resolved_schema, + friendly_name, + ["generated", "components", "#{component_type}s"], + "components/generated/#{component_type}s.cue" ) end def render_and_import_component_schema(root_schema, schema_name, component_type, component_name) + friendly_name = "'#{component_name}' #{component_type} configuration" + unwrapped_resolved_schema = unwrap_resolved_schema(root_schema, schema_name, friendly_name) + render_and_import_schema( + unwrapped_resolved_schema, + friendly_name, + ["generated", "components", "#{component_type}s", component_name], + "components/#{component_type}s/generated/#{component_name}.cue" + ) +end + +def render_and_import_generated_api_schema(root_schema, apis) + api_schema = {} + apis.each do |component_name, schema_name| + friendly_name = "'#{component_name}' #{schema_name} configuration" + resolved_schema = unwrap_resolved_schema(root_schema, schema_name, friendly_name) + api_schema[component_name] = resolved_schema + end + render_and_import_schema( - root_schema, - schema_name, - "'#{component_name}' #{component_type} configuration", - ["#{component_type}s", component_name], - "#{component_type}s/base/#{component_name}.cue" + api_schema, + "configuration", + ["generated", "api"], + "generated/api.cue" + ) +end + +def render_and_import_generated_global_option_schema(root_schema, global_options) + global_option_schema = {} + + global_options.each do |component_name, schema_name| + friendly_name = "'#{component_name}' #{schema_name} configuration" + + if component_name == "global_option" + # Flattening global options + unwrap_resolved_schema(root_schema, schema_name, friendly_name) + .each { |name, schema| global_option_schema[name] = schema } + else + # Resolving and assigning other global options + global_option_schema[component_name] = resolve_schema_by_name(root_schema, schema_name) + end + end + + render_and_import_schema( + global_option_schema, + "configuration", + ["generated", "configuration"], + "generated/configuration.cue" ) end @@ -1747,7 +1781,7 @@ def render_and_import_component_schema(root_schema, schema_name, component_type, # First off, we generate the component type configuration bases. These are the high-level # configuration settings that are universal on a per-component type basis. # -# For example, the "base" configuration for a sink would be the inputs, buffer settings, healthcheck +# For example, the "generated" configuration for a sink would be the inputs, buffer settings, healthcheck # settings, and proxy settings... and then the configuration for a sink would be those, plus # whatever the sink itself defines. component_bases = root_schema['definitions'].filter_map do |key, definition| @@ -1757,7 +1791,7 @@ def render_and_import_component_schema(root_schema, schema_name, component_type, .reduce { |acc, item| nested_merge(acc, item) } component_bases.each do |component_type, schema_name| - render_and_import_base_component_schema(root_schema, schema_name, component_type) + render_and_import_generated_component_schema(root_schema, schema_name, component_type) end # Now we'll generate the base configuration for each component. @@ -1773,3 +1807,23 @@ def render_and_import_component_schema(root_schema, schema_name, component_type, render_and_import_component_schema(root_schema, schema_name, component_type, component_name) end end + +apis = root_schema['definitions'].filter_map do |key, definition| + component_type = get_schema_metadata(definition, 'docs::component_type') + component_name = get_schema_metadata(definition, 'docs::component_name') + { component_name => key } if component_type == "api" +end +.reduce { |acc, item| nested_merge(acc, item) } + +render_and_import_generated_api_schema(root_schema, apis) + + +# At last, we generate the global options configuration. +global_options = root_schema['definitions'].filter_map do |key, definition| + component_type = get_schema_metadata(definition, 'docs::component_type') + component_name = get_schema_metadata(definition, 'docs::component_name') + { component_name => key } if component_type == "global_option" +end +.reduce { |acc, item| nested_merge(acc, item) } + +render_and_import_generated_global_option_schema(root_schema, global_options) diff --git a/scripts/generate-release-cue.rb b/scripts/generate-release-cue.rb index c308f44f8b..9a03cda437 100755 --- a/scripts/generate-release-cue.rb +++ b/scripts/generate-release-cue.rb @@ -14,22 +14,56 @@ require "json" require "time" +require "optparse" +require 'pathname' require_relative "util/commit" require_relative "util/git_log_commit" require_relative "util/printer" require_relative "util/release" require_relative "util/version" +# Function to find the repository root by looking for .git directory +def find_repo_root + # Get the absolute path of the current script + script_path = File.expand_path(__FILE__) + dir = Pathname.new(script_path).dirname + + # Walk up the directory tree until we find .git or reach the filesystem root + loop do + return dir.to_s if File.exist?(File.join(dir, '.git')) + parent = dir.parent + raise "Could not find repository root (no .git directory found)" if parent == dir # Reached filesystem root + dir = parent + end +end + # # Constants # -ROOT = ".." +ROOT = find_repo_root RELEASE_REFERENCE_DIR = File.join(ROOT, "website", "cue", "reference", "releases") CHANGELOG_DIR = File.join(ROOT, "changelog.d") TYPES = ["chore", "docs", "feat", "fix", "enhancement", "perf"] TYPES_THAT_REQUIRE_SCOPES = ["feat", "enhancement", "fix"] +# Parse command-line options +options = {} +OptionParser.new do |opts| + opts.banner = "Usage: #{File.basename(__FILE__)} [options]" + + opts.on("--new-version VERSION", "Specify the new version (e.g., 1.2.3)") do |v| + options[:new_version] = v + end + opts.on("--[no-]interactive", "Enable/disable interactive prompts (default: true)") do |i| + options[:interactive] = i + end + opts.on_tail("-h", "--help", "Show this help message") do + puts opts + exit + end +end.parse! + # # Functions # @@ -43,66 +77,60 @@ # This file is created from outstanding commits since the last release. # It's meant to be a starting point. The resulting file should be reviewed # and edited by a human before being turned into a cue file. -def create_log_file!(current_commits, new_version) +def create_log_file!(current_commits, new_version, interactive) release_log_path = "#{RELEASE_REFERENCE_DIR}/#{new_version}.log" # Grab all existing commits existing_commits = get_existing_commits! - # Ensure this release does not include duplicate commits. Notice that we - # check the parsed PR numbers. This is necessary to ensure we do not include - # cherry-picked commits made available in other releases. - # - # For example, if we cherry pick a commit from `master` to the `0.8` branch - # it will have a different commit sha. Without checking something besides the - # sha, this commit would also show up in the next release. - new_commits = - current_commits.select do |current_commit| - !existing_commits.any? do |existing_commit| - existing_commit.eql?(current_commit) - end - end + # Filter out duplicate commits + new_commits = current_commits.select do |current_commit| + !existing_commits.any? { |existing_commit| existing_commit.eql?(current_commit) } + end new_commit_lines = new_commits.collect { |c| c.to_git_log_commit.to_raw }.join("\n") if new_commits.any? if File.exists?(release_log_path) - words = - <<~EOF - I found #{new_commits.length} new commits since you last ran this - command. So that I don't erase any other work in that file, please - manually add the following commit lines: + if interactive + words = <<~EOF + I found #{new_commits.length} new commits since you last ran this + command. So that I don't erase any other work in that file, please + manually add the following commit lines: - #{new_commit_lines.split("\n").collect { |line| " #{line}" }.join("\n")} + #{new_commit_lines.split("\n").collect { |line| " #{line}" }.join("\n")} - To: + To: - #{release_log_path} + #{release_log_path} - All done? Ready to proceed? + All done? Ready to proceed? EOF - if Util::Printer.get(words, ["y", "n"]) == "n" - Util::Printer.error!("Ok, re-run this command when you're ready.") + if Util::Printer.get(words, ["y", "n"]) == "n" + Util::Printer.error!("Ok, re-run this command when you're ready.") + end end else File.open(release_log_path, 'w+') do |file| file.write(new_commit_lines) end - words = - <<~EOF - I've created a release log file here: + puts interactive + if interactive + words = <<~EOF + I've created a release log file here: - #{release_log_path} + #{release_log_path} - Please review the commits and *adjust the commit messages as necessary*. + Please review the commits and *adjust the commit messages as necessary*. - All done? Ready to proceed? + All done? Ready to proceed? EOF - if Util::Printer.get(words, ["y", "n"]) == "n" - Util::Printer.error!("Ok, re-run this command when you're ready.") + if Util::Printer.get(words, ["y", "n"]) == "n" + Util::Printer.error!("Ok, re-run this command when you're ready.") + end end end end @@ -391,16 +419,32 @@ def migrate_highlights(new_version) # # Execute # - -Dir.chdir "scripts" +script_dir = File.expand_path(File.dirname(__FILE__)) +Dir.chdir script_dir Util::Printer.title("Creating release meta file...") last_tag = `git describe --tags $(git rev-list --tags --max-count=1)`.chomp last_version = Util::Version.new(last_tag.gsub(/^v/, '')) current_commits = get_commits_since(last_version) -new_version = get_new_version(last_version, current_commits) -log_file_path = create_log_file!(current_commits, new_version) + +new_version_string = options[:new_version] +if new_version_string + begin + new_version = Util::Version.new(new_version_string) + if last_version.bump_type(new_version).nil? + Util::Printer.error!("The specified version '#{new_version_string}' must be a single patch, minor, or major bump from #{last_version}") + exit 1 + end + rescue ArgumentError => e + Util::Printer.error!("Invalid version specified: #{e.message}") + exit 1 + end +else + new_version = get_new_version(last_version, current_commits) +end + +log_file_path = create_log_file!(current_commits, new_version, options[":interactive"]) create_release_file!(new_version) File.delete(log_file_path) diff --git a/scripts/integration/Dockerfile b/scripts/integration/Dockerfile index 07ffd19738..3e6b68dbfb 100644 --- a/scripts/integration/Dockerfile +++ b/scripts/integration/Dockerfile @@ -1,4 +1,4 @@ -ARG RUST_VERSION +ARG RUST_VERSION=1.85 FROM docker.io/rust:${RUST_VERSION}-slim-bookworm RUN apt-get update && apt-get install -y --no-install-recommends \ @@ -16,7 +16,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ git \ && rm -rf /var/lib/apt/lists/* -RUN rustup run "${RUST_VERSION}" cargo install cargo-nextest --version 0.9.72 --locked +RUN cargo install cargo-nextest --version 0.9.95 --locked COPY scripts/environment/install-protoc.sh / COPY tests/data/ca/certs /certs diff --git a/scripts/integration/amqp/compose.yaml b/scripts/integration/amqp/compose.yaml index 865c35266f..a60a27155d 100644 --- a/scripts/integration/amqp/compose.yaml +++ b/scripts/integration/amqp/compose.yaml @@ -12,5 +12,4 @@ services: - RABBITMQ_SSL_CACERTFILE=/code/tests/data/ca/intermediate_server/certs/ca-chain.cert.pem - RABBITMQ_SSL_FAIL_IF_NO_PEER_CERT=false volumes: - - ${PWD}:/code - + - ../../..:/code diff --git a/scripts/integration/aws/compose.yaml b/scripts/integration/aws/compose.yaml index c45cbe7f2e..55e926dcdc 100644 --- a/scripts/integration/aws/compose.yaml +++ b/scripts/integration/aws/compose.yaml @@ -2,11 +2,11 @@ version: '3' services: mock-ec2-metadata: - image: public.ecr.aws/aws-ec2/amazon-ec2-metadata-mock:v1.11.2 + image: public.ecr.aws/aws-ec2/amazon-ec2-metadata-mock:v1.13.0 mock-localstack: - image: docker.io/localstack/localstack:3 + image: docker.io/localstack/localstack:stable environment: - - SERVICES=kinesis,s3,cloudwatch,es,firehose,sqs,sns,logs + - SERVICES=kinesis,s3,cloudwatch,es,firehose,kms,sqs,sns,logs mock-ecs: image: docker.io/amazon/amazon-ecs-local-container-endpoints:latest volumes: diff --git a/scripts/integration/aws/test.yaml b/scripts/integration/aws/test.yaml index 71ff65cae1..ba1d901dfc 100644 --- a/scripts/integration/aws/test.yaml +++ b/scripts/integration/aws/test.yaml @@ -11,6 +11,7 @@ env: ECS_ADDRESS: http://mock-ecs ELASTICSEARCH_ADDRESS: http://mock-localstack:4566 KINESIS_ADDRESS: http://mock-localstack:4566 + KMS_ADDRESS: http://mock-localstack:4566 S3_ADDRESS: http://mock-localstack:4566 SQS_ADDRESS: http://mock-localstack:4566 SNS_ADDRESS: http://mock-localstack:4566 diff --git a/scripts/integration/greptimedb/test.yaml b/scripts/integration/greptimedb/test.yaml index 6a1301c4f8..eb17543641 100644 --- a/scripts/integration/greptimedb/test.yaml +++ b/scripts/integration/greptimedb/test.yaml @@ -12,7 +12,7 @@ runner: matrix: # Temporarily pegging to the latest known stable release # since using `latest` is failing consistently. - version: [v0.9.5] + version: [v0.13.2] # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch diff --git a/scripts/integration/mqtt/test.yaml b/scripts/integration/mqtt/test.yaml index 607da45eeb..ec0c637e09 100644 --- a/scripts/integration/mqtt/test.yaml +++ b/scripts/integration/mqtt/test.yaml @@ -10,3 +10,4 @@ paths: - "src/internal_events/mqtt.rs" - "src/sinks/mqtt/**" - "src/sinks/util/**" +- "src/sources/mqtt/**" diff --git a/scripts/integration/nats/compose.yaml b/scripts/integration/nats/compose.yaml index fb36e78d81..9be71dc529 100644 --- a/scripts/integration/nats/compose.yaml +++ b/scripts/integration/nats/compose.yaml @@ -43,3 +43,10 @@ services: - /usr/share/nats/config/nats-jwt.conf volumes: - ../../../tests/data/nats:/usr/share/nats/config + nats-jetstream-test: + image: docker.io/library/nats:${CONFIG_VERSION} + command: + - --config + - /usr/share/nats/config/nats-jetstream.conf + volumes: + - ../../../tests/data/nats:/usr/share/nats/config diff --git a/scripts/integration/nats/test.yaml b/scripts/integration/nats/test.yaml index 1615b5f244..171a1c4ae6 100644 --- a/scripts/integration/nats/test.yaml +++ b/scripts/integration/nats/test.yaml @@ -1,7 +1,7 @@ features: -- nats-integration-tests + - nats-integration-tests -test_filter: '::nats::' +test_filter: "::nats::" env: NATS_ADDRESS: nats://nats:4222 @@ -11,6 +11,7 @@ env: NATS_TLS_CLIENT_CERT_ADDRESS: nats://nats-tls-client-cert:4222 NATS_TOKEN_ADDRESS: nats://nats-token:4222 NATS_USERPASS_ADDRESS: nats://nats-userpass:4222 + NATS_JETSTREAM_ADDRESS: nats://nats-jetstream-test:4222 matrix: version: [latest] @@ -18,10 +19,10 @@ matrix: # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/internal_events/nats.rs" -- "src/sources/nats.rs" -- "src/sources/util/**" -- "src/sinks/nats.rs" -- "src/sinks/util/**" -- "src/nats.rs" -- "scripts/integration/nats/**" + - "src/internal_events/nats.rs" + - "src/sources/nats.rs" + - "src/sources/util/**" + - "src/sinks/nats/**" + - "src/sinks/util/**" + - "src/nats.rs" + - "scripts/integration/nats/**" diff --git a/scripts/integration/postgres/test.yaml b/scripts/integration/postgres/test.yaml index 67aa2ddc10..a89049f545 100644 --- a/scripts/integration/postgres/test.yaml +++ b/scripts/integration/postgres/test.yaml @@ -1,5 +1,6 @@ features: - postgresql_metrics-integration-tests +- postgres_sink-integration-tests test_filter: ::postgres @@ -18,6 +19,7 @@ matrix: # expressions are evaluated using https://github.com/micromatch/picomatch paths: - "src/internal_events/postgresql_metrics.rs" +- "src/sinks/postgres/**" - "src/sources/postgresql_metrics.rs" - "src/sources/util/**" - "scripts/integration/postgres/**" diff --git a/scripts/integration/pulsar/compose.yaml b/scripts/integration/pulsar/compose.yaml index b73d35909b..0e963cd2e0 100644 --- a/scripts/integration/pulsar/compose.yaml +++ b/scripts/integration/pulsar/compose.yaml @@ -3,6 +3,16 @@ version: '3' services: pulsar: image: docker.io/apachepulsar/pulsar:${CONFIG_VERSION} - command: bin/pulsar standalone + command: sh -c "bin/apply-config-from-env.py conf/standalone.conf && bin/pulsar standalone" ports: - - 6650:6650 + - 6650:6650 + - 6651:6651 + environment: + - PULSAR_PREFIX_brokerServicePortTls=6651 + - PULSAR_PREFIX_tlsKeyFilePath=/etc/pulsar/certs/pulsar.key.pem + - PULSAR_PREFIX_tlsCertificateFilePath=/etc/pulsar/certs/pulsar.cert.pem + - PULSAR_PREFIX_tlsTrustCertsFilePath=/etc/pulsar/certs/ca-chain.cert.pem + volumes: + - ../../../tests/data/ca/intermediate_server/private/pulsar.key.pem:/etc/pulsar/certs/pulsar.key.pem:ro + - ../../../tests/data//ca/intermediate_server/certs/pulsar.cert.pem:/etc/pulsar/certs/pulsar.cert.pem:ro + - ../../../tests/data/ca/intermediate_server/certs/ca-chain.cert.pem:/etc/pulsar/certs/ca-chain.cert.pem:ro diff --git a/scripts/integration/pulsar/test.yaml b/scripts/integration/pulsar/test.yaml index 824f0e0f29..b629a582d0 100644 --- a/scripts/integration/pulsar/test.yaml +++ b/scripts/integration/pulsar/test.yaml @@ -4,7 +4,7 @@ features: test_filter: '::pulsar::integration_tests::' env: - PULSAR_ADDRESS: pulsar://pulsar:6650 + PULSAR_HOST: pulsar matrix: version: [latest] diff --git a/scripts/integration/redis/compose.yaml b/scripts/integration/redis/compose.yaml index a5dd865e43..1399a12b2a 100644 --- a/scripts/integration/redis/compose.yaml +++ b/scripts/integration/redis/compose.yaml @@ -1,5 +1,26 @@ version: '3' services: - redis: + redis-primary: image: docker.io/redis:${CONFIG_VERSION} + container_name: redis-primary + hostname: redis-primary + ports: + - "6379:6379" + + redis-sentinel: + image: docker.io/redis:${CONFIG_VERSION} + container_name: redis-sentinel + hostname: redis-sentinel + depends_on: + - redis-primary + ports: + - "26379:26379" + command: > + sh -c 'echo "bind 0.0.0.0" > /etc/sentinel.conf && + echo "sentinel monitor vector redis-primary 6379 1" >> /etc/sentinel.conf && + echo "sentinel resolve-hostnames yes" >> /etc/sentinel.conf && + echo "sentinel down-after-milliseconds vector 5000" >> /etc/sentinel.conf && + echo "sentinel failover-timeout vector 5000" >> /etc/sentinel.conf && + echo "sentinel parallel-syncs vector 1" >> /etc/sentinel.conf && + redis-sentinel /etc/sentinel.conf' diff --git a/scripts/integration/redis/test.yaml b/scripts/integration/redis/test.yaml index d2d0577e84..6b2d9bfa37 100644 --- a/scripts/integration/redis/test.yaml +++ b/scripts/integration/redis/test.yaml @@ -4,7 +4,8 @@ features: test_filter: "::redis::" env: - REDIS_URL: redis://redis:6379/0 + REDIS_URL: redis://redis-primary:6379/0 + SENTINEL_URL: redis://redis-sentinel:26379/ matrix: version: [6-alpine] diff --git a/scripts/package-deb.sh b/scripts/package-deb.sh index cec0ed5038..29f2c793d1 100755 --- a/scripts/package-deb.sh +++ b/scripts/package-deb.sh @@ -71,6 +71,11 @@ cat LICENSE NOTICE >"$PROJECT_ROOT/target/debian-license.txt" # --no-build # because this step should follow a build +# TODO: Remove this after the Vector docker image contains a newer cargo-deb version. +# Temporary override of cargo-deb to support Rust 2024 edition. +if [[ "$(cargo-deb --version 2>/dev/null)" != "2.9.3" ]]; then + cargo install cargo-deb --version 2.9.3 --force --locked +fi cargo deb --target "$TARGET" --deb-version "${PACKAGE_VERSION}-1" --variant "$TARGET" --no-build --no-strip # Rename the resulting .deb file to remove TARGET from name. diff --git a/scripts/run-integration-test.sh b/scripts/run-integration-test.sh new file mode 100755 index 0000000000..36c254834c --- /dev/null +++ b/scripts/run-integration-test.sh @@ -0,0 +1,147 @@ +#!/usr/bin/env bash + +# Used in CI to run and stop an integration test and upload the results of it. +# This is useful to allow retrying the integration test at a higher level than +# nextest and reduce code duplication in the workflow file. + +set -u + +if [[ "${ACTIONS_RUNNER_DEBUG:-}" == "true" ]]; then + set -x +fi + +SCRIPT_DIR=$(realpath "$(dirname "${BASH_SOURCE[0]}")") + +print_compose_logs_on_failure() { + local LAST_RETURN_CODE=$1 + if [[ "$LAST_RETURN_CODE" -ne 0 || "${ACTIONS_RUNNER_DEBUG:-}" == "true" ]]; then + (docker compose --project-name "${TEST_NAME}" logs) || echo "Failed to collect logs" + fi +} + +usage() { + cat >&2 <<'USAGE' +Usage: + scripts/run-integration-test.sh [OPTIONS] (int|e2e) TEST_NAME + +Required positional arguments: + TEST_TYPE One of: int, e2e + TEST_NAME Name of the test/app (used as docker compose project name) + +Options: + -h Show this help and exit + -r Number of retries for the "test" phase (default: 2) + -v Increase verbosity; repeat for more (e.g. -vv or -vvv) + -e One or more environments to run (repeatable or comma-separated). + If provided, these are used as TEST_ENVIRONMENTS instead of auto-discovery. + +Notes: + - All existing two-argument invocations remain compatible: + scripts/run-integration-test.sh int opentelemetry-logs + - Additional options can be added later without breaking callers. +USAGE +} + +# Parse options +# Note: options must come before positional args (standard getopts behavior) +TEST_ENV="" +while getopts ":hr:v:e:" opt; do + case "$opt" in + h) + usage + exit 0 + ;; + r) + RETRIES="$OPTARG" + if ! [[ "$RETRIES" =~ ^[0-9]+$ ]] || [[ "$RETRIES" -lt 0 ]]; then + echo "error: -r requires a non-negative integer (got: $RETRIES)" >&2 + exit 2 + fi + ;; + v) + VERBOSITY+="v" + ;; + e) + TEST_ENV="$OPTARG" + ;; + \?) + echo "error: unknown option: -$OPTARG" >&2 + usage + exit 2 + ;; + :) + echo "error: option -$OPTARG requires an argument" >&2 + usage + exit 2 + ;; + esac +done +shift $((OPTIND - 1)) + +RETRIES=${RETRIES:-2} +VERBOSITY=${VERBOSITY:-'-v'} + +# Validate required positional args +if [[ $# -ne 2 ]]; then + echo "error: missing required positional arguments" >&2 + usage + exit 1 +fi + +TEST_TYPE=$1 # either "int" or "e2e" +TEST_NAME=$2 + +case "$TEST_TYPE" in + int|e2e) ;; + *) + echo "error: TEST_TYPE must be 'int' or 'e2e' (got: $TEST_TYPE)" >&2 + usage + exit 1 + ;; +esac + +# Determine environments to run +if [[ ${#TEST_ENV} -gt 0 ]]; then + # Use the environments supplied via -e + TEST_ENVIRONMENTS="${TEST_ENV}" +else + # Collect all available environments via auto-discovery + mapfile -t TEST_ENVIRONMENTS < <(cargo vdev "${VERBOSITY}" "${TEST_TYPE}" show -e "${TEST_NAME}") +fi + +for TEST_ENV in "${TEST_ENVIRONMENTS[@]}"; do + # Pre-run cleanup + if [[ "$TEST_NAME" == "opentelemetry-logs" ]]; then + # TODO use Docker compose volumes + find "${SCRIPT_DIR}/../tests/data/e2e/opentelemetry/logs/output" -type f -name '*.log' -delete + # Like 777, but users can only delete their own files. This allows the docker instances to write output files. + chmod 1777 "${SCRIPT_DIR}/../tests/data/e2e/opentelemetry/logs/output" + fi + + cargo vdev "${VERBOSITY}" "${TEST_TYPE}" start -a "${TEST_NAME}" "${TEST_ENV}" || true + START_RET=$? + print_compose_logs_on_failure "$START_RET" + + if [[ "$START_RET" -eq 0 ]]; then + cargo vdev "${VERBOSITY}" "${TEST_TYPE}" test --retries "$RETRIES" -a "${TEST_NAME}" "${TEST_ENV}" + RET=$? + print_compose_logs_on_failure "$RET" + else + echo "Skipping test phase because 'vdev start' failed" + RET=$START_RET + fi + + cargo vdev "${VERBOSITY}" "${TEST_TYPE}" stop -a "${TEST_NAME}" || true + + # Post-run cleanup + if [[ "$TEST_NAME" == "opentelemetry-logs" ]]; then + chmod 0755 "${SCRIPT_DIR}/../tests/data/e2e/opentelemetry/logs/output" # revert to default permissions + fi + + # Only upload test results if CI is defined + if [[ -n "${CI:-}" ]]; then + ./scripts/upload-test-results.sh + fi +done + +exit "$RET" diff --git a/scripts/test-e2e-kubernetes.sh b/scripts/test-e2e-kubernetes.sh index 38d7fc13b0..ec8332c85e 100755 --- a/scripts/test-e2e-kubernetes.sh +++ b/scripts/test-e2e-kubernetes.sh @@ -107,7 +107,7 @@ if [[ -z "${CONTAINER_IMAGE:-}" ]]; then # Build docker image with Vector - the same way it's done for releases. Don't # do the push - we'll handle it later. - REPO="$CONTAINER_IMAGE_REPO" \ + REPOS="$CONTAINER_IMAGE_REPO" \ CHANNEL="test" \ BASE="$BASE_TAG" \ TAG="$VERSION_TAG" \ diff --git a/scripts/util/release.rb b/scripts/util/release.rb index 5533552af8..112ee39e19 100644 --- a/scripts/util/release.rb +++ b/scripts/util/release.rb @@ -9,7 +9,8 @@ def all!(dir) release_meta_paths. collect do |release_meta_path| - release_json = `cue export #{release_meta_path}/../../urls.cue #{release_meta_path}` + urls_cue_path = File.join(File.dirname(release_meta_path), "..", "urls.cue") + release_json = `cue export #{urls_cue_path} #{release_meta_path}` release_hash = JSON.parse(release_json) name = release_hash.fetch("releases").keys.first hash = release_hash.fetch("releases").values.first diff --git a/src/api/handler.rs b/src/api/handler.rs index 3e4ec2c5b5..72ecfce87c 100644 --- a/src/api/handler.rs +++ b/src/api/handler.rs @@ -1,10 +1,10 @@ use std::sync::{ - atomic::{self, AtomicBool}, Arc, + atomic::{self, AtomicBool}, }; use serde_json::json; -use warp::{reply::json, Rejection, Reply}; +use warp::{Rejection, Reply, reply::json}; // Health handler, responds with '{ ok: true }' when running and '{ ok: false}' // when shutting down diff --git a/src/api/schema/components/mod.rs b/src/api/schema/components/mod.rs index 7398a7c215..69d509e449 100644 --- a/src/api/schema/components/mod.rs +++ b/src/api/schema/components/mod.rs @@ -10,7 +10,7 @@ use std::{ }; use async_graphql::{Enum, InputObject, Interface, Object, Subscription}; -use tokio_stream::{wrappers::BroadcastStream, Stream, StreamExt}; +use tokio_stream::{Stream, StreamExt, wrappers::BroadcastStream}; use vector_lib::internal_event::DEFAULT_OUTPUT; use crate::{ @@ -19,7 +19,7 @@ use crate::{ filter::{self, filter_items}, relay, sort, }, - config::{get_transform_output_ids, ComponentKey, Config}, + config::{ComponentKey, Config, get_transform_output_ids}, filter_check, }; @@ -235,7 +235,7 @@ pub struct ComponentsSubscription; #[Subscription] impl ComponentsSubscription { /// Subscribes to all newly added components - async fn component_added(&self) -> impl Stream { + async fn component_added(&self) -> impl Stream + use<> { BroadcastStream::new(COMPONENT_CHANGED.subscribe()).filter_map(|c| match c { Ok(ComponentChanged::Added(c)) => Some(c), _ => None, @@ -243,7 +243,7 @@ impl ComponentsSubscription { } /// Subscribes to all removed components - async fn component_removed(&self) -> impl Stream { + async fn component_removed(&self) -> impl Stream + use<> { BroadcastStream::new(COMPONENT_CHANGED.subscribe()).filter_map(|c| match c { Ok(ComponentChanged::Removed(c)) => Some(c), _ => None, @@ -256,7 +256,14 @@ pub fn update_config(config: &Config) { let mut new_components = HashMap::new(); // Sources - for (component_key, source) in config.sources() { + let table_sources = config + .enrichment_tables() + .filter_map(|(k, e)| e.as_source(k)) + .collect::>(); + for (component_key, source) in config + .sources() + .chain(table_sources.iter().map(|(k, s)| (k, s))) + { new_components.insert( component_key.clone(), Component::Source(source::Source(source::Data { @@ -301,7 +308,14 @@ pub fn update_config(config: &Config) { } // Sinks - for (component_key, sink) in config.sinks() { + let table_sinks = config + .enrichment_tables() + .filter_map(|(k, e)| e.as_sink(k)) + .collect::>(); + for (component_key, sink) in config + .sinks() + .chain(table_sinks.iter().map(|(k, s)| (k, s))) + { new_components.insert( component_key.clone(), Component::Sink(sink::Sink(sink::Data { diff --git a/src/api/schema/components/sink.rs b/src/api/schema/components/sink.rs index 919a0ec63f..99bcca38cc 100644 --- a/src/api/schema/components/sink.rs +++ b/src/api/schema/components/sink.rs @@ -2,7 +2,7 @@ use std::cmp; use async_graphql::{Enum, InputObject, Object}; -use super::{source, state, transform, Component}; +use super::{Component, source, state, transform}; use crate::{ api::schema::{ filter, @@ -28,7 +28,7 @@ impl Sink { &self.0.component_key } - pub fn get_component_type(&self) -> &str { + pub const fn get_component_type(&self) -> &str { self.0.component_type.as_str() } } diff --git a/src/api/schema/components/source.rs b/src/api/schema/components/source.rs index 65dcb691d8..c2289c4b6a 100644 --- a/src/api/schema/components/source.rs +++ b/src/api/schema/components/source.rs @@ -2,11 +2,11 @@ use std::cmp; use async_graphql::{Enum, InputObject, Object}; -use super::{sink, state, transform, Component}; +use super::{Component, sink, state, transform}; use crate::{ api::schema::{ filter, - metrics::{self, outputs_by_component_key, IntoSourceMetrics, Output}, + metrics::{self, IntoSourceMetrics, Output, outputs_by_component_key}, sort, }, config::{ComponentKey, DataType, OutputId}, @@ -43,7 +43,7 @@ impl Source { pub fn get_component_key(&self) -> &ComponentKey { &self.0.component_key } - pub fn get_component_type(&self) -> &str { + pub const fn get_component_type(&self) -> &str { self.0.component_type.as_str() } pub fn get_output_types(&self) -> Vec { diff --git a/src/api/schema/components/state.rs b/src/api/schema/components/state.rs index 615f3b2e26..1bb04894d8 100644 --- a/src/api/schema/components/state.rs +++ b/src/api/schema/components/state.rs @@ -3,7 +3,7 @@ use std::{ sync::{Arc, LazyLock, RwLock}, }; -use super::{sink, source, transform, Component}; +use super::{Component, sink, source, transform}; use crate::config::{ComponentKey, OutputId}; pub const INVARIANT: &str = "Couldn't acquire lock on Vector components. Please report this."; diff --git a/src/api/schema/components/transform.rs b/src/api/schema/components/transform.rs index 6d53a9bbd3..7fe95238b7 100644 --- a/src/api/schema/components/transform.rs +++ b/src/api/schema/components/transform.rs @@ -2,11 +2,11 @@ use std::cmp; use async_graphql::{Enum, InputObject, Object}; -use super::{sink, source, state, Component}; +use super::{Component, sink, source, state}; use crate::{ api::schema::{ filter, - metrics::{self, outputs_by_component_key, IntoTransformMetrics, Output}, + metrics::{self, IntoTransformMetrics, Output, outputs_by_component_key}, sort, }, config::{ComponentKey, Inputs, OutputId}, @@ -28,7 +28,7 @@ impl Transform { pub const fn get_component_key(&self) -> &ComponentKey { &self.0.component_key } - pub fn get_component_type(&self) -> &str { + pub const fn get_component_type(&self) -> &str { self.0.component_type.as_str() } pub fn get_outputs(&self) -> &[String] { diff --git a/src/api/schema/events/metric.rs b/src/api/schema/events/metric.rs index f25b383f4a..8fbd54d09a 100644 --- a/src/api/schema/events/metric.rs +++ b/src/api/schema/events/metric.rs @@ -126,10 +126,14 @@ impl Metric { .expect("logfmt serialization of metric event failed: conversion to serde Value failed. Please report."); match json { Value::Object(map) => encode_logfmt::encode_map( - &map.into_iter().map(|(k,v)| (event::KeyString::from(k), v)).collect(), + &map.into_iter() + .map(|(k, v)| (event::KeyString::from(k), v)) + .collect(), ) .expect("logfmt serialization of metric event failed. Please report."), - _ => panic!("logfmt serialization of metric event failed: metric converted to unexpected serde Value. Please report."), + _ => panic!( + "logfmt serialization of metric event failed: metric converted to unexpected serde Value. Please report." + ), } } } diff --git a/src/api/schema/events/mod.rs b/src/api/schema/events/mod.rs index 5bb2434068..dc6f26b254 100644 --- a/src/api/schema/events/mod.rs +++ b/src/api/schema/events/mod.rs @@ -6,9 +6,10 @@ pub mod trace; use async_graphql::{Context, Subscription}; use encoding::EventEncodingType; -use futures::{stream, Stream, StreamExt}; -use output::{from_tap_payload_to_output_events, OutputEventsPayload}; -use rand::{rngs::SmallRng, Rng, SeedableRng}; +use futures::{Stream, StreamExt, stream}; +use output::{OutputEventsPayload, from_tap_payload_to_output_events}; +use rand::{Rng, SeedableRng, rngs::SmallRng}; +use std::time::{SystemTime, UNIX_EPOCH}; use tokio::{select, sync::mpsc, time}; use tokio_stream::wrappers::ReceiverStream; use vector_lib::tap::{ @@ -81,7 +82,11 @@ pub(crate) fn create_events_stream( // Random number generator to allow for sampling. Speed trumps cryptographic security here. // The RNG must be Send + Sync to use with the `select!` loop below, hence `SmallRng`. - let mut rng = SmallRng::from_entropy(); + let seed = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos() as u64; + let mut rng = SmallRng::seed_from_u64(seed); // Keep a count of the batch size, which will be used as a seed for random eviction // per the sampling strategy used below. @@ -113,7 +118,7 @@ pub(crate) fn create_events_stream( if limit > results.len() { results.push(payload); } else { - let random_number = rng.gen_range(0..batch); + let random_number = rng.random_range(0..batch); if random_number < results.len() { results[random_number] = payload; } diff --git a/src/api/schema/filter.rs b/src/api/schema/filter.rs index c73b88a6c0..ac8f7f551d 100644 --- a/src/api/schema/filter.rs +++ b/src/api/schema/filter.rs @@ -2,12 +2,12 @@ use std::collections::BTreeSet; use async_graphql::{InputObject, InputType}; -use super::components::{source, ComponentKind}; +use super::components::{ComponentKind, source}; /// Takes an `&Option` and returns early if false #[macro_export] macro_rules! filter_check { - ($($match:expr),+) => { + ($($match:expr_2021),+) => { $( if matches!($match, Some(t) if !t) { return false; diff --git a/src/api/schema/gen.rs b/src/api/schema/gen.rs index 33e27f6584..c359f0db3b 100644 --- a/src/api/schema/gen.rs +++ b/src/api/schema/gen.rs @@ -110,7 +110,7 @@ async fn main() { fs::write( "lib/vector-api-client/graphql/schema.json", - format!("{}\n", json), + format!("{json}\n"), ) .expect("Couldn't save schema file"); } diff --git a/src/api/schema/health.rs b/src/api/schema/health.rs index ebd24184ca..0ee3d1d897 100644 --- a/src/api/schema/health.rs +++ b/src/api/schema/health.rs @@ -1,7 +1,7 @@ use async_graphql::{Object, SimpleObject, Subscription}; use chrono::{DateTime, Utc}; use tokio::time::Duration; -use tokio_stream::{wrappers::IntervalStream, Stream, StreamExt}; +use tokio_stream::{Stream, StreamExt, wrappers::IntervalStream}; #[derive(SimpleObject)] pub struct Heartbeat { @@ -34,7 +34,7 @@ impl HealthSubscription { async fn heartbeat( &self, #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream { + ) -> impl Stream + use<> { IntervalStream::new(tokio::time::interval(Duration::from_millis( interval as u64, ))) diff --git a/src/api/schema/metrics/filter.rs b/src/api/schema/metrics/filter.rs index 52446cdc73..c85379edee 100644 --- a/src/api/schema/metrics/filter.rs +++ b/src/api/schema/metrics/filter.rs @@ -5,8 +5,8 @@ use tokio::time::Duration; use tokio_stream::{Stream, StreamExt}; use super::{ - filter_output_metric, OutputThroughput, ReceivedBytesTotal, ReceivedEventsTotal, - SentBytesTotal, SentEventsTotal, + OutputThroughput, ReceivedBytesTotal, ReceivedEventsTotal, SentBytesTotal, SentEventsTotal, + filter_output_metric, }; use crate::{ config::ComponentKey, @@ -27,11 +27,7 @@ pub fn sum_metrics<'a, I: IntoIterator>(metrics: I) -> Option Some(iter.fold( m.clone(), |mut m1, m2| { - if m1.update(m2) { - m1 - } else { - m2.clone() - } + if m1.update(m2) { m1 } else { m2.clone() } }, )) } @@ -303,7 +299,7 @@ pub fn component_sent_events_totals_metrics_with_outputs( match m.value() { MetricValue::Counter { value } if cache - .insert(format!("{}.{}", id, output), *value) + .insert(format!("{id}.{output}"), *value) .unwrap_or(0.00) < *value => { @@ -349,8 +345,7 @@ pub fn component_sent_events_total_throughputs_with_outputs( .iter() .filter_map(|output| { let m = filter_output_metric(metrics.as_ref(), output.as_ref())?; - let throughput = - throughput(&m, format!("{}.{}", id, output), &mut cache)?; + let throughput = throughput(&m, format!("{id}.{output}"), &mut cache)?; Some(OutputThroughput::new(output.clone(), throughput as i64)) }) .collect::>(); diff --git a/src/api/schema/metrics/host.rs b/src/api/schema/metrics/host.rs index 6a814b1c5f..a75243253c 100644 --- a/src/api/schema/metrics/host.rs +++ b/src/api/schema/metrics/host.rs @@ -1,9 +1,9 @@ -use async_graphql::Object; - use crate::{ event::{Metric, MetricValue}, sources::host_metrics, }; +use async_graphql::Object; +use cfg_if::cfg_if; pub struct MemoryMetrics(Vec); @@ -259,23 +259,27 @@ impl DiskMetrics { } } -pub struct TCPMetrics(Vec); - -#[Object] -impl TCPMetrics { - /// Total TCP connections - async fn tcp_conns_total(&self) -> f64 { - filter_host_metric(&self.0, "tcp_connections_total") - } - - /// Total bytes in the send queue across all connections. - async fn tcp_tx_queued_bytes_total(&self) -> f64 { - filter_host_metric(&self.0, "tcp_tx_queued_bytes_total") - } - - /// Total bytes in the receive queue across all connections. - async fn tcp_rx_queued_bytes_total(&self) -> f64 { - filter_host_metric(&self.0, "tcp_rx_queued_bytes_total") +cfg_if! { + if #[cfg(target_os = "linux")] { + pub struct TCPMetrics(Vec); + + #[Object] + impl TCPMetrics { + /// Total TCP connections + async fn tcp_conns_total(&self) -> f64 { + filter_host_metric(&self.0, "tcp_connections_total") + } + + /// Total bytes in the send queue across all connections. + async fn tcp_tx_queued_bytes_total(&self) -> f64 { + filter_host_metric(&self.0, "tcp_tx_queued_bytes_total") + } + + /// Total bytes in the receive queue across all connections. + async fn tcp_rx_queued_bytes_total(&self) -> f64 { + filter_host_metric(&self.0, "tcp_rx_queued_bytes_total") + } + } } } diff --git a/src/api/schema/metrics/mod.rs b/src/api/schema/metrics/mod.rs index 08baa7cc12..056b3c27a7 100644 --- a/src/api/schema/metrics/mod.rs +++ b/src/api/schema/metrics/mod.rs @@ -42,6 +42,7 @@ pub enum MetricType { Uptime(Uptime), } +#[cfg(feature = "sources-host_metrics")] #[derive(Default)] pub struct MetricsQuery; @@ -63,7 +64,7 @@ impl MetricsSubscription { async fn uptime( &self, #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream { + ) -> impl Stream + use<> { get_metrics(interval).filter_map(|m| match m.name() { "uptime_seconds" => Some(Uptime::new(m)), _ => None, @@ -75,7 +76,7 @@ impl MetricsSubscription { async fn received_events_total( &self, #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream { + ) -> impl Stream + use<> { get_metrics(interval).filter_map(|m| match m.name() { "component_received_events_total" => Some(ReceivedEventsTotal::new(m)), _ => None, @@ -87,7 +88,7 @@ impl MetricsSubscription { async fn received_events_throughput( &self, #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream { + ) -> impl Stream + use<> { counter_throughput(interval, &|m| m.name() == "component_received_events_total") .map(|(_, throughput)| throughput as i64) } @@ -96,7 +97,7 @@ impl MetricsSubscription { async fn component_received_events_throughputs( &self, #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream> { + ) -> impl Stream> + use<> { component_counter_throughputs(interval, &|m| m.name() == "component_received_events_total") .map(|m| { m.into_iter() @@ -114,7 +115,7 @@ impl MetricsSubscription { async fn component_received_events_totals( &self, #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream> { + ) -> impl Stream> + use<> { component_counter_metrics(interval, &|m| m.name() == "component_received_events_total").map( |m| { m.into_iter() @@ -129,7 +130,7 @@ impl MetricsSubscription { async fn sent_events_total( &self, #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream { + ) -> impl Stream + use<> { get_metrics(interval).filter_map(|m| match m.name() { "component_sent_events_total" => Some(SentEventsTotal::new(m)), _ => None, @@ -141,7 +142,7 @@ impl MetricsSubscription { async fn sent_events_throughput( &self, #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream { + ) -> impl Stream + use<> { counter_throughput(interval, &|m| m.name() == "component_sent_events_total") .map(|(_, throughput)| throughput as i64) } @@ -150,7 +151,7 @@ impl MetricsSubscription { async fn component_sent_events_throughputs( &self, #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream> { + ) -> impl Stream> + use<> { component_sent_events_total_throughputs_with_outputs(interval).map(|m| { m.into_iter() .map(|(key, total_throughput, outputs)| { @@ -164,7 +165,7 @@ impl MetricsSubscription { async fn component_sent_events_totals( &self, #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream> { + ) -> impl Stream> + use<> { component_sent_events_totals_metrics_with_outputs(interval).map(|ms| { ms.into_iter() .map(|(m, m_by_outputs)| ComponentSentEventsTotal::new(m, m_by_outputs)) @@ -176,7 +177,7 @@ impl MetricsSubscription { async fn component_received_bytes_totals( &self, #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream> { + ) -> impl Stream> + use<> { component_counter_metrics(interval, &|m| m.name() == "component_received_bytes_total").map( |m| { m.into_iter() @@ -190,7 +191,7 @@ impl MetricsSubscription { async fn component_received_bytes_throughputs( &self, #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream> { + ) -> impl Stream> + use<> { component_counter_throughputs(interval, &|m| m.name() == "component_received_bytes_total") .map(|m| { m.into_iter() @@ -208,7 +209,7 @@ impl MetricsSubscription { async fn component_sent_bytes_totals( &self, #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream> { + ) -> impl Stream> + use<> { component_counter_metrics(interval, &|m| m.name() == "component_sent_bytes_total") .map(|m| m.into_iter().map(ComponentSentBytesTotal::new).collect()) } @@ -217,7 +218,7 @@ impl MetricsSubscription { async fn component_sent_bytes_throughputs( &self, #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream> { + ) -> impl Stream> + use<> { component_counter_throughputs(interval, &|m| m.name() == "component_sent_bytes_total").map( |m| { m.into_iter() @@ -236,7 +237,7 @@ impl MetricsSubscription { async fn errors_total( &self, #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream { + ) -> impl Stream + use<> { get_metrics(interval) .filter(|m| m.name().ends_with("_errors_total")) .map(ErrorsTotal::new) @@ -246,7 +247,7 @@ impl MetricsSubscription { async fn allocated_bytes( &self, #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream { + ) -> impl Stream + use<> { get_metrics(interval) .filter(|m| m.name() == "component_allocated_bytes") .map(AllocatedBytes::new) @@ -256,7 +257,7 @@ impl MetricsSubscription { async fn component_allocated_bytes( &self, #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream> { + ) -> impl Stream> + use<> { component_gauge_metrics(interval, &|m| m.name() == "component_allocated_bytes") .map(|m| m.into_iter().map(ComponentAllocatedBytes::new).collect()) } @@ -265,7 +266,7 @@ impl MetricsSubscription { async fn component_errors_totals( &self, #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream> { + ) -> impl Stream> + use<> { component_counter_metrics(interval, &|m| m.name().ends_with("_errors_total")) .map(|m| m.into_iter().map(ComponentErrorsTotal::new).collect()) } @@ -274,7 +275,7 @@ impl MetricsSubscription { async fn metrics( &self, #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream { + ) -> impl Stream + use<> { get_metrics(interval).filter_map(|m| match m.name() { "uptime_seconds" => Some(MetricType::Uptime(m.into())), _ => None, diff --git a/src/api/schema/metrics/output.rs b/src/api/schema/metrics/output.rs index 1bf67791dc..e1fd0976e7 100644 --- a/src/api/schema/metrics/output.rs +++ b/src/api/schema/metrics/output.rs @@ -1,7 +1,7 @@ use async_graphql::Object; use vector_lib::config::ComponentKey; -use super::{by_component_key, sum_metrics, SentEventsTotal}; +use super::{SentEventsTotal, by_component_key, sum_metrics}; use crate::event::Metric; #[derive(Debug, Clone)] diff --git a/src/api/schema/metrics/source/file.rs b/src/api/schema/metrics/source/file.rs index bebe6bbf6b..43c2a2751c 100644 --- a/src/api/schema/metrics/source/file.rs +++ b/src/api/schema/metrics/source/file.rs @@ -4,7 +4,7 @@ use async_graphql::{Enum, InputObject, Object}; use crate::{ api::schema::{ - filter::{filter_items, CustomFilter, StringFilter}, + filter::{CustomFilter, StringFilter, filter_items}, metrics::{self, MetricsFilter}, relay, sort, }, @@ -25,13 +25,13 @@ impl<'a> FileSourceMetricFile<'a> { Self { name, metrics } } - pub fn get_name(&self) -> &str { + pub const fn get_name(&self) -> &str { self.name.as_str() } } #[Object] -impl<'a> FileSourceMetricFile<'a> { +impl FileSourceMetricFile<'_> { /// File name async fn name(&self) -> &str { &*self.name @@ -135,10 +135,11 @@ pub struct FileSourceMetricsFilesFilter { impl CustomFilter> for FileSourceMetricsFilesFilter { fn matches(&self, file: &FileSourceMetricFile<'_>) -> bool { - filter_check!(self - .name - .as_ref() - .map(|f| f.iter().all(|f| f.filter_value(file.get_name())))); + filter_check!( + self.name + .as_ref() + .map(|f| f.iter().all(|f| f.filter_value(file.get_name()))) + ); true } @@ -214,7 +215,7 @@ mod tests { } } - fn get_metric(&self) -> FileSourceMetricFile { + fn get_metric(&self) -> FileSourceMetricFile<'_> { FileSourceMetricFile::from_tuple(( self.name.to_string(), vec![&self.bytes_metric, &self.events_metric], @@ -249,7 +250,7 @@ mod tests { sort::by_fields(&mut files, &fields); for (i, f) in ["1", "2", "3"].iter().enumerate() { - assert_eq!(files[i].name.as_str(), format!("/path/to/file/{}", f)); + assert_eq!(files[i].name.as_str(), format!("/path/to/file/{f}")); } } @@ -268,7 +269,7 @@ mod tests { sort::by_fields(&mut files, &fields); for (i, f) in ["3", "2", "1"].iter().enumerate() { - assert_eq!(files[i].name.as_str(), format!("/path/to/file/{}", f)); + assert_eq!(files[i].name.as_str(), format!("/path/to/file/{f}")); } } diff --git a/src/api/schema/relay.rs b/src/api/schema/relay.rs index 366a71723a..3124a2272a 100644 --- a/src/api/schema/relay.rs +++ b/src/api/schema/relay.rs @@ -1,10 +1,10 @@ use std::convert::Infallible; use async_graphql::{ - connection::{self, Connection, CursorType, Edge, EmptyFields}, Result, SimpleObject, + connection::{self, Connection, CursorType, Edge, EmptyFields}, }; -use base64::prelude::{Engine as _, BASE64_URL_SAFE_NO_PAD}; +use base64::prelude::{BASE64_URL_SAFE_NO_PAD, Engine as _}; /// Base64 invalid states, used by `Base64Cursor`. pub enum Base64CursorError { @@ -49,7 +49,7 @@ impl Base64Cursor { let cursor = String::from_utf8(bytes).map_err(|_| Base64CursorError::Invalid)?; let index = cursor .split(':') - .last() + .next_back() .map(|s| s.parse::()) .ok_or(Base64CursorError::Invalid)? .map_err(|_| Base64CursorError::Invalid)?; diff --git a/src/api/schema/sort.rs b/src/api/schema/sort.rs index aaefe41293..8e40750c88 100644 --- a/src/api/schema/sort.rs +++ b/src/api/schema/sort.rs @@ -8,8 +8,8 @@ use itertools::{ use crate::api::schema::{ components::{ - sink::SinksSortFieldName, source::SourcesSortFieldName, transform::TransformsSortFieldName, - ComponentsSortFieldName, + ComponentsSortFieldName, sink::SinksSortFieldName, source::SourcesSortFieldName, + transform::TransformsSortFieldName, }, metrics::source::file::FileSourceMetricFilesSortFieldName, }; diff --git a/src/api/server.rs b/src/api/server.rs index caf6512b62..76e3f24ce2 100644 --- a/src/api/server.rs +++ b/src/api/server.rs @@ -1,21 +1,21 @@ use std::{ convert::Infallible, net::SocketAddr, - sync::{atomic::AtomicBool, Arc}, + sync::{Arc, atomic::AtomicBool}, }; use async_graphql::{ - http::{playground_source, GraphQLPlaygroundConfig, WebSocketProtocols}, Data, Request, Schema, + http::{GraphQLPlaygroundConfig, WebSocketProtocols, playground_source}, }; -use async_graphql_warp::{graphql_protocol, GraphQLResponse, GraphQLWebSocket}; -use hyper::{server::conn::AddrIncoming, service::make_service_fn, Server as HyperServer}; +use async_graphql_warp::{GraphQLResponse, GraphQLWebSocket, graphql_protocol}; +use hyper::{Server as HyperServer, server::conn::AddrIncoming, service::make_service_fn}; use tokio::runtime::Handle; use tokio::sync::oneshot; use tower::ServiceBuilder; use tracing::Span; use vector_lib::tap::topology; -use warp::{filters::BoxedFilter, http::Response, ws::Ws, Filter, Reply}; +use warp::{Filter, Reply, filters::BoxedFilter, http::Response, ws::Ws}; use super::{handler, schema}; use crate::{ diff --git a/src/api/tests.rs b/src/api/tests.rs index b9f06ac3ed..40203b0168 100644 --- a/src/api/tests.rs +++ b/src/api/tests.rs @@ -69,12 +69,12 @@ async fn sink_events() { Some(TapPayload::Notification(Notification::Matched(matched))) if matched.pattern == pattern_matched => { - continue + continue; } Some(TapPayload::Notification(Notification::NotMatched(not_matched))) if not_matched.pattern == pattern_not_matched => { - continue + continue; } _ => panic!("unexpected payload"), } @@ -198,13 +198,13 @@ async fn integration_test_source_metric() { "to_metric", &["in"], LogToMetricConfig { - metrics: vec![MetricConfig { + metrics: Some(vec![MetricConfig { field: "message".try_into().expect("Fixed template string"), name: None, namespace: None, tags: None, metric: MetricTypeConfig::Gauge, - }], + }]), all_metrics: None, }, ); @@ -340,13 +340,17 @@ async fn integration_test_transform_input() { assert_notification(tap_events[1][0].clone()), assert_notification(tap_events[2][0].clone()), ]; - assert!(notifications - .iter() - .any(|n| *n == Notification::Matched(Matched::new("transform".to_string())))); + assert!( + notifications + .iter() + .any(|n| *n == Notification::Matched(Matched::new("transform".to_string()))) + ); // "in" is not matched since it corresponds to a source - assert!(notifications - .iter() - .any(|n| *n == Notification::NotMatched(NotMatched::new("in".to_string())))); + assert!( + notifications + .iter() + .any(|n| *n == Notification::NotMatched(NotMatched::new("in".to_string()))) + ); // "in" generates an invalid match notification to warn against an // attempt to tap the input of a source assert!(notifications.iter().any(|n| *n diff --git a/src/app.rs b/src/app.rs index 32586b4c8c..b00208980e 100644 --- a/src/app.rs +++ b/src/app.rs @@ -10,15 +10,14 @@ use std::{ use exitcode::ExitCode; use futures::StreamExt; use tokio::runtime::{self, Runtime}; -use tokio::sync::{broadcast::error::RecvError, MutexGuard}; +use tokio::sync::{MutexGuard, broadcast::error::RecvError}; use tokio_stream::wrappers::UnboundedReceiverStream; -use crate::extra_context::ExtraContext; #[cfg(feature = "api")] use crate::{api, internal_events::ApiStarted}; use crate::{ - cli::{handle_config_errors, LogFormat, Opts, RootOpts, WatchConfigMethod}, - config::{self, Config, ConfigPath}, + cli::{LogFormat, Opts, RootOpts, WatchConfigMethod, handle_config_errors}, + config::{self, ComponentConfig, Config, ConfigPath}, heartbeat, internal_events::{VectorConfigLoadError, VectorQuit, VectorStarted, VectorStopped}, signal::{SignalHandler, SignalPair, SignalRx, SignalTo}, @@ -28,6 +27,7 @@ use crate::{ }, trace, }; +use crate::{config::ComponentType, extra_context::ExtraContext}; #[cfg(unix)] use std::os::unix::process::ExitStatusExt; @@ -158,7 +158,9 @@ impl ApplicationConfig { } } } else { - info!(message="API is disabled, enable by setting `api.enabled` to `true` and use commands like `vector top`."); + info!( + message = "API is disabled, enable by setting `api.enabled` to `true` and use commands like `vector top`." + ); None } } @@ -206,7 +208,9 @@ impl Application { // Can only log this after initializing the logging subsystem if opts.root.openssl_no_probe { - debug!(message = "Disabled probing and configuration of root certificate locations on the system for OpenSSL."); + debug!( + message = "Disabled probing and configuration of root certificate locations on the system for OpenSSL." + ); } let runtime = build_runtime(opts.root.threads, "vector-worker")?; @@ -336,6 +340,27 @@ async fn handle_signal( allow_empty_config: bool, ) -> Option { match signal { + Ok(SignalTo::ReloadComponents(components_to_reload)) => { + let mut topology_controller = topology_controller.lock().await; + topology_controller + .topology + .extend_reload_set(components_to_reload); + + // Reload paths + if let Some(paths) = config::process_paths(config_paths) { + topology_controller.config_paths = paths; + } + + // Reload config + let new_config = config::load_from_paths_with_provider_and_secrets( + &topology_controller.config_paths, + signal_handler, + allow_empty_config, + ) + .await; + + reload_config_from_result(topology_controller, new_config).await + } Ok(SignalTo::ReloadFromConfigBuilder(config_builder)) => { let topology_controller = topology_controller.lock().await; reload_config_from_result(topology_controller, config_builder.build()).await @@ -358,6 +383,15 @@ async fn handle_signal( reload_config_from_result(topology_controller, new_config).await } + Ok(SignalTo::ReloadEnrichmentTables) => { + let topology_controller = topology_controller.lock().await; + + topology_controller + .topology + .reload_enrichment_tables() + .await; + None + } Err(RecvError::Lagged(amt)) => { warn!("Overflow, dropped {} signals.", amt); None @@ -491,23 +525,14 @@ pub async fn load_configs( ) -> Result { let config_paths = config::process_paths(config_paths).ok_or(exitcode::CONFIG)?; - if let Some(watcher_conf) = watcher_conf { - // Start listening for config changes immediately. - config::watcher::spawn_thread( - watcher_conf, - signal_handler.clone_tx(), - config_paths.iter().map(Into::into), - None, - ) - .map_err(|error| { - error!(message = "Unable to start config watcher.", %error); - exitcode::CONFIG - })?; - } + let watched_paths = config_paths + .iter() + .map(<&PathBuf>::from) + .collect::>(); info!( message = "Loading configs.", - paths = ?config_paths.iter().map(<&PathBuf>::from).collect::>() + paths = ?watched_paths ); let mut config = config::load_from_paths_with_provider_and_secrets( @@ -518,6 +543,62 @@ pub async fn load_configs( .await .map_err(handle_config_errors)?; + let mut watched_component_paths = Vec::new(); + + if let Some(watcher_conf) = watcher_conf { + for (name, transform) in config.transforms() { + let files = transform.inner.files_to_watch(); + let component_config = ComponentConfig::new( + files.into_iter().cloned().collect(), + name.clone(), + ComponentType::Transform, + ); + watched_component_paths.push(component_config); + } + + for (name, sink) in config.sinks() { + let files = sink.inner.files_to_watch(); + let component_config = ComponentConfig::new( + files.into_iter().cloned().collect(), + name.clone(), + ComponentType::Sink, + ); + watched_component_paths.push(component_config); + } + + for (name, table) in config.enrichment_tables() { + let files = table.inner.files_to_watch(); + let component_config = ComponentConfig::new( + files.into_iter().cloned().collect(), + name.clone(), + ComponentType::EnrichmentTable, + ); + watched_component_paths.push(component_config); + } + + info!( + message = "Starting watcher.", + paths = ?watched_paths + ); + info!( + message = "Components to watch.", + paths = ?watched_component_paths + ); + + // Start listening for config changes. + config::watcher::spawn_thread( + watcher_conf, + signal_handler.clone_tx(), + watched_paths, + watched_component_paths, + None, + ) + .map_err(|error| { + error!(message = "Unable to start config watcher.", %error); + exitcode::CONFIG + })?; + } + config::init_log_schema(config.global.log_schema.clone(), true); config::init_telemetry(config.global.telemetry.clone(), true); diff --git a/src/async_read.rs b/src/async_read.rs index 00a3c53024..0ab27f2a01 100644 --- a/src/async_read.rs +++ b/src/async_read.rs @@ -39,7 +39,7 @@ impl AllowReadUntil { &self.reader } - pub fn get_mut(&mut self) -> &mut S { + pub const fn get_mut(&mut self) -> &mut S { &mut self.reader } } @@ -66,7 +66,7 @@ where mod tests { use futures::FutureExt; use tokio::{ - fs::{remove_file, File}, + fs::{File, remove_file}, io::{AsyncBufReadExt, AsyncWriteExt, BufReader, BufWriter}, }; diff --git a/src/aws/auth.rs b/src/aws/auth.rs index 0cf952319c..7fca5f30d3 100644 --- a/src/aws/auth.rs +++ b/src/aws/auth.rs @@ -2,20 +2,15 @@ use std::time::Duration; use aws_config::{ - default_provider::credentials::DefaultCredentialsChain, - identity::IdentityCache, - imds, - profile::{ - profile_file::{ProfileFileKind, ProfileFiles}, - ProfileFileCredentialsProvider, - }, - provider_config::ProviderConfig, + default_provider::credentials::DefaultCredentialsChain, identity::IdentityCache, imds, + profile::ProfileFileCredentialsProvider, provider_config::ProviderConfig, sts::AssumeRoleProviderBuilder, }; -use aws_credential_types::{provider::SharedCredentialsProvider, Credentials}; +use aws_credential_types::{Credentials, provider::SharedCredentialsProvider}; +use aws_runtime::env_config::file::{EnvConfigFileKind, EnvConfigFiles}; use aws_smithy_async::time::SystemTimeSource; use aws_smithy_runtime_api::client::identity::SharedIdentityCache; -use aws_types::{region::Region, SdkConfig}; +use aws_types::{SdkConfig, region::Region}; use serde_with::serde_as; use vector_lib::configurable::configurable_component; use vector_lib::{config::proxy::ProxyConfig, sensitive_string::SensitiveString, tls::TlsConfig}; @@ -28,7 +23,7 @@ const DEFAULT_PROFILE_NAME: &str = "default"; /// IMDS Client Configuration for authenticating with AWS. #[serde_as] #[configurable_component] -#[derive(Copy, Clone, Debug, Derivative)] +#[derive(Copy, Clone, Debug, Derivative, Eq, PartialEq)] #[derivative(Default)] #[serde(deny_unknown_fields)] pub struct ImdsAuthentication { @@ -62,7 +57,7 @@ const fn default_timeout() -> Duration { /// Configuration of the authentication strategy for interacting with AWS services. #[configurable_component] -#[derive(Clone, Debug, Derivative)] +#[derive(Clone, Debug, Derivative, Eq, PartialEq)] #[derivative(Default)] #[serde(deny_unknown_fields, untagged)] pub enum AwsAuthentication { @@ -76,6 +71,11 @@ pub enum AwsAuthentication { #[configurable(metadata(docs::examples = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"))] secret_access_key: SensitiveString, + /// The AWS session token. + /// See [AWS temporary credentials](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_use-resources.html) + #[configurable(metadata(docs::examples = "AQoDYXdz...AQoDYXdz..."))] + session_token: Option, + /// The ARN of an [IAM role][iam_role] to assume. /// /// [iam_role]: https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles.html @@ -100,7 +100,7 @@ pub enum AwsAuthentication { /// The optional [RoleSessionName][role_session_name] is a unique session identifier for your assumed role. /// /// Should be unique per principal or reason. - /// If not set, session name will be autogenerated like assume-role-provider-1736428351340 + /// If not set, the session name is autogenerated like assume-role-provider-1736428351340 /// /// [role_session_name]: https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html #[configurable(metadata(docs::examples = "vector-indexer-role"))] @@ -123,6 +123,15 @@ pub enum AwsAuthentication { #[configurable(metadata(docs::examples = "develop"))] #[serde(default = "default_profile")] profile: String, + + /// The [AWS region][aws_region] to send STS requests to. + /// + /// If not set, this defaults to the configured region + /// for the service itself. + /// + /// [aws_region]: https://docs.aws.amazon.com/general/latest/gr/rande.html#regional-endpoints + #[configurable(metadata(docs::examples = "us-west-2"))] + region: Option, }, /// Assume the given role ARN. @@ -163,7 +172,7 @@ pub enum AwsAuthentication { /// The optional [RoleSessionName][role_session_name] is a unique session identifier for your assumed role. /// /// Should be unique per principal or reason. - /// If not set, session name will be autogenerated like assume-role-provider-1736428351340 + /// If not set, the session name is autogenerated like assume-role-provider-1736428351340 /// /// [role_session_name]: https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html #[configurable(metadata(docs::examples = "vector-indexer-role"))] @@ -271,11 +280,12 @@ impl AwsAuthentication { external_id, region, session_name, + session_token, } => { let provider = SharedCredentialsProvider::new(Credentials::from_keys( access_key_id.inner(), secret_access_key.inner(), - None, + session_token.clone().map(|v| v.inner().into()), )); if let Some(assume_role) = assume_role { let auth_region = region.clone().map(Region::new).unwrap_or(service_region); @@ -297,16 +307,20 @@ impl AwsAuthentication { AwsAuthentication::File { credentials_file, profile, + region, } => { let connector = super::connector(proxy, tls_options)?; // The SDK uses the default profile out of the box, but doesn't provide an optional // type in the builder. We can just hardcode it so that everything works. - let profile_files = ProfileFiles::builder() - .with_file(ProfileFileKind::Credentials, credentials_file) + let profile_files = EnvConfigFiles::builder() + .with_file(EnvConfigFileKind::Credentials, credentials_file) .build(); - let provider_config = ProviderConfig::empty().with_http_client(connector); + let auth_region = region.clone().map(Region::new).unwrap_or(service_region); + let provider_config = ProviderConfig::empty() + .with_region(Option::from(auth_region)) + .with_http_client(connector); let profile_provider = ProfileFileCredentialsProvider::builder() .profile_files(profile_files) @@ -364,6 +378,7 @@ impl AwsAuthentication { external_id: None, region: None, session_name: None, + session_token: None, } } } @@ -690,6 +705,7 @@ mod tests { r#" auth.credentials_file = "/path/to/file" auth.profile = "foo" + auth.region = "us-west-2" "#, ) .unwrap(); @@ -698,9 +714,11 @@ mod tests { AwsAuthentication::File { credentials_file, profile, + region, } => { assert_eq!(&credentials_file, "/path/to/file"); assert_eq!(&profile, "foo"); + assert_eq!(region.unwrap(), "us-west-2"); } _ => panic!(), } @@ -716,6 +734,7 @@ mod tests { AwsAuthentication::File { credentials_file, profile, + .. } => { assert_eq!(&credentials_file, "/path/to/file"); assert_eq!(profile, "default".to_string()); diff --git a/src/aws/mod.rs b/src/aws/mod.rs index 0a70228cd2..6b8857f7e5 100644 --- a/src/aws/mod.rs +++ b/src/aws/mod.rs @@ -5,7 +5,7 @@ pub mod timeout; pub use auth::{AwsAuthentication, ImdsAuthentication}; use aws_config::{ - meta::region::ProvideRegion, retry::RetryConfig, timeout::TimeoutConfig, Region, SdkConfig, + Region, SdkConfig, meta::region::ProvideRegion, retry::RetryConfig, timeout::TimeoutConfig, }; use aws_credential_types::provider::{ProvideCredentials, SharedCredentialsProvider}; use aws_sigv4::{ @@ -28,7 +28,7 @@ use aws_types::sdk_config::SharedHttpClient; use bytes::Bytes; use futures_util::FutureExt; use http::HeaderMap; -use http_body::{combinators::BoxBody, Body}; +use http_body::{Body, combinators::BoxBody}; use pin_project::pin_project; use regex::RegexSet; pub use region::RegionOrEndpoint; @@ -37,8 +37,8 @@ use std::{ error::Error, pin::Pin, sync::{ - atomic::{AtomicUsize, Ordering}, Arc, OnceLock, + atomic::{AtomicUsize, Ordering}, }, task::{Context, Poll}, time::{Duration, SystemTime}, @@ -126,10 +126,11 @@ pub trait ClientBuilder { fn build(&self, config: &SdkConfig) -> Self::Client; } -fn region_provider( +/// Provides the configured AWS region. +pub fn region_provider( proxy: &ProxyConfig, tls_options: Option<&TlsConfig>, -) -> crate::Result { +) -> crate::Result> { let config = aws_config::provider_config::ProviderConfig::default() .with_http_client(connector(proxy, tls_options)?); @@ -222,6 +223,10 @@ where if let Some(endpoint_override) = endpoint { config_builder = config_builder.endpoint_url(endpoint_override); + } else if let Some(endpoint_from_config) = + aws_config::default_provider::endpoint_url::endpoint_url_provider(&provider_config).await + { + config_builder = config_builder.endpoint_url(endpoint_from_config); } if let Some(use_fips) = @@ -361,13 +366,13 @@ where HttpConnectorFuture::new(fut.inspect(move |result| { let byte_size = bytes_sent.load(Ordering::Relaxed); - if let Ok(result) = result { - if result.status().is_success() { - emit!(AwsBytesSent { - byte_size, - region: Some(region), - }); - } + if let Ok(result) = result + && result.status().is_success() + { + emit!(AwsBytesSent { + byte_size, + region: Some(region), + }); } })) } diff --git a/src/aws/region.rs b/src/aws/region.rs index 4c13d53b02..4a777a4657 100644 --- a/src/aws/region.rs +++ b/src/aws/region.rs @@ -55,24 +55,30 @@ mod tests { #[test] fn optional() { - assert!(toml::from_str::(indoc! {" - "}) - .is_ok()); + assert!( + toml::from_str::(indoc! {" + "}) + .is_ok() + ); } #[test] fn region_optional() { - assert!(toml::from_str::(indoc! {r#" + assert!( + toml::from_str::(indoc! {r#" endpoint = "http://localhost:8080" "#}) - .is_ok()); + .is_ok() + ); } #[test] fn endpoint_optional() { - assert!(toml::from_str::(indoc! {r#" + assert!( + toml::from_str::(indoc! {r#" region = "us-east-1" "#}) - .is_ok()); + .is_ok() + ); } } diff --git a/src/cli.rs b/src/cli.rs index 6ab179ddfa..2795601545 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -258,7 +258,9 @@ impl RootOpts { pub fn init_global(&self) { if !self.openssl_no_probe { - openssl_probe::init_ssl_cert_env_vars(); + unsafe { + openssl_probe::init_openssl_env_vars(); + } } crate::metrics::init_global().expect("metrics initialization failed"); @@ -284,11 +286,13 @@ pub enum SubCommand { /// Generate the configuration schema for this version of Vector. (experimental) /// - /// A JSON Schema document will be written to stdout that represents the valid schema for a + /// A JSON Schema document will be generated that represents the valid schema for a /// Vector configuration. This schema is based on the "full" configuration, such that for usages /// where a configuration is split into multiple files, the schema would apply to those files /// only when concatenated together. - GenerateSchema, + /// + /// By default all output is writen to stdout. The `output_path` option can be used to redirect to a file. + GenerateSchema(generate_schema::Opts), /// Output a provided Vector configuration file/dir as a single JSON object, useful for checking in to version control. #[command(hide = true)] @@ -330,7 +334,7 @@ impl SubCommand { Self::Config(c) => config::cmd(c), Self::ConvertConfig(opts) => convert_config::cmd(opts), Self::Generate(g) => generate::cmd(g), - Self::GenerateSchema => generate_schema::cmd(), + Self::GenerateSchema(opts) => generate_schema::cmd(opts), Self::Graph(g) => graph::cmd(g), Self::List(l) => list::cmd(l), #[cfg(windows)] diff --git a/src/codecs/decoding/decoder.rs b/src/codecs/decoding/decoder.rs index 816cedfdd8..6569742ff2 100644 --- a/src/codecs/decoding/decoder.rs +++ b/src/codecs/decoding/decoder.rs @@ -1,8 +1,8 @@ use bytes::{Bytes, BytesMut}; use smallvec::SmallVec; use vector_lib::codecs::decoding::{ - format::Deserializer as _, BoxedFramingError, BytesDeserializer, Deserializer, Error, Framer, - NewlineDelimitedDecoder, + BoxedFramingError, BytesDeserializer, Deserializer, Error, Framer, NewlineDelimitedDecoder, + format::Deserializer as _, }; use vector_lib::config::LogNamespace; @@ -103,11 +103,11 @@ impl tokio_util::codec::Decoder for Decoder { mod tests { use super::Decoder; use bytes::Bytes; - use futures::{stream, StreamExt}; + use futures::{StreamExt, stream}; use tokio_util::{codec::FramedRead, io::StreamReader}; use vector_lib::codecs::{ - decoding::{Deserializer, Framer}, JsonDeserializer, NewlineDelimitedDecoder, StreamDecodingError, + decoding::{Deserializer, Framer}, }; use vrl::value::Value; diff --git a/src/codecs/encoding/config.rs b/src/codecs/encoding/config.rs index b11fe751a7..f3653bc76e 100644 --- a/src/codecs/encoding/config.rs +++ b/src/codecs/encoding/config.rs @@ -1,14 +1,15 @@ use crate::codecs::Transformer; use vector_lib::codecs::{ - encoding::{Framer, FramingConfig, Serializer, SerializerConfig}, CharacterDelimitedEncoder, LengthDelimitedEncoder, NewlineDelimitedEncoder, + encoding::{Framer, FramingConfig, Serializer, SerializerConfig}, }; use vector_lib::configurable::configurable_component; /// Encoding configuration. #[configurable_component] #[derive(Clone, Debug)] -#[configurable(description = "Configures how events are encoded into raw bytes.")] +/// Configures how events are encoded into raw bytes. +/// The selected encoding also determines which input types (logs, metrics, traces) are supported. pub struct EncodingConfig { #[serde(flatten)] encoding: SerializerConfig, @@ -155,7 +156,7 @@ where #[cfg(test)] mod test { - use vector_lib::lookup::lookup_v2::{parse_value_path, ConfigValuePath}; + use vector_lib::lookup::lookup_v2::{ConfigValuePath, parse_value_path}; use super::*; use crate::codecs::encoding::TimestampFormat; diff --git a/src/codecs/encoding/encoder.rs b/src/codecs/encoding/encoder.rs index e461e6834f..e5d9875ef6 100644 --- a/src/codecs/encoding/encoder.rs +++ b/src/codecs/encoding/encoder.rs @@ -1,8 +1,8 @@ use bytes::BytesMut; use tokio_util::codec::Encoder as _; use vector_lib::codecs::{ - encoding::{Error, Framer, Serializer}, CharacterDelimitedEncoder, NewlineDelimitedEncoder, TextSerializerConfig, + encoding::{Error, Framer, Serializer}, }; use crate::{ @@ -93,12 +93,14 @@ impl Encoder { } /// Get the suffix that encloses a batch of events. - pub const fn batch_suffix(&self) -> &[u8] { - match (&self.framer, &self.serializer) { + pub const fn batch_suffix(&self, empty: bool) -> &[u8] { + match (&self.framer, &self.serializer, empty) { ( Framer::CharacterDelimited(CharacterDelimitedEncoder { delimiter: b',' }), Serializer::Json(_) | Serializer::NativeJson(_), + _, ) => b"]", + (Framer::NewlineDelimited(_), _, false) => b"\n", _ => &[], } } @@ -237,7 +239,7 @@ mod tests { fn encode(&mut self, _: (), dst: &mut BytesMut) -> Result<(), Self::Error> { self.0.encode((), dst)?; let result = if self.1 == self.2 { - Err(Box::new(std::io::Error::new(std::io::ErrorKind::Other, "error")) as _) + Err(Box::new(std::io::Error::other("error")) as _) } else { Ok(()) }; @@ -323,4 +325,23 @@ mod tests { let sink = framed.into_inner(); assert_eq!(sink, b"(foo)(bar)"); } + + #[tokio::test] + async fn test_encode_batch_newline() { + let encoder = Encoder::::new( + Framer::NewlineDelimited(NewlineDelimitedEncoder::default()), + TextSerializerConfig::default().build().into(), + ); + let source = futures::stream::iter(vec![ + Event::Log(LogEvent::from("bar")), + Event::Log(LogEvent::from("baz")), + Event::Log(LogEvent::from("bat")), + ]) + .map(Ok); + let sink: Vec = Vec::new(); + let mut framed = FramedWrite::new(sink, encoder); + source.forward(&mut framed).await.unwrap(); + let sink = framed.into_inner(); + assert_eq!(sink, b"bar\nbaz\nbat\n"); + } } diff --git a/src/codecs/encoding/transformer.rs b/src/codecs/encoding/transformer.rs index 5539df5dd3..10752a12a1 100644 --- a/src/codecs/encoding/transformer.rs +++ b/src/codecs/encoding/transformer.rs @@ -9,7 +9,7 @@ use serde::{Deserialize, Deserializer}; use vector_lib::configurable::configurable_component; use vector_lib::event::{LogEvent, MaybeAsLogMut}; use vector_lib::lookup::lookup_v2::ConfigValuePath; -use vector_lib::lookup::{event_path, PathPrefix}; +use vector_lib::lookup::{PathPrefix, event_path}; use vector_lib::schema::meaning; use vrl::path::OwnedValuePath; use vrl::value::Value; @@ -105,15 +105,12 @@ impl Transformer { only_fields: Option<&Vec>, except_fields: Option<&Vec>, ) -> crate::Result<()> { - if let (Some(only_fields), Some(except_fields)) = (only_fields, except_fields) { - if except_fields + if let (Some(only_fields), Some(except_fields)) = (only_fields, except_fields) + && except_fields .iter() .any(|f| only_fields.iter().any(|v| v == f)) - { - return Err( - "`except_fields` and `only_fields` should be mutually exclusive.".into(), - ); - } + { + return Err("`except_fields` and `only_fields` should be mutually exclusive.".into()); } Ok(()) } @@ -168,11 +165,11 @@ impl Transformer { .meaning_path(meaning::SERVICE); // If we are removing the service field we need to store this in a `dropped_fields` list as we may need to // refer to this later when emitting metrics. - if let (Some(v), Some(service_path)) = (value, service_path) { - if service_path.path == *value_path { - log.metadata_mut() - .add_dropped_field(meaning::SERVICE.into(), v); - } + if let (Some(v), Some(service_path)) = (value, service_path) + && service_path.path == *value_path + { + log.metadata_mut() + .add_dropped_field(meaning::SERVICE.into(), v); } } } @@ -267,7 +264,7 @@ pub enum TimestampFormat { mod tests { use indoc::indoc; use vector_lib::btreemap; - use vector_lib::config::{log_schema, LogNamespace}; + use vector_lib::config::{LogNamespace, log_schema}; use vector_lib::lookup::path::parse_target_path; use vrl::value::Kind; @@ -396,7 +393,7 @@ mod tests { ), ]; for (fmt, expected) in cases { - let config: String = format!(r#"timestamp_format = "{}""#, fmt); + let config: String = format!(r#"timestamp_format = "{fmt}""#); let transformer: Transformer = toml::from_str(&config).unwrap(); let mut event = base.clone(); transformer.transform(&mut event); diff --git a/src/codecs/ready_frames.rs b/src/codecs/ready_frames.rs index 4aed021b57..773b979fea 100644 --- a/src/codecs/ready_frames.rs +++ b/src/codecs/ready_frames.rs @@ -52,7 +52,7 @@ where } /// Returns a mutable reference to the underlying stream. - pub fn get_mut(&mut self) -> &mut T { + pub const fn get_mut(&mut self) -> &mut T { &mut self.inner } @@ -115,7 +115,7 @@ where #[cfg(test)] mod test { - use futures::{channel::mpsc, poll, task::Poll, SinkExt, StreamExt}; + use futures::{SinkExt, StreamExt, channel::mpsc, poll, task::Poll}; use super::ReadyFrames; diff --git a/src/common/backoff.rs b/src/common/backoff.rs new file mode 100644 index 0000000000..cbf9e275cb --- /dev/null +++ b/src/common/backoff.rs @@ -0,0 +1,81 @@ +use std::time::Duration; + +// `tokio-retry` crate +// MIT License +// Copyright (c) 2017 Sam Rijs +// +/// A retry strategy driven by exponential back-off. +/// +/// The power corresponds to the number of past attempts. +#[derive(Debug, Clone)] +pub(crate) struct ExponentialBackoff { + current: u64, + base: u64, + factor: u64, + max_delay: Option, +} + +impl ExponentialBackoff { + /// Constructs a new exponential back-off strategy, + /// given a base duration in milliseconds. + /// + /// The resulting duration is calculated by taking the base to the `n`-th power, + /// where `n` denotes the number of past attempts. + pub(crate) const fn from_millis(base: u64) -> ExponentialBackoff { + ExponentialBackoff { + current: base, + base, + factor: 1u64, + max_delay: None, + } + } + + /// A multiplicative factor that will be applied to the retry delay. + /// + /// For example, using a factor of `1000` will make each delay in units of seconds. + /// + /// Default factor is `1`. + pub(crate) const fn factor(mut self, factor: u64) -> ExponentialBackoff { + self.factor = factor; + self + } + + /// Apply a maximum delay. No retry delay will be longer than this `Duration`. + pub(crate) const fn max_delay(mut self, duration: Duration) -> ExponentialBackoff { + self.max_delay = Some(duration); + self + } + + /// Resents the exponential back-off strategy to its initial state. + pub(crate) const fn reset(&mut self) { + self.current = self.base; + } +} + +impl Iterator for ExponentialBackoff { + type Item = Duration; + + fn next(&mut self) -> Option { + // set delay duration by applying factor + let duration = if let Some(duration) = self.current.checked_mul(self.factor) { + Duration::from_millis(duration) + } else { + Duration::from_millis(u64::MAX) + }; + + // check if we reached max delay + if let Some(ref max_delay) = self.max_delay + && duration > *max_delay + { + return Some(*max_delay); + } + + if let Some(next) = self.current.checked_mul(self.base) { + self.current = next; + } else { + self.current = u64::MAX; + } + + Some(duration) + } +} diff --git a/src/common/datadog.rs b/src/common/datadog.rs index 93f49dab0a..9b84a2f835 100644 --- a/src/common/datadog.rs +++ b/src/common/datadog.rs @@ -17,6 +17,8 @@ pub(crate) const DD_EU_SITE: &str = "datadoghq.eu"; /// The datadog tags event path. pub const DDTAGS: &str = "ddtags"; +/// The datadog message event path. +pub const MESSAGE: &str = "message"; /// Mapping of the semantic meaning of well known Datadog reserved attributes /// to the field name that Datadog intake expects. @@ -30,6 +32,13 @@ pub const DD_RESERVED_SEMANTIC_ATTRS: [(&str, &str); 6] = [ (meaning::TAGS, DDTAGS), ]; +/// Returns true if the parameter `attr` is one of the reserved Datadog log attributes +pub fn is_reserved_attribute(attr: &str) -> bool { + DD_RESERVED_SEMANTIC_ATTRS + .iter() + .any(|(_, attr_str)| &attr == attr_str) +} + /// DatadogSeriesMetric #[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] pub struct DatadogSeriesMetric { @@ -84,7 +93,7 @@ pub struct DatadogPoint(pub i64, pub T); /// /// If `endpoint` is not specified, we fallback to `site`. pub(crate) fn get_api_base_endpoint(endpoint: Option<&str>, site: &str) -> String { - endpoint.map_or_else(|| format!("https://api.{}", site), compute_api_endpoint) + endpoint.map_or_else(|| format!("https://api.{site}"), compute_api_endpoint) } /// Computes the Datadog API endpoint from a given endpoint string. diff --git a/src/common/expansion.rs b/src/common/expansion.rs index 11d6940183..04c737498f 100644 --- a/src/common/expansion.rs +++ b/src/common/expansion.rs @@ -25,7 +25,7 @@ pub(crate) fn pair_expansion( // key_* -> key_one, key_two, key_three // * -> one, two, three for (k, v) in output { - let key = slugify_text(&format!("{}{}", opening_prefix, k)); + let key = slugify_text(&format!("{opening_prefix}{k}")); let val = Value::from(v).to_string_lossy().into_owned(); if val == "" { warn!("Encountered \"null\" value for dynamic pair. key: {}", key); diff --git a/src/sources/util/http/error.rs b/src/common/http/error.rs similarity index 80% rename from src/sources/util/http/error.rs rename to src/common/http/error.rs index 06298558b2..8e03fc882b 100644 --- a/src/sources/util/http/error.rs +++ b/src/common/http/error.rs @@ -2,6 +2,7 @@ use std::{error::Error, fmt}; use serde::Serialize; +/// HTTP error, containing HTTP status code and a message #[derive(Serialize, Debug)] pub struct ErrorMessage { code: u16, @@ -15,6 +16,7 @@ pub struct ErrorMessage { feature = "sources-datadog_agent" ))] impl ErrorMessage { + /// Create a new `ErrorMessage` from HTTP status code and a message #[allow(unused)] // triggered by check-component-features pub fn new(code: http::StatusCode, message: String) -> Self { ErrorMessage { @@ -23,6 +25,7 @@ impl ErrorMessage { } } + /// Returns the HTTP status code #[allow(unused)] // triggered by check-component-features pub fn status_code(&self) -> http::StatusCode { http::StatusCode::from_u16(self.code).unwrap_or(http::StatusCode::INTERNAL_SERVER_ERROR) @@ -31,11 +34,13 @@ impl ErrorMessage { #[cfg(feature = "sources-utils-http-prelude")] impl ErrorMessage { + /// Returns the raw HTTP status code pub const fn code(&self) -> u16 { self.code } - pub fn message(&self) -> &str { + /// Returns the error message + pub const fn message(&self) -> &str { self.message.as_str() } } diff --git a/src/common/http/mod.rs b/src/common/http/mod.rs new file mode 100644 index 0000000000..8608f5e153 --- /dev/null +++ b/src/common/http/mod.rs @@ -0,0 +1,12 @@ +//! Common module between modules that use HTTP +#[cfg(all( + feature = "sources-utils-http-auth", + feature = "sources-utils-http-error" +))] +pub mod server_auth; + +#[cfg(feature = "sources-utils-http-error")] +mod error; + +#[cfg(feature = "sources-utils-http-error")] +pub use error::ErrorMessage; diff --git a/src/common/http/server_auth.rs b/src/common/http/server_auth.rs new file mode 100644 index 0000000000..4874418460 --- /dev/null +++ b/src/common/http/server_auth.rs @@ -0,0 +1,603 @@ +//! Shared authentication config between components that use HTTP. +use std::{collections::HashMap, fmt, net::SocketAddr}; + +use bytes::Bytes; +use headers::{Authorization, authorization::Credentials}; +use http::{HeaderMap, HeaderValue, StatusCode, header::AUTHORIZATION}; +use serde::{ + Deserialize, + de::{Error, MapAccess, Visitor}, +}; +use vector_config::configurable_component; +use vector_lib::{ + TimeZone, compile_vrl, + event::{Event, LogEvent, VrlTarget}, + sensitive_string::SensitiveString, +}; +use vrl::{ + compiler::{CompilationResult, CompileConfig, Program, runtime::Runtime}, + core::Value, + diagnostic::Formatter, + prelude::TypeState, + value::{KeyString, ObjectMap}, +}; + +use super::ErrorMessage; + +/// Configuration of the authentication strategy for server mode sinks and sources. +/// +/// Use the HTTP authentication with HTTPS only. The authentication credentials are passed as an +/// HTTP header without any additional encryption beyond what is provided by the transport itself. +#[configurable_component(no_deser)] +#[derive(Clone, Debug, Eq, PartialEq)] +#[configurable(metadata(docs::enum_tag_description = "The authentication strategy to use."))] +#[serde(tag = "strategy", rename_all = "snake_case")] +pub enum HttpServerAuthConfig { + /// Basic authentication. + /// + /// The username and password are concatenated and encoded using [base64][base64]. + /// + /// [base64]: https://en.wikipedia.org/wiki/Base64 + Basic { + /// The basic authentication username. + #[configurable(metadata(docs::examples = "${USERNAME}"))] + #[configurable(metadata(docs::examples = "username"))] + username: String, + + /// The basic authentication password. + #[configurable(metadata(docs::examples = "${PASSWORD}"))] + #[configurable(metadata(docs::examples = "password"))] + password: SensitiveString, + }, + + /// Custom authentication using VRL code. + /// + /// Takes in request and validates it using VRL code. + Custom { + /// The VRL boolean expression. + source: String, + }, +} + +// Custom deserializer implementation to default `strategy` to `basic` +impl<'de> Deserialize<'de> for HttpServerAuthConfig { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + struct HttpServerAuthConfigVisitor; + + const FIELD_KEYS: [&str; 4] = ["strategy", "username", "password", "source"]; + + impl<'de> Visitor<'de> for HttpServerAuthConfigVisitor { + type Value = HttpServerAuthConfig; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a valid authentication strategy (basic or custom)") + } + + fn visit_map(self, mut map: A) -> Result + where + A: MapAccess<'de>, + { + let mut fields: HashMap<&str, String> = HashMap::default(); + + while let Some(key) = map.next_key::()? { + if let Some(field_index) = FIELD_KEYS.iter().position(|k| *k == key.as_str()) { + if fields.contains_key(FIELD_KEYS[field_index]) { + return Err(Error::duplicate_field(FIELD_KEYS[field_index])); + } + fields.insert(FIELD_KEYS[field_index], map.next_value()?); + } else { + return Err(Error::unknown_field(&key, &FIELD_KEYS)); + } + } + + // Default to "basic" if strategy is missing + let strategy = fields + .get("strategy") + .map(String::as_str) + .unwrap_or_else(|| "basic"); + + match strategy { + "basic" => { + let username = fields + .remove("username") + .ok_or_else(|| Error::missing_field("username"))?; + let password = fields + .remove("password") + .ok_or_else(|| Error::missing_field("password"))?; + Ok(HttpServerAuthConfig::Basic { + username, + password: SensitiveString::from(password), + }) + } + "custom" => { + let source = fields + .remove("source") + .ok_or_else(|| Error::missing_field("source"))?; + Ok(HttpServerAuthConfig::Custom { source }) + } + _ => Err(Error::unknown_variant(strategy, &["basic", "custom"])), + } + } + } + + deserializer.deserialize_map(HttpServerAuthConfigVisitor) + } +} + +impl HttpServerAuthConfig { + /// Builds an auth matcher based on provided configuration. + /// Used to validate configuration if needed, before passing it to the + /// actual component for usage. + pub fn build( + &self, + enrichment_tables: &vector_lib::enrichment::TableRegistry, + ) -> crate::Result { + match self { + HttpServerAuthConfig::Basic { username, password } => { + Ok(HttpServerAuthMatcher::AuthHeader( + Authorization::basic(username, password.inner()).0.encode(), + "Invalid username/password", + )) + } + HttpServerAuthConfig::Custom { source } => { + let functions = vrl::stdlib::all() + .into_iter() + .chain(vector_lib::enrichment::vrl_functions()) + .chain(vector_vrl_functions::all()) + .collect::>(); + + let state = TypeState::default(); + + let mut config = CompileConfig::default(); + config.set_custom(enrichment_tables.clone()); + config.set_read_only(); + + let CompilationResult { + program, + warnings, + config: _, + } = compile_vrl(source, &functions, &state, config).map_err(|diagnostics| { + Formatter::new(source, diagnostics).colored().to_string() + })?; + + if !program.final_type_info().result.is_boolean() { + return Err("VRL conditions must return a boolean.".into()); + } + + if !warnings.is_empty() { + let warnings = Formatter::new(source, warnings).colored().to_string(); + warn!(message = "VRL compilation warning.", %warnings); + } + + Ok(HttpServerAuthMatcher::Vrl { program }) + } + } + } +} + +/// Built auth matcher with validated configuration +/// Can be used directly in a component to validate authentication in HTTP requests +#[allow(clippy::large_enum_variant)] +#[derive(Clone, Debug)] +pub enum HttpServerAuthMatcher { + /// Matcher for comparing exact value of Authorization header + AuthHeader(HeaderValue, &'static str), + /// Matcher for running VRL script for requests, to allow for custom validation + Vrl { + /// Compiled VRL script + program: Program, + }, +} + +impl HttpServerAuthMatcher { + /// Compares passed headers to the matcher + pub fn handle_auth( + &self, + address: Option<&SocketAddr>, + headers: &HeaderMap, + path: &str, + ) -> Result<(), ErrorMessage> { + match self { + HttpServerAuthMatcher::AuthHeader(expected, err_message) => { + if let Some(header) = headers.get(AUTHORIZATION) { + if expected == header { + Ok(()) + } else { + Err(ErrorMessage::new( + StatusCode::UNAUTHORIZED, + err_message.to_string(), + )) + } + } else { + Err(ErrorMessage::new( + StatusCode::UNAUTHORIZED, + "No authorization header".to_owned(), + )) + } + } + HttpServerAuthMatcher::Vrl { program } => { + self.handle_vrl_auth(address, headers, path, program) + } + } + } + + fn handle_vrl_auth( + &self, + address: Option<&SocketAddr>, + headers: &HeaderMap, + path: &str, + program: &Program, + ) -> Result<(), ErrorMessage> { + let mut target = VrlTarget::new( + Event::Log(LogEvent::from_map( + ObjectMap::from([ + ( + "headers".into(), + Value::Object( + headers + .iter() + .map(|(k, v)| { + ( + KeyString::from(k.to_string()), + Value::Bytes(Bytes::copy_from_slice(v.as_bytes())), + ) + }) + .collect::(), + ), + ), + ( + "address".into(), + address.map_or(Value::Null, |a| Value::from(a.ip().to_string())), + ), + ("path".into(), Value::from(path.to_owned())), + ]), + Default::default(), + )), + program.info(), + false, + ); + let timezone = TimeZone::default(); + + let result = Runtime::default().resolve(&mut target, program, &timezone); + match result.map_err(|e| { + warn!("Handling auth failed: {}", e); + ErrorMessage::new(StatusCode::UNAUTHORIZED, "Auth failed".to_owned()) + })? { + vrl::core::Value::Boolean(result) => { + if result { + Ok(()) + } else { + Err(ErrorMessage::new( + StatusCode::UNAUTHORIZED, + "Auth failed".to_owned(), + )) + } + } + _ => Err(ErrorMessage::new( + StatusCode::UNAUTHORIZED, + "Invalid return value".to_owned(), + )), + } + } +} + +#[cfg(test)] +mod tests { + use crate::test_util::{next_addr, random_string}; + use indoc::indoc; + + use super::*; + + impl HttpServerAuthMatcher { + fn auth_header(self) -> (HeaderValue, &'static str) { + match self { + HttpServerAuthMatcher::AuthHeader(header_value, error_message) => { + (header_value, error_message) + } + HttpServerAuthMatcher::Vrl { .. } => { + panic!("Expected HttpServerAuthMatcher::AuthHeader") + } + } + } + } + + #[test] + fn config_should_default_to_basic() { + let config: HttpServerAuthConfig = serde_yaml::from_str(indoc! { r#" + username: foo + password: bar + "# + }) + .unwrap(); + + if let HttpServerAuthConfig::Basic { username, password } = config { + assert_eq!(username, "foo"); + assert_eq!(password.inner(), "bar"); + } else { + panic!("Expected HttpServerAuthConfig::Basic"); + } + } + + #[test] + fn config_should_support_explicit_basic_strategy() { + let config: HttpServerAuthConfig = serde_yaml::from_str(indoc! { r#" + strategy: basic + username: foo + password: bar + "# + }) + .unwrap(); + + if let HttpServerAuthConfig::Basic { username, password } = config { + assert_eq!(username, "foo"); + assert_eq!(password.inner(), "bar"); + } else { + panic!("Expected HttpServerAuthConfig::Basic"); + } + } + + #[test] + fn config_should_support_custom_strategy() { + let config: HttpServerAuthConfig = serde_yaml::from_str(indoc! { r#" + strategy: custom + source: "true" + "# + }) + .unwrap(); + + assert!(matches!(config, HttpServerAuthConfig::Custom { .. })); + if let HttpServerAuthConfig::Custom { source } = config { + assert_eq!(source, "true"); + } else { + panic!("Expected HttpServerAuthConfig::Custom"); + } + } + + #[test] + fn build_basic_auth_should_always_work() { + let basic_auth = HttpServerAuthConfig::Basic { + username: random_string(16), + password: random_string(16).into(), + }; + + let matcher = basic_auth.build(&Default::default()); + + assert!(matcher.is_ok()); + assert!(matches!( + matcher.unwrap(), + HttpServerAuthMatcher::AuthHeader { .. } + )); + } + + #[test] + fn build_basic_auth_should_use_username_password_related_message() { + let basic_auth = HttpServerAuthConfig::Basic { + username: random_string(16), + password: random_string(16).into(), + }; + + let (_, error_message) = basic_auth.build(&Default::default()).unwrap().auth_header(); + assert_eq!("Invalid username/password", error_message); + } + + #[test] + fn build_basic_auth_should_use_encode_basic_header() { + let username = random_string(16); + let password = random_string(16); + let basic_auth = HttpServerAuthConfig::Basic { + username: username.clone(), + password: password.clone().into(), + }; + + let (header, _) = basic_auth.build(&Default::default()).unwrap().auth_header(); + assert_eq!( + Authorization::basic(&username, &password).0.encode(), + header + ); + } + + #[test] + fn build_custom_should_fail_on_invalid_source() { + let custom_auth = HttpServerAuthConfig::Custom { + source: "invalid VRL source".to_string(), + }; + + assert!(custom_auth.build(&Default::default()).is_err()); + } + + #[test] + fn build_custom_should_fail_on_non_boolean_return_type() { + let custom_auth = HttpServerAuthConfig::Custom { + source: indoc! {r#" + .success = true + . + "#} + .to_string(), + }; + + assert!(custom_auth.build(&Default::default()).is_err()); + } + + #[test] + fn build_custom_should_success_on_proper_source_with_boolean_return_type() { + let custom_auth = HttpServerAuthConfig::Custom { + source: indoc! {r#" + .headers.authorization == "Basic test" + "#} + .to_string(), + }; + + assert!(custom_auth.build(&Default::default()).is_ok()); + } + + #[test] + fn basic_auth_matcher_should_return_401_when_missing_auth_header() { + let basic_auth = HttpServerAuthConfig::Basic { + username: random_string(16), + password: random_string(16).into(), + }; + + let matcher = basic_auth.build(&Default::default()).unwrap(); + + let result = matcher.handle_auth(Some(&next_addr()), &HeaderMap::new(), "/"); + + assert!(result.is_err()); + let error = result.unwrap_err(); + assert_eq!(401, error.code()); + assert_eq!("No authorization header", error.message()); + } + + #[test] + fn basic_auth_matcher_should_return_401_and_with_wrong_credentials() { + let basic_auth = HttpServerAuthConfig::Basic { + username: random_string(16), + password: random_string(16).into(), + }; + + let matcher = basic_auth.build(&Default::default()).unwrap(); + + let mut headers = HeaderMap::new(); + headers.insert(AUTHORIZATION, HeaderValue::from_static("Basic wrong")); + let result = matcher.handle_auth(Some(&next_addr()), &headers, "/"); + + assert!(result.is_err()); + let error = result.unwrap_err(); + assert_eq!(401, error.code()); + assert_eq!("Invalid username/password", error.message()); + } + + #[test] + fn basic_auth_matcher_should_return_ok_for_correct_credentials() { + let username = random_string(16); + let password = random_string(16); + let basic_auth = HttpServerAuthConfig::Basic { + username: username.clone(), + password: password.clone().into(), + }; + + let matcher = basic_auth.build(&Default::default()).unwrap(); + + let mut headers = HeaderMap::new(); + headers.insert( + AUTHORIZATION, + Authorization::basic(&username, &password).0.encode(), + ); + let result = matcher.handle_auth(Some(&next_addr()), &headers, "/"); + + assert!(result.is_ok()); + } + + #[test] + fn custom_auth_matcher_should_return_ok_for_true_vrl_script_result() { + let custom_auth = HttpServerAuthConfig::Custom { + source: r#".headers.authorization == "test""#.to_string(), + }; + + let matcher = custom_auth.build(&Default::default()).unwrap(); + + let mut headers = HeaderMap::new(); + headers.insert(AUTHORIZATION, HeaderValue::from_static("test")); + let result = matcher.handle_auth(Some(&next_addr()), &headers, "/"); + + assert!(result.is_ok()); + } + + #[test] + fn custom_auth_matcher_should_be_able_to_check_address() { + let addr = next_addr(); + let addr_string = addr.ip().to_string(); + let custom_auth = HttpServerAuthConfig::Custom { + source: format!(".address == \"{addr_string}\""), + }; + + let matcher = custom_auth.build(&Default::default()).unwrap(); + + let headers = HeaderMap::new(); + let result = matcher.handle_auth(Some(&next_addr()), &headers, "/"); + + assert!(result.is_ok()); + } + + #[test] + fn custom_auth_matcher_should_work_with_missing_address_too() { + let addr = next_addr(); + let addr_string = addr.ip().to_string(); + let custom_auth = HttpServerAuthConfig::Custom { + source: format!(".address == \"{addr_string}\""), + }; + + let matcher = custom_auth.build(&Default::default()).unwrap(); + + let headers = HeaderMap::new(); + let result = matcher.handle_auth(None, &headers, "/"); + + assert!(result.is_err()); + } + + #[test] + fn custom_auth_matcher_should_be_able_to_check_path() { + let custom_auth = HttpServerAuthConfig::Custom { + source: r#".path == "/ok""#.to_string(), + }; + + let matcher = custom_auth.build(&Default::default()).unwrap(); + + let headers = HeaderMap::new(); + let result = matcher.handle_auth(Some(&next_addr()), &headers, "/ok"); + + assert!(result.is_ok()); + } + + #[test] + fn custom_auth_matcher_should_return_401_with_wrong_path() { + let custom_auth = HttpServerAuthConfig::Custom { + source: r#".path == "/ok""#.to_string(), + }; + + let matcher = custom_auth.build(&Default::default()).unwrap(); + + let headers = HeaderMap::new(); + let result = matcher.handle_auth(Some(&next_addr()), &headers, "/bad"); + + assert!(result.is_err()); + } + + #[test] + fn custom_auth_matcher_should_return_401_for_false_vrl_script_result() { + let custom_auth = HttpServerAuthConfig::Custom { + source: r#".headers.authorization == "test""#.to_string(), + }; + + let matcher = custom_auth.build(&Default::default()).unwrap(); + + let mut headers = HeaderMap::new(); + headers.insert(AUTHORIZATION, HeaderValue::from_static("wrong value")); + let result = matcher.handle_auth(Some(&next_addr()), &headers, "/"); + + assert!(result.is_err()); + let error = result.unwrap_err(); + assert_eq!(401, error.code()); + assert_eq!("Auth failed", error.message()); + } + + #[test] + fn custom_auth_matcher_should_return_401_for_failed_script_execution() { + let custom_auth = HttpServerAuthConfig::Custom { + source: "abort".to_string(), + }; + + let matcher = custom_auth.build(&Default::default()).unwrap(); + + let mut headers = HeaderMap::new(); + headers.insert(AUTHORIZATION, HeaderValue::from_static("test")); + let result = matcher.handle_auth(Some(&next_addr()), &headers, "/"); + + assert!(result.is_err()); + let error = result.unwrap_err(); + assert_eq!(401, error.code()); + assert_eq!("Auth failed", error.message()); + } +} diff --git a/src/common/mod.rs b/src/common/mod.rs index 6e29cbeebe..1c9a6eb45e 100644 --- a/src/common/mod.rs +++ b/src/common/mod.rs @@ -18,5 +18,19 @@ pub(crate) mod sqs; #[cfg(any(feature = "sources-aws_s3", feature = "sinks-aws_s3"))] pub(crate) mod s3; +#[cfg(any(feature = "sources-websocket", feature = "sinks-websocket"))] +pub(crate) mod websocket; + +pub(crate) mod backoff; +#[cfg(any(feature = "sources-mqtt", feature = "sinks-mqtt",))] +/// Common MQTT configuration shared by MQTT components. +pub mod mqtt; + #[cfg(any(feature = "transforms-log_to_metric", feature = "sinks-loki"))] pub(crate) mod expansion; + +#[cfg(any( + feature = "sources-utils-http-auth", + feature = "sources-utils-http-error" +))] +pub mod http; diff --git a/src/common/mqtt.rs b/src/common/mqtt.rs new file mode 100644 index 0000000000..c5e1190b0b --- /dev/null +++ b/src/common/mqtt.rs @@ -0,0 +1,132 @@ +use rumqttc::{AsyncClient, EventLoop, MqttOptions}; +use snafu::Snafu; +use vector_config_macros::configurable_component; +use vector_lib::tls::{TlsEnableableConfig, TlsError}; + +use crate::template::TemplateParseError; + +/// Shared MQTT configuration for sources and sinks. +#[configurable_component] +#[derive(Clone, Debug, Derivative)] +#[derivative(Default)] +#[serde(deny_unknown_fields)] +pub struct MqttCommonConfig { + /// MQTT server address (The broker’s domain name or IP address). + #[configurable(metadata(docs::examples = "mqtt.example.com", docs::examples = "127.0.0.1"))] + pub host: String, + + /// TCP port of the MQTT server to connect to. + #[configurable(derived)] + #[serde(default = "default_port")] + #[derivative(Default(value = "default_port()"))] + pub port: u16, + + /// MQTT username. + #[serde(default)] + #[configurable(derived)] + pub user: Option, + + /// MQTT password. + #[serde(default)] + #[configurable(derived)] + pub password: Option, + + /// MQTT client ID. + #[serde(default)] + #[configurable(derived)] + pub client_id: Option, + + /// Connection keep-alive interval. + #[serde(default = "default_keep_alive")] + #[derivative(Default(value = "default_keep_alive()"))] + pub keep_alive: u16, + + /// Maximum packet size + #[serde(default = "default_max_packet_size")] + #[derivative(Default(value = "default_max_packet_size()"))] + pub max_packet_size: usize, + + /// TLS configuration. + #[configurable(derived)] + pub tls: Option, +} + +const fn default_port() -> u16 { + 1883 +} + +const fn default_keep_alive() -> u16 { + 60 +} + +const fn default_max_packet_size() -> usize { + 10 * 1024 +} + +/// MQTT Error Types +#[derive(Debug, Snafu)] +#[snafu(visibility(pub))] +pub enum MqttError { + /// Topic template parsing failed + #[snafu(display("invalid topic template: {source}"))] + TopicTemplate { + /// Source of error + source: TemplateParseError, + }, + /// TLS error + #[snafu(display("TLS error: {source}"))] + Tls { + /// Source of error + source: TlsError, + }, + /// Configuration error + #[snafu(display("MQTT configuration error: {source}"))] + Configuration { + /// Source of error + source: ConfigurationError, + }, +} + +/// MQTT Configuration error types +#[derive(Clone, Debug, Eq, PartialEq, Snafu)] +pub enum ConfigurationError { + /// Empty client ID error + #[snafu(display("Client ID is not allowed to be empty."))] + EmptyClientId, + /// Invalid credentials provided error + #[snafu(display("Username and password must be either both provided or both missing."))] + InvalidCredentials, + /// Invalid client ID provied error + #[snafu(display( + "Client ID must be 1-23 characters long and must consist of only alphanumeric characters." + ))] + InvalidClientId, + /// Credentials provided were incomplete + #[snafu(display("Username and password must be either both or neither provided."))] + IncompleteCredentials, +} + +#[derive(Clone)] +/// Mqtt connector wrapper +pub struct MqttConnector { + /// Mqtt connection options + pub options: MqttOptions, +} + +impl MqttConnector { + /// Creates a new MqttConnector + pub const fn new(options: MqttOptions) -> Self { + Self { options } + } + + /// Connects the connector and generates a client and eventloop + pub fn connect(&self) -> (AsyncClient, EventLoop) { + let (client, eventloop) = AsyncClient::new(self.options.clone(), 1024); + (client, eventloop) + } + + /// TODO: Right now there is no way to implement the healthcheck properly: + pub async fn healthcheck(&self) -> crate::Result<()> { + Ok(()) + } +} diff --git a/src/common/websocket.rs b/src/common/websocket.rs new file mode 100644 index 0000000000..e46aa01221 --- /dev/null +++ b/src/common/websocket.rs @@ -0,0 +1,240 @@ +use std::{ + fmt::Debug, + net::SocketAddr, + num::NonZeroU64, + task::{Context, Poll}, + time::Duration, +}; +use vector_config_macros::configurable_component; + +use snafu::{ResultExt, Snafu}; +use tokio::{net::TcpStream, time}; +use tokio_tungstenite::{ + WebSocketStream, client_async_with_config, + tungstenite::{ + client::{IntoClientRequest, uri_mode}, + error::{Error as TungsteniteError, ProtocolError, UrlError}, + handshake::client::Request, + protocol::WebSocketConfig, + stream::Mode as UriMode, + }, +}; + +use crate::{ + common::backoff::ExponentialBackoff, + dns, + http::Auth, + internal_events::{WebSocketConnectionEstablished, WebSocketConnectionFailedError}, + tls::{MaybeTlsSettings, MaybeTlsStream, TlsEnableableConfig, TlsError}, +}; + +#[allow(unreachable_pub)] +#[derive(Debug, Snafu)] +#[snafu(visibility(pub))] +pub enum WebSocketError { + #[snafu(display("Creating WebSocket client failed: {}", source))] + CreateFailed { source: TungsteniteError }, + #[snafu(display("Connect error: {}", source))] + ConnectError { source: TlsError }, + #[snafu(display("Unable to resolve DNS: {}", source))] + DnsError { source: dns::DnsError }, + #[snafu(display("No addresses returned."))] + NoAddresses, +} + +#[derive(Clone)] +pub(crate) struct WebSocketConnector { + uri: String, + host: String, + port: u16, + tls: MaybeTlsSettings, + auth: Option, +} + +impl WebSocketConnector { + pub(crate) fn new( + uri: String, + tls: MaybeTlsSettings, + auth: Option, + ) -> Result { + let request = (&uri).into_client_request().context(CreateFailedSnafu)?; + let (host, port) = Self::extract_host_and_port(&request).context(CreateFailedSnafu)?; + + Ok(Self { + uri, + host, + port, + tls, + auth, + }) + } + + fn extract_host_and_port(request: &Request) -> Result<(String, u16), TungsteniteError> { + let host = request + .uri() + .host() + .ok_or(TungsteniteError::Url(UrlError::NoHostName))? + .to_string(); + let mode = uri_mode(request.uri())?; + let port = request.uri().port_u16().unwrap_or(match mode { + UriMode::Tls => 443, + UriMode::Plain => 80, + }); + + Ok((host, port)) + } + + const fn fresh_backoff() -> ExponentialBackoff { + ExponentialBackoff::from_millis(2) + .factor(250) + .max_delay(Duration::from_secs(60)) + } + + async fn tls_connect(&self) -> Result, WebSocketError> { + let ip = dns::Resolver + .lookup_ip(self.host.clone()) + .await + .context(DnsSnafu)? + .next() + .ok_or(WebSocketError::NoAddresses)?; + + let addr = SocketAddr::new(ip, self.port); + self.tls + .connect(&self.host, &addr) + .await + .context(ConnectSnafu) + } + + async fn connect(&self) -> Result>, WebSocketError> { + let mut request = (&self.uri) + .into_client_request() + .context(CreateFailedSnafu)?; + + if let Some(auth) = &self.auth { + auth.apply(&mut request); + } + + let maybe_tls = self.tls_connect().await?; + + let ws_config = WebSocketConfig::default(); + + let (ws_stream, _response) = client_async_with_config(request, maybe_tls, Some(ws_config)) + .await + .context(CreateFailedSnafu)?; + + Ok(ws_stream) + } + + pub(crate) async fn connect_backoff(&self) -> WebSocketStream> { + let mut backoff = Self::fresh_backoff(); + loop { + match self.connect().await { + Ok(ws_stream) => { + emit!(WebSocketConnectionEstablished {}); + return ws_stream; + } + Err(error) => { + emit!(WebSocketConnectionFailedError { + error: Box::new(error) + }); + time::sleep(backoff.next().unwrap()).await; + } + } + } + } + + #[cfg(feature = "sinks-websocket")] + pub(crate) async fn healthcheck(&self) -> crate::Result<()> { + self.connect().await.map(|_| ()).map_err(Into::into) + } +} + +pub(crate) const fn is_closed(error: &TungsteniteError) -> bool { + matches!( + error, + TungsteniteError::ConnectionClosed + | TungsteniteError::AlreadyClosed + | TungsteniteError::Protocol(ProtocolError::ResetWithoutClosingHandshake) + ) +} + +pub(crate) struct PingInterval { + interval: Option, +} + +impl PingInterval { + pub(crate) fn new(period: Option) -> Self { + Self { + interval: period.map(|period| time::interval(Duration::from_secs(period))), + } + } + + pub(crate) fn poll_tick(&mut self, cx: &mut Context<'_>) -> Poll { + match self.interval.as_mut() { + Some(interval) => interval.poll_tick(cx), + None => Poll::Pending, + } + } + + pub(crate) async fn tick(&mut self) -> time::Instant { + std::future::poll_fn(|cx| self.poll_tick(cx)).await + } +} + +/// Shared websocket configuration for sources and sinks. +#[configurable_component] +#[derive(Clone, Debug)] +#[serde(deny_unknown_fields)] +pub struct WebSocketCommonConfig { + /// The WebSocket URI to connect to. + /// + /// This should include the protocol and host, but can also include the port, path, and any other valid part of a URI. + /// **Note**: Using the `wss://` protocol requires enabling `tls`. + #[configurable(metadata(docs::examples = "ws://localhost:8080"))] + #[configurable(metadata(docs::examples = "wss://example.com/socket"))] + pub uri: String, + + /// The interval, in seconds, between sending [Ping][ping]s to the remote peer. + /// + /// If this option is not configured, pings are not sent on an interval. + /// + /// If the `ping_timeout` is not set, pings are still sent but there is no expectation of pong + /// response times. + /// + /// [ping]: https://www.rfc-editor.org/rfc/rfc6455#section-5.5.2 + #[configurable(metadata(docs::type_unit = "seconds"))] + #[configurable(metadata(docs::advanced))] + #[configurable(metadata(docs::examples = 30))] + pub ping_interval: Option, + + /// The number of seconds to wait for a [Pong][pong] response from the remote peer. + /// + /// If a response is not received within this time, the connection is re-established. + /// + /// [pong]: https://www.rfc-editor.org/rfc/rfc6455#section-5.5.3 + // NOTE: this option is not relevant if the `ping_interval` is not configured. + #[configurable(metadata(docs::type_unit = "seconds"))] + #[configurable(metadata(docs::advanced))] + #[configurable(metadata(docs::examples = 5))] + pub ping_timeout: Option, + + /// TLS configuration. + #[configurable(derived)] + pub tls: Option, + + /// HTTP Authentication. + #[configurable(derived)] + pub auth: Option, +} + +impl Default for WebSocketCommonConfig { + fn default() -> Self { + Self { + uri: "ws://127.0.0.1:8080".to_owned(), + ping_interval: None, + ping_timeout: None, + tls: None, + auth: None, + } + } +} diff --git a/src/components/validation/mod.rs b/src/components/validation/mod.rs index 514153ad2b..cd0b2e2715 100644 --- a/src/components/validation/mod.rs +++ b/src/components/validation/mod.rs @@ -319,7 +319,7 @@ fn run_validation(configuration: ValidationConfiguration, test_case_data_path: s } else { let formatted = success .iter() - .map(|s| format!(" - {}\n", s)) + .map(|s| format!(" - {s}\n")) .collect::>(); details.push(format!( @@ -340,7 +340,7 @@ fn run_validation(configuration: ValidationConfiguration, test_case_data_path: s } else { let formatted = failure .iter() - .map(|s| format!(" - {}\n", s)) + .map(|s| format!(" - {s}\n")) .collect::>(); details.push(format!( @@ -368,10 +368,9 @@ fn run_validation(configuration: ValidationConfiguration, test_case_data_path: s ); } } - Err(e) => panic!( - "Failed to complete validation run for component '{}': {}", - component_name, e - ), + Err(e) => { + panic!("Failed to complete validation run for component '{component_name}': {e}") + } } }); } @@ -437,7 +436,10 @@ fn get_validation_configuration_from_test_case_path( #[cfg(feature = "component-validation-runner")] pub fn validate_component(test_case_data_path: std::path::PathBuf) { if !test_case_data_path.exists() { - panic!("Component validation test invoked with path to test case data that could not be found: {}", test_case_data_path.to_string_lossy()); + panic!( + "Component validation test invoked with path to test case data that could not be found: {}", + test_case_data_path.to_string_lossy() + ); } let configuration = get_validation_configuration_from_test_case_path(&test_case_data_path) diff --git a/src/components/validation/resources/event.rs b/src/components/validation/resources/event.rs index b7a6e102ea..ddd4d72eda 100644 --- a/src/components/validation/resources/event.rs +++ b/src/components/validation/resources/event.rs @@ -3,14 +3,13 @@ use std::collections::HashMap; use bytes::BytesMut; use serde::Deserialize; use serde_json::Value; -use snafu::Snafu; use tokio_util::codec::Encoder as _; use vector_lib::codecs::encoding::format::JsonSerializerOptions; use crate::codecs::Encoder; use vector_lib::codecs::{ - encoding, JsonSerializer, LengthDelimitedEncoder, LogfmtSerializer, MetricTagValues, - NewlineDelimitedEncoder, + JsonSerializer, LengthDelimitedEncoder, LogfmtSerializer, MetricTagValues, + NewlineDelimitedEncoder, encoding, }; use vector_lib::event::{Event, LogEvent}; @@ -101,7 +100,7 @@ impl TestEvent { } } - pub fn get_event(&mut self) -> &mut Event { + pub const fn get_event(&mut self) -> &mut Event { match self { Self::Passthrough(event) => event, Self::FailWithAlternateEncoder(event) => event, @@ -136,9 +135,6 @@ impl TestEvent { } } -#[derive(Clone, Debug, Eq, PartialEq, Snafu)] -pub enum RawTestEventParseError {} - impl From for TestEvent { fn from(other: RawTestEvent) -> Self { match other { diff --git a/src/components/validation/resources/http.rs b/src/components/validation/resources/http.rs index 2bfc04deba..439f1766ca 100644 --- a/src/components/validation/resources/http.rs +++ b/src/components/validation/resources/http.rs @@ -7,30 +7,29 @@ use std::{ }; use axum::{ + Router, response::IntoResponse, routing::{MethodFilter, MethodRouter}, - Router, }; use bytes::{BufMut as _, BytesMut}; use http::{Method, Request, StatusCode, Uri}; use hyper::{Body, Client, Server}; use tokio::{ select, - sync::{mpsc, oneshot, Mutex, Notify}, + sync::{Mutex, Notify, mpsc, oneshot}, }; use tokio_util::codec::Decoder; use crate::components::validation::{ - sync::{Configuring, TaskCoordinator}, RunnerMetrics, + sync::{Configuring, TaskCoordinator}, }; use vector_lib::{ - codecs::encoding::Framer, codecs::encoding::Serializer::Json, - codecs::CharacterDelimitedEncoder, config::LogNamespace, event::Event, - EstimatedJsonEncodedSizeOf, + EstimatedJsonEncodedSizeOf, codecs::CharacterDelimitedEncoder, codecs::encoding::Framer, + codecs::encoding::Serializer::Json, config::LogNamespace, event::Event, }; -use super::{encode_test_event, ResourceCodec, ResourceDirection, TestEvent}; +use super::{ResourceCodec, ResourceDirection, TestEvent, encode_test_event}; /// An HTTP resource. #[derive(Clone)] @@ -113,16 +112,19 @@ fn spawn_input_http_server( async move { let mut sendable_events = sendable_events.lock().await; - if let Some(event) = sendable_events.pop_front() { - let mut buffer = BytesMut::new(); - encode_test_event(&mut encoder, &mut buffer, event); - - buffer.into_response() - } else { - // We'll send an empty 200 in the response since some - // sources throw errors for anything other than a valid - // response. - StatusCode::OK.into_response() + match sendable_events.pop_front() { + Some(event) => { + let mut buffer = BytesMut::new(); + encode_test_event(&mut encoder, &mut buffer, event); + + buffer.into_response() + } + _ => { + // We'll send an empty 200 in the response since some + // sources throw errors for anything other than a valid + // response. + StatusCode::OK.into_response() + } } } }, @@ -325,16 +327,25 @@ impl HttpResourceOutputContext<'_> { match hyper::body::to_bytes(request.into_body()).await { Err(_) => StatusCode::INTERNAL_SERVER_ERROR.into_response(), Ok(body) => { + let byte_size = body.len(); let mut body = BytesMut::from(&body[..]); loop { match decoder.decode_eof(&mut body) { - Ok(Some((events, byte_size))) => { + // `decoded_byte_size` is the decoded size of an individual frame. `byte_size` represents the size of the + // entire payload which may contain multiple frames and their delimiters. + Ok(Some((events, decoded_byte_size))) => { if should_reject { - info!("HTTP server external output resource decoded {byte_size} bytes but test case configured to reject."); + info!( + internal_log_rate_limit = true, + "HTTP server external output resource decoded {decoded_byte_size:?} bytes but test case configured to reject.", + ); } else { let mut output_runner_metrics = output_runner_metrics.lock().await; - info!("HTTP server external output resource decoded {byte_size} bytes."); + info!( + internal_log_rate_limit = true, + "HTTP server external output resource decoded {decoded_byte_size:?} bytes." + ); // Update the runner metrics for the received events. This will later // be used in the Validators, as the "expected" case. diff --git a/src/components/validation/resources/mod.rs b/src/components/validation/resources/mod.rs index 61d705c00e..c667e8c5b4 100644 --- a/src/components/validation/resources/mod.rs +++ b/src/components/validation/resources/mod.rs @@ -3,15 +3,15 @@ mod http; use std::sync::Arc; -use tokio::sync::{mpsc, Mutex}; +use tokio::sync::{Mutex, mpsc}; use vector_lib::{ codecs::{ + BytesEncoder, decoding::{self, DeserializerConfig}, encoding::{ self, Framer, FramingConfig, JsonSerializerConfig, SerializerConfig, TextSerializerConfig, }, - BytesEncoder, }, config::LogNamespace, }; @@ -19,13 +19,13 @@ use vector_lib::{config::DataType, event::Event}; use crate::codecs::{Decoder, DecodingConfig, Encoder, EncodingConfig, EncodingConfigWithFraming}; -pub use self::event::{encode_test_event, TestEvent}; +pub use self::event::{TestEvent, encode_test_event}; pub use self::http::HttpResourceConfig; use self::http::HttpResourceOutputContext; use super::{ - sync::{Configuring, TaskCoordinator}, RunnerMetrics, + sync::{Configuring, TaskCoordinator}, }; /// The codec used by the external resource. @@ -200,6 +200,13 @@ fn decoder_framing_to_encoding_framer(framing: &decoding::FramingConfig) -> enco decoding::FramingConfig::OctetCounting(_) => todo!(), // TODO: chunked gelf is not supported yet in encoding decoding::FramingConfig::ChunkedGelf(_) => todo!(), + decoding::FramingConfig::VarintLengthDelimited(config) => { + encoding::FramingConfig::VarintLengthDelimited( + encoding::VarintLengthDelimitedEncoderConfig { + max_frame_length: config.max_frame_length, + }, + ) + } }; framing_config.build() @@ -250,6 +257,13 @@ fn encoder_framing_to_decoding_framer(framing: encoding::FramingConfig) -> decod encoding::FramingConfig::NewlineDelimited => { decoding::FramingConfig::NewlineDelimited(Default::default()) } + vector_lib::codecs::encoding::FramingConfig::VarintLengthDelimited(config) => { + decoding::FramingConfig::VarintLengthDelimited( + decoding::VarintLengthDelimitedDecoderConfig { + max_frame_length: config.max_frame_length, + }, + ) + } }; framing_config.build() diff --git a/src/components/validation/runner/config.rs b/src/components/validation/runner/config.rs index 5f70f56670..1dca540bb1 100644 --- a/src/components/validation/runner/config.rs +++ b/src/components/validation/runner/config.rs @@ -2,10 +2,10 @@ use vector_lib::config::LogNamespace; use crate::{ components::validation::{ + ComponentConfiguration, ComponentType, ValidationConfiguration, component_names::*, sync::{Configuring, TaskCoordinator}, util::GrpcAddress, - ComponentConfiguration, ComponentType, ValidationConfiguration, }, config::{BoxedSink, BoxedSource, BoxedTransform, ConfigBuilder}, sinks::vector::VectorConfig as VectorSinkConfig, @@ -33,8 +33,7 @@ impl TopologyBuilder { let component_configuration = configuration .component_configuration_for_test_case(config_name) .ok_or(format!( - "No test case name defined for configuration {:?}.", - config_name + "No test case name defined for configuration {config_name:?}." ))?; Ok(match component_configuration { diff --git a/src/components/validation/runner/io.rs b/src/components/validation/runner/io.rs index bc06761b17..94607e61aa 100644 --- a/src/components/validation/runner/io.rs +++ b/src/components/validation/runner/io.rs @@ -4,10 +4,10 @@ use http::{Request, Response}; use hyper::Body; use tokio::{pin, select, sync::mpsc}; use tonic::{ + Status, body::BoxBody, server::NamedService, transport::{Channel, Endpoint}, - Status, }; use tower::Service; use vector_lib::shutdown::ShutdownSignal; @@ -15,9 +15,9 @@ use vector_lib::{event::Event, tls::MaybeTlsSettings}; use crate::{ components::validation::{ + TestEvent, sync::{Configuring, TaskCoordinator}, util::GrpcAddress, - TestEvent, }, proto::vector::{ Client as VectorClient, HealthCheckRequest, HealthCheckResponse, PushEventsRequest, diff --git a/src/components/validation/runner/mod.rs b/src/components/validation/runner/mod.rs index 8c5937bbf8..a7ed5c6f06 100644 --- a/src/components/validation/runner/mod.rs +++ b/src/components/validation/runner/mod.rs @@ -10,15 +10,15 @@ use tokio::{ runtime::Builder, select, sync::{ - mpsc::{self, Receiver, Sender}, Mutex, + mpsc::{self, Receiver, Sender}, }, task::JoinHandle, }; use tokio_util::codec::Encoder as _; use vector_lib::{ - codecs::encoding, config::LogNamespace, event::Event, EstimatedJsonEncodedSizeOf, + EstimatedJsonEncodedSizeOf, codecs::encoding, config::LogNamespace, event::Event, }; use crate::{ @@ -30,9 +30,9 @@ use crate::{ }; use super::{ + ComponentType, TestCaseExpectation, TestEvent, ValidationConfiguration, Validator, encode_test_event, sync::{Configuring, TaskCoordinator}, - ComponentType, TestCaseExpectation, TestEvent, ValidationConfiguration, Validator, }; pub use self::config::TopologyBuilder; @@ -70,8 +70,12 @@ impl RunnerInput { controlled_edge: Option>, ) -> mpsc::Sender { match (self, controlled_edge) { - (Self::External(_), Some(_)) => panic!("Runner input declared as external resource, but controlled input edge was also specified."), - (Self::Controlled, None) => panic!("Runner input declared as controlled, but no controlled input edge was specified."), + (Self::External(_), Some(_)) => panic!( + "Runner input declared as external resource, but controlled input edge was also specified." + ), + (Self::Controlled, None) => panic!( + "Runner input declared as controlled, but no controlled input edge was specified." + ), (Self::External(tx), None) => tx, (Self::Controlled, Some(tx)) => tx, } @@ -113,8 +117,12 @@ impl RunnerOutput { controlled_edge: Option>>, ) -> mpsc::Receiver> { match (self, controlled_edge) { - (Self::External(_), Some(_)) => panic!("Runner output declared as external resource, but controlled output edge was also specified."), - (Self::Controlled, None) => panic!("Runner output declared as controlled, but no controlled output edge was specified."), + (Self::External(_), Some(_)) => panic!( + "Runner output declared as external resource, but controlled output edge was also specified." + ), + (Self::Controlled, None) => panic!( + "Runner output declared as controlled, but no controlled output edge was specified." + ), (Self::External(rx), None) => rx, (Self::Controlled, Some(rx)) => rx, } @@ -188,13 +196,11 @@ impl Runner { .insert(validator_name.to_string(), validator) .is_some() { - panic!( - "attempted to add duplicate validator '{}' to runner", - validator_name - ); + panic!("attempted to add duplicate validator '{validator_name}' to runner"); } } + #[allow(clippy::print_stdout)] pub async fn run_validation(self) -> Result, vector_lib::Error> { // Initialize our test environment. initialize_test_environment(); @@ -205,8 +211,8 @@ impl Runner { let test_cases = load_component_test_cases(&self.test_case_data_path)?; for test_case in test_cases { - println!(""); - println!(""); + println!(); + println!(); info!( "Running test '{}' case for component '{}' (type: {:?})...", test_case.name, @@ -432,18 +438,10 @@ impl Runner { /// returned explaining the cause. fn load_component_test_cases(test_case_data_path: &PathBuf) -> Result, String> { std::fs::File::open(test_case_data_path) - .map_err(|e| { - format!( - "I/O error during open of component validation test cases file: {}", - e - ) - }) + .map_err(|e| format!("I/O error during open of component validation test cases file: {e}")) .and_then(|file| { serde_yaml::from_reader(file).map_err(|e| { - format!( - "Deserialization error for component validation test cases file: {}", - e - ) + format!("Deserialization error for component validation test cases file: {e}") }) }) } @@ -589,10 +587,10 @@ fn spawn_input_driver( // the controlled edge (vector source) adds metadata to the event when it is received. // thus we need to add it here so the expected values for the comparisons on transforms // and sinks are accurate. - if component_type != ComponentType::Source { - if let Event::Log(ref mut log) = input_event.get_event() { - log_namespace.insert_standard_vector_source_metadata(log, "vector", now); - } + if component_type != ComponentType::Source + && let Event::Log(log) = input_event.get_event() + { + log_namespace.insert_standard_vector_source_metadata(log, "vector", now); } let (failure_case, mut event) = input_event.clone().get(); @@ -622,19 +620,18 @@ fn spawn_input_driver( // For example, the `datadog_agent` source. This only takes effect when // the test case YAML file defining the event, constructs it with the log // builder variant, and specifies an integer in milliseconds for the timestamp. - if component_type == ComponentType::Source { - if let Event::Log(ref mut log) = event { - if let Some(ts) = log.remove_timestamp() { - let ts = match ts.as_integer() { - Some(ts) => chrono::DateTime::from_timestamp_millis(ts) - .expect(&format!("invalid timestamp in input test event {ts}")) - .into(), - None => ts, - }; - log.parse_path_and_insert("timestamp", ts) - .expect("failed to insert timestamp"); - } - } + if component_type == ComponentType::Source + && let Event::Log(ref mut log) = event + && let Some(ts) = log.remove_timestamp() + { + let ts = match ts.as_integer() { + Some(ts) => chrono::DateTime::from_timestamp_millis(ts) + .unwrap_or_else(|| panic!("invalid timestamp in input test event {ts}")) + .into(), + None => ts, + }; + log.parse_path_and_insert("timestamp", ts) + .expect("failed to insert timestamp"); } // This particular metric is tricky because a component can run the diff --git a/src/components/validation/runner/telemetry.rs b/src/components/validation/runner/telemetry.rs index 83c12c02db..37830099d2 100644 --- a/src/components/validation/runner/telemetry.rs +++ b/src/components/validation/runner/telemetry.rs @@ -17,7 +17,7 @@ use crate::{ test_util::next_addr, }; -use super::io::{spawn_grpc_server, EventForwardService}; +use super::io::{EventForwardService, spawn_grpc_server}; const INTERNAL_LOGS_KEY: &str = "_telemetry_logs"; const INTERNAL_METRICS_KEY: &str = "_telemetry_metrics"; diff --git a/src/components/validation/sync.rs b/src/components/validation/sync.rs index 92f359af6a..694e8cb98b 100644 --- a/src/components/validation/sync.rs +++ b/src/components/validation/sync.rs @@ -1,9 +1,9 @@ use std::sync::{ - atomic::{AtomicUsize, Ordering}, Arc, Mutex, + atomic::{AtomicUsize, Ordering}, }; -use tokio::sync::{oneshot, Notify}; +use tokio::sync::{Notify, oneshot}; struct WaitGroupState { registered: AtomicUsize, diff --git a/src/components/validation/validators/component_spec/mod.rs b/src/components/validation/validators/component_spec/mod.rs index 98db84a1ce..68aa3d287d 100644 --- a/src/components/validation/validators/component_spec/mod.rs +++ b/src/components/validation/validators/component_spec/mod.rs @@ -1,5 +1,5 @@ use crate::components::validation::{ - component_names::*, ComponentType, RunnerMetrics, TestCaseExpectation, TestEvent, + ComponentType, RunnerMetrics, TestCaseExpectation, TestEvent, component_names::*, }; use vector_lib::event::{Event, Metric, MetricKind}; @@ -129,11 +129,7 @@ fn validate_telemetry( } }); - if errs.is_empty() { - Ok(out) - } else { - Err(errs) - } + if errs.is_empty() { Ok(out) } else { Err(errs) } } fn validate_metric( @@ -224,10 +220,10 @@ fn filter_events_by_metric_and_component<'a>( .filter(|&m| { if m.name() == metric.to_string() { debug!("{}", m); - if let Some(tags) = m.tags() { - if tags.get("component_id").unwrap_or("") == component_id { - return true; - } + if let Some(tags) = m.tags() + && tags.get("component_id").unwrap_or("") == component_id + { + return true; } } @@ -256,7 +252,7 @@ fn sum_counters( sum += *value; } } - _ => errs.push(format!("{}: metric value is not a counter", metric_name,)), + _ => errs.push(format!("{metric_name}: metric value is not a counter",)), } } diff --git a/src/conditions/datadog_search.rs b/src/conditions/datadog_search.rs index 87bf97affe..d506e8b7f2 100644 --- a/src/conditions/datadog_search.rs +++ b/src/conditions/datadog_search.rs @@ -5,7 +5,7 @@ use bytes::Bytes; use vector_lib::configurable::configurable_component; use vector_lib::event::{Event, LogEvent, Value}; use vrl::datadog_filter::regex::{wildcard_regex, word_regex}; -use vrl::datadog_filter::{build_matcher, Filter, Matcher, Resolver, Run}; +use vrl::datadog_filter::{Filter, Matcher, Resolver, Run, build_matcher}; use vrl::datadog_search_syntax::{Comparison, ComparisonValue, Field, QueryNode}; use super::{Condition, Conditional, ConditionalConfig}; @@ -26,6 +26,12 @@ impl Default for DatadogSearchConfig { } } +impl DatadogSearchConfig { + pub fn build_matcher(&self) -> crate::Result>> { + Ok(as_log(build_matcher(&self.source, &EventFilter)?)) + } +} + impl FromStr for DatadogSearchConfig { type Err = ::Err; fn from_str(s: &str) -> Result { @@ -48,10 +54,22 @@ pub struct DatadogSearchRunner { matcher: Box>, } +impl TryFrom<&DatadogSearchConfig> for DatadogSearchRunner { + type Error = crate::Error; + fn try_from(config: &DatadogSearchConfig) -> Result { + config.build_matcher().map(|matcher| Self { matcher }) + } +} + +impl DatadogSearchRunner { + pub fn matches(&self, event: &Event) -> bool { + self.matcher.run(event) + } +} + impl Conditional for DatadogSearchRunner { - fn check(&self, e: Event) -> (bool, Event) { - let result = self.matcher.run(&e); - (result, e) + fn check(&self, event: Event) -> (bool, Event) { + (self.matches(&event), event) } } @@ -60,9 +78,7 @@ impl ConditionalConfig for DatadogSearchConfig { &self, _enrichment_tables: &vector_lib::enrichment::TableRegistry, ) -> crate::Result { - let matcher = as_log(build_matcher(&self.source, &EventFilter).map_err(|e| e.to_string())?); - - Ok(Condition::DatadogSearch(DatadogSearchRunner { matcher })) + Ok(Condition::DatadogSearch(self.try_into()?)) } } @@ -84,24 +100,22 @@ impl Filter for EventFilter { fn exists(&self, field: Field) -> Result>, PathParseError> { Ok(match field { Field::Tag(tag) => { - let starts_with = format!("{}:", tag); + let starts_with = format!("{tag}:"); - any_string_match("tags", move |value| { + any_string_match_multiple(vec!["ddtags", "tags"], move |value| { value == tag || value.starts_with(&starts_with) }) } // Literal field 'tags' needs to be compared by key. Field::Reserved(field) if field == "tags" => { - any_string_match("tags", move |value| value == field) + any_string_match_multiple(vec!["ddtags", "tags"], move |value| value == field) } - Field::Default(f) | Field::Attribute(f) | Field::Reserved(f) => { - Run::boxed(move |log: &LogEvent| { - log.parse_path_and_get_value(f.as_str()) - .ok() - .flatten() - .is_some() - }) + // A literal "source" field should string match in "source" and "ddsource" fields (OR condition). + Field::Reserved(field) if field == "source" => { + exists_match_multiple(vec!["ddsource", "source"]) } + + Field::Default(f) | Field::Attribute(f) | Field::Reserved(f) => exists_match(f), }) } @@ -121,15 +135,23 @@ impl Filter for EventFilter { Field::Reserved(field) if field == "tags" => { let to_match = to_match.to_owned(); - array_match(field, move |values| { + array_match_multiple(vec!["ddtags", "tags"], move |values| { values.contains(&Value::Bytes(Bytes::copy_from_slice(to_match.as_bytes()))) }) } // Individual tags are compared by element key:value. Field::Tag(tag) => { - let value_bytes = Value::Bytes(format!("{}:{}", tag, to_match).into()); + let value_bytes = Value::Bytes(format!("{tag}:{to_match}").into()); - array_match("tags", move |values| values.contains(&value_bytes)) + array_match_multiple(vec!["ddtags", "tags"], move |values| { + values.contains(&value_bytes) + }) + } + // A literal "source" field should string match in "source" and "ddsource" fields (OR condition). + Field::Reserved(field) if field == "source" => { + let to_match = to_match.to_owned(); + + string_match_multiple(vec!["ddsource", "source"], move |value| value == to_match) } // Reserved values are matched by string equality. Field::Reserved(field) => { @@ -154,16 +176,27 @@ impl Filter for EventFilter { Ok(match field { // Default fields are matched by word boundary. Field::Default(field) => { - let re = word_regex(&format!("{}*", prefix)); + let re = word_regex(&format!("{prefix}*")); string_match(field, move |value| re.is_match(&value)) } // Tags are recursed until a match is found. Field::Tag(tag) => { - let starts_with = format!("{}:{}", tag, prefix); + let starts_with = format!("{tag}:{prefix}"); - any_string_match("tags", move |value| value.starts_with(&starts_with)) + any_string_match_multiple(vec!["ddtags", "tags"], move |value| { + value.starts_with(&starts_with) + }) } + // A literal "source" field should string match in "source" and "ddsource" fields (OR condition). + Field::Reserved(field) if field == "source" => { + let prefix = prefix.to_owned(); + + string_match_multiple(vec!["ddsource", "source"], move |value| { + value.starts_with(&prefix) + }) + } + // All other field types are compared by complete value. Field::Reserved(field) | Field::Attribute(field) => { let prefix = prefix.to_owned(); @@ -185,9 +218,15 @@ impl Filter for EventFilter { string_match(field, move |value| re.is_match(&value)) } Field::Tag(tag) => { - let re = wildcard_regex(&format!("{}:{}", tag, wildcard)); + let re = wildcard_regex(&format!("{tag}:{wildcard}")); - any_string_match("tags", move |value| re.is_match(&value)) + any_string_match_multiple(vec!["ddtags", "tags"], move |value| re.is_match(&value)) + } + // A literal "source" field should string match in "source" and "ddsource" fields (OR condition). + Field::Reserved(field) if field == "source" => { + let re = wildcard_regex(wildcard); + + string_match_multiple(vec!["ddsource", "source"], move |value| re.is_match(&value)) } Field::Reserved(field) | Field::Attribute(field) => { let re = wildcard_regex(wildcard); @@ -277,19 +316,30 @@ impl Filter for EventFilter { }) } // Tag values need extracting by "key:value" to be compared. - Field::Tag(tag) => any_string_match("tags", move |value| match value.split_once(':') { - Some((t, lhs)) if t == tag => { - let lhs = Cow::from(lhs); - - match comparator { - Comparison::Lt => lhs < rhs, - Comparison::Lte => lhs <= rhs, - Comparison::Gt => lhs > rhs, - Comparison::Gte => lhs >= rhs, + Field::Tag(tag) => any_string_match_multiple(vec!["ddtags", "tags"], move |value| { + match value.split_once(':') { + Some((t, lhs)) if t == tag => { + let lhs = Cow::from(lhs); + + match comparator { + Comparison::Lt => lhs < rhs, + Comparison::Lte => lhs <= rhs, + Comparison::Gt => lhs > rhs, + Comparison::Gte => lhs >= rhs, + } } + _ => false, } - _ => false, }), + // A literal "source" field should string match in "source" and "ddsource" fields (OR condition). + Field::Reserved(field) if field == "source" => { + string_match_multiple(vec!["ddsource", "source"], move |lhs| match comparator { + Comparison::Lt => lhs < rhs, + Comparison::Lte => lhs <= rhs, + Comparison::Gt => lhs > rhs, + Comparison::Gte => lhs >= rhs, + }) + } // All other tag types are compared by string. Field::Default(field) | Field::Reserved(field) => { string_match(field, move |lhs| match comparator { @@ -303,6 +353,21 @@ impl Filter for EventFilter { } } +// Returns a `Matcher` that returns true if the field exists. +fn exists_match(field: S) -> Box> +where + S: Into, +{ + let field = field.into(); + + Run::boxed(move |log: &LogEvent| { + log.parse_path_and_get_value(field.as_str()) + .ok() + .flatten() + .is_some() + }) +} + /// Returns a `Matcher` that returns true if the field resolves to a string, /// numeric, or boolean which matches the provided `func`. fn simple_scalar_match(field: S, func: F) -> Box> @@ -340,46 +405,71 @@ where }) } -/// Returns a `Matcher` that returns true if the log event resolves to an array, where -/// the vector of `Value`s the array contains matches the provided `func`. -fn array_match(field: S, func: F) -> Box> +// Returns a `Matcher` that returns true if any provided field exists. +fn exists_match_multiple(fields: Vec) -> Box> where - S: Into, - F: Fn(&Vec) -> bool + Send + Sync + Clone + 'static, + S: Into + Clone + Send + Sync + 'static, { - let field = field.into(); - Run::boxed(move |log: &LogEvent| { - match log.parse_path_and_get_value(field.as_str()).ok().flatten() { - Some(Value::Array(values)) => func(values), - _ => false, - } + fields + .iter() + .any(|field| exists_match(field.clone()).run(log)) }) } -/// Returns a `Matcher` that returns true if the log event resolves to an array, where -/// at least one `Value` it contains matches the provided `func`. -fn any_match(field: S, func: F) -> Box> +/// Returns a `Matcher` that returns true if any provided field resolves to a string which +/// matches the provided `func`. +fn string_match_multiple(fields: Vec, func: F) -> Box> where - S: Into, - F: Fn(&Value) -> bool + Send + Sync + Clone + 'static, + S: Into + Clone + Send + Sync + 'static, + F: Fn(Cow) -> bool + Send + Sync + Clone + 'static, { - array_match(field, move |values| values.iter().any(&func)) + Run::boxed(move |log: &LogEvent| { + fields + .iter() + .any(|field| string_match(field.clone(), func.clone()).run(log)) + }) } -/// Returns a `Matcher` that returns true if the log event resolves to an array of strings, -/// where at least one string matches the provided `func`. -fn any_string_match(field: S, func: F) -> Box> +fn any_string_match_multiple(fields: Vec, func: F) -> Box> where - S: Into, + S: Into + Clone + Send + Sync + 'static, F: Fn(Cow) -> bool + Send + Sync + Clone + 'static, { - any_match(field, move |value| { + any_match_multiple(fields, move |value| { let bytes = value.coerce_to_bytes(); func(String::from_utf8_lossy(&bytes)) }) } +/// Returns a `Matcher` that returns true if any provided field of the log event resolves to an array, where +/// at least one `Value` it contains matches the provided `func`. +fn any_match_multiple(fields: Vec, func: F) -> Box> +where + S: Into + Clone + Send + Sync + 'static, + F: Fn(&Value) -> bool + Send + Sync + Clone + 'static, +{ + array_match_multiple(fields, move |values| values.iter().any(&func)) +} + +/// Returns a `Matcher` that returns true if any provided field of the log event resolves to an array, where +/// the vector of `Value`s the array contains matches the provided `func`. +fn array_match_multiple(fields: Vec, func: F) -> Box> +where + S: Into + Clone + Send + Sync + 'static, + F: Fn(&Vec) -> bool + Send + Sync + Clone + 'static, +{ + Run::boxed(move |log: &LogEvent| { + fields.iter().any(|field| { + let field = field.clone().into(); + match log.parse_path_and_get_value(field.as_str()).ok().flatten() { + Some(Value::Array(values)) => func(values), + _ => false, + } + }) + }) +} + #[cfg(test)] mod test { use super::*; @@ -1187,6 +1277,314 @@ mod test { log_event!["field" => false, "field2" => "value2"], log_event!["field" => true, "field2" => "value2"], ), + // tags checks with 'ddtags' (DD Agent Source naming) + + // Tag exists. + ( + "_exists_:a", // Source + log_event!["ddtags" => vec!["a:foo"]], // Pass + log_event!["ddtags" => vec!["b:foo"]], // Fail + ), + // Tag exists with - in name. + ( + "_exists_:a-b", // Source + log_event!["ddtags" => vec!["a-b:foo"]], // Pass + log_event!["ddtags" => vec!["ab:foo"]], // Fail + ), + // Tag exists (negate). + ( + "NOT _exists_:a", + log_event!["ddtags" => vec!["b:foo"]], + log_event!("ddtags" => vec!["a:foo"]), + ), + // Tag exists (negate w/-). + ( + "-_exists_:a", + log_event!["ddtags" => vec!["b:foo"]], + log_event!["ddtags" => vec!["a:foo"]], + ), + // Tag doesn't exist. + ( + "_missing_:a", + log_event![], + log_event!["ddtags" => vec!["a:foo"]], + ), + // Tag doesn't exist (negate). + ( + "NOT _missing_:a", + log_event!["ddtags" => vec!["a:foo"]], + log_event![], + ), + // Tag doesn't exist (negate w/-). + ( + "-_missing_:a", + log_event!["ddtags" => vec!["a:foo"]], + log_event![], + ), + // Tag match. + ( + "a:bla", + log_event!["ddtags" => vec!["a:bla"]], + log_event!["ddtags" => vec!["b:bla"]], + ), + // Tag match (negate). + ( + "NOT a:bla", + log_event!["ddtags" => vec!["b:bla"]], + log_event!["ddtags" => vec!["a:bla"]], + ), + // Reserved tag match (negate). + ( + "NOT host:foo", + log_event!["ddtags" => vec!["host:fo o"]], + log_event!["host" => "foo"], + ), + // Tag match (negate w/-). + ( + "-a:bla", + log_event!["ddtags" => vec!["b:bla"]], + log_event!["ddtags" => vec!["a:bla"]], + ), + // Quoted tag match. + ( + r#"a:"bla""#, + log_event!["ddtags" => vec!["a:bla"]], + log_event!["a" => "bla"], + ), + // Quoted tag match (negate). + ( + r#"NOT a:"bla""#, + log_event!["a" => "bla"], + log_event!["ddtags" => vec!["a:bla"]], + ), + // Quoted tag match (negate w/-). + ( + r#"-a:"bla""#, + log_event!["a" => "bla"], + log_event!["ddtags" => vec!["a:bla"]], + ), + // String attribute match. + ( + "@a:bla", + log_event!["a" => "bla"], + log_event!["ddtags" => vec!["a:bla"]], + ), + // String attribute match (negate). + ( + "NOT @a:bla", + log_event!["ddtags" => vec!["a:bla"]], + log_event!["a" => "bla"], + ), + // String attribute match (negate w/-). + ( + "-@a:bla", + log_event!["ddtags" => vec!["a:bla"]], + log_event!["a" => "bla"], + ), + // Quoted attribute match. + ( + r#"@a:"bla""#, + log_event!["a" => "bla"], + log_event!["ddtags" => vec!["a:bla"]], + ), + // Quoted attribute match (negate). + ( + r#"NOT @a:"bla""#, + log_event!["ddtags" => vec!["a:bla"]], + log_event!["a" => "bla"], + ), + // Quoted attribute match (negate w/-). + ( + r#"-@a:"bla""#, + log_event!["ddtags" => vec!["a:bla"]], + log_event!["a" => "bla"], + ), + // Integer attribute match. + ( + "@a:200", + log_event!["a" => 200], + log_event!["ddtags" => vec!["a:200"]], + ), + // Float attribute match. + ( + "@a:0.75", + log_event!["a" => 0.75], + log_event!["ddtags" => vec!["a:0.75"]], + ), + ( + "a:*bla", + log_event!["ddtags" => vec!["a:foobla"]], + log_event!["ddtags" => vec!["a:blafoo"]], + ), + // Wildcard prefix - tag (negate). + ( + "NOT a:*bla", + log_event!["ddtags" => vec!["a:blafoo"]], + log_event!["ddtags" => vec!["a:foobla"]], + ), + // Wildcard prefix - tag (negate w/-). + ( + "-a:*bla", + log_event!["ddtags" => vec!["a:blafoo"]], + log_event!["ddtags" => vec!["a:foobla"]], + ), + // Wildcard suffix - tag. + ( + "b:bla*", + log_event!["ddtags" => vec!["b:blabop"]], + log_event!["ddtags" => vec!["b:bopbla"]], + ), + // Wildcard suffix - tag (negate). + ( + "NOT b:bla*", + log_event!["ddtags" => vec!["b:bopbla"]], + log_event!["ddtags" => vec!["b:blabop"]], + ), + // Wildcard suffix - tag (negate w/-). + ( + "-b:bla*", + log_event!["ddtags" => vec!["b:bopbla"]], + log_event!["ddtags" => vec!["b:blabop"]], + ), + // Multiple wildcards - tag. + ( + "c:*b*la*", + log_event!["ddtags" => vec!["c:foobla"]], + log_event!["custom" => r#"{"title" => "foobla"}"#], + ), + // Multiple wildcards - tag (negate). + ( + "NOT c:*b*la*", + log_event!["custom" => r#"{"title" => "foobla"}"#], + log_event!["ddtags" => vec!["c:foobla"]], + ), + // Multiple wildcards - tag (negate w/-). + ( + "-c:*b*la*", + log_event!["custom" => r#"{"title" => "foobla"}"#], + log_event!["ddtags" => vec!["c:foobla"]], + ), + // Wildcard prefix - attribute. + ( + "@a:*bla", + log_event!["a" => "foobla"], + log_event!["ddtags" => vec!["a:foobla"]], + ), + // Wildcard prefix - attribute (negate). + ( + "NOT @a:*bla", + log_event!["ddtags" => vec!["a:foobla"]], + log_event!["a" => "foobla"], + ), + // Wildcard prefix - attribute (negate w/-). + ( + "-@a:*bla", + log_event!["ddtags" => vec!["a:foobla"]], + log_event!["a" => "foobla"], + ), + // Wildcard suffix - attribute. + ( + "@b:bla*", + log_event!["b" => "blabop"], + log_event!["ddtags" => vec!["b:blabop"]], + ), + // Wildcard suffix - attribute (negate). + ( + "NOT @b:bla*", + log_event!["ddtags" => vec!["b:blabop"]], + log_event!["b" => "blabop"], + ), + // Wildcard suffix - attribute (negate w/-). + ( + "-@b:bla*", + log_event!["ddtags" => vec!["b:blabop"]], + log_event!["b" => "blabop"], + ), + // Multiple wildcards - attribute. + ( + "@c:*b*la*", + log_event!["c" => "foobla"], + log_event!["ddtags" => vec!["c:foobla"]], + ), + // Multiple wildcards - attribute (negate). + ( + "NOT @c:*b*la*", + log_event!["ddtags" => vec!["c:foobla"]], + log_event!["c" => "foobla"], + ), + // Multiple wildcards - attribute (negate w/-). + ( + "-@c:*b*la*", + log_event!["ddtags" => vec!["c:foobla"]], + log_event!["c" => "foobla"], + ), + // Special case for tags. + ( + "tags:a", + log_event!["ddtags" => vec!["a", "b", "c"]], + log_event!["ddtags" => vec!["d", "e", "f"]], + ), + // Special case for tags (negate). + ( + "NOT tags:a", + log_event!["ddtags" => vec!["d", "e", "f"]], + log_event!["ddtags" => vec!["a", "b", "c"]], + ), + // Special case for tags (negate w/-). + ( + "-tags:a", + log_event!["ddtags" => vec!["d", "e", "f"]], + log_event!["ddtags" => vec!["a", "b", "c"]], + ), + // Special case: 'source' looks up on 'source' and 'ddsource' (OR condition) + // source + ( + "source:foo", + log_event!["source" => "foo"], + log_event!["tags" => vec!["source:foo"]], + ), + ( + "source:foo", + log_event!["source" => "foo"], + log_event!["source" => "foobar"], + ), + ( + "source:foo", + log_event!["source" => "foo"], + log_event!["source" => r#"{"value": "foo"}"#], + ), + // ddsource + ( + "source:foo", + log_event!["ddsource" => "foo"], + log_event!["tags" => vec!["ddsource:foo"]], + ), + ( + "source:foo", + log_event!["ddsource" => "foo"], + log_event!["ddsource" => "foobar"], + ), + ( + "source:foo", + log_event!["ddsource" => "foo"], + log_event!["ddsource" => r#"{"value": "foo"}"#], + ), + // both source and ddsource + ( + "source:foo", + log_event!["source" => "foo", "ddsource" => "foo"], + log_event!["source" => "foobar", "ddsource" => "foobar"], + ), + ( + "source:foo", + log_event!["source" => "foo", "ddsource" => "foobar"], + log_event!["source" => "foobar", "ddsource" => "foobar"], + ), + ( + "source:foo", + log_event!["source" => "foobar", "ddsource" => "foo"], + log_event!["source" => "foobar", "ddsource" => "foobar"], + ), ] } @@ -1229,7 +1627,7 @@ mod test { // Every query should build successfully. let cond = config .build(&Default::default()) - .unwrap_or_else(|_| panic!("build failed: {}", source)); + .unwrap_or_else(|_| panic!("build failed: {source}")); assert!( cond.check_with_context(pass.clone()).0.is_ok(), diff --git a/src/conditions/is_log.rs b/src/conditions/is_log.rs index 940b1d5011..1b365f12cd 100644 --- a/src/conditions/is_log.rs +++ b/src/conditions/is_log.rs @@ -17,8 +17,8 @@ pub(crate) fn check_is_log_with_context(e: Event) -> (Result<(), String>, Event) mod test { use super::check_is_log; use crate::event::{ - metric::{Metric, MetricKind, MetricValue}, Event, LogEvent, + metric::{Metric, MetricKind, MetricValue}, }; #[test] diff --git a/src/conditions/is_metric.rs b/src/conditions/is_metric.rs index 9d68f847e3..a7ca5c1cd6 100644 --- a/src/conditions/is_metric.rs +++ b/src/conditions/is_metric.rs @@ -17,8 +17,8 @@ pub(crate) fn check_is_metric_with_context(e: Event) -> (Result<(), String>, Eve mod test { use super::check_is_metric; use crate::event::{ - metric::{Metric, MetricKind, MetricValue}, Event, LogEvent, + metric::{Metric, MetricKind, MetricValue}, }; #[test] diff --git a/src/conditions/is_trace.rs b/src/conditions/is_trace.rs index 29aa7dbde3..1166439a2e 100644 --- a/src/conditions/is_trace.rs +++ b/src/conditions/is_trace.rs @@ -17,8 +17,8 @@ pub(crate) fn check_is_trace_with_context(e: Event) -> (Result<(), String>, Even mod test { use super::check_is_trace; use crate::event::{ - metric::{Metric, MetricKind, MetricValue}, Event, LogEvent, TraceEvent, + metric::{Metric, MetricKind, MetricValue}, }; #[test] diff --git a/src/conditions/vrl.rs b/src/conditions/vrl.rs index e8170e3ec5..d47810aa8e 100644 --- a/src/conditions/vrl.rs +++ b/src/conditions/vrl.rs @@ -1,5 +1,5 @@ use vector_lib::configurable::configurable_component; -use vector_lib::{compile_vrl, emit, TimeZone}; +use vector_lib::{TimeZone, compile_vrl, emit}; use vrl::compiler::runtime::{Runtime, RuntimeResult, Terminate}; use vrl::compiler::{CompilationResult, CompileConfig, Program, TypeState, VrlRuntime}; use vrl::diagnostic::Formatter; @@ -142,7 +142,7 @@ impl Conditional for Vrl { ) .colored() .to_string(); - format!("source execution aborted: {}", err) + format!("source execution aborted: {err}") } Terminate::Error(err) => { let err = Formatter::new( @@ -153,7 +153,7 @@ impl Conditional for Vrl { ) .colored() .to_string(); - format!("source execution failed: {}", err) + format!("source execution failed: {err}") } }); diff --git a/src/config/api.rs b/src/config/api.rs index c9ed3802d9..fca088f751 100644 --- a/src/config/api.rs +++ b/src/config/api.rs @@ -4,27 +4,42 @@ use url::Url; use vector_lib::configurable::configurable_component; /// API options. -#[configurable_component] +#[configurable_component(api("api"))] #[derive(Clone, Copy, Debug, Eq, PartialEq)] #[serde(default, deny_unknown_fields)] pub struct Options { - /// Whether or not the API endpoint is available. + /// Whether the GraphQL API is enabled for this Vector instance. #[serde(default = "default_enabled")] + #[configurable(metadata(docs::common = true, docs::required = false))] pub enabled: bool, - /// The socket address to listen on for the API endpoint. + /// The network address to which the API should bind. If you're running + /// Vector in a Docker container, bind to `0.0.0.0`. Otherwise + /// the API will not be exposed outside the container. #[serde(default = "default_address")] + #[configurable(metadata(docs::examples = "0.0.0.0:8686"))] + #[configurable(metadata(docs::examples = "127.0.0.1:1234"))] + #[configurable(metadata(docs::common = true, docs::required = false))] pub address: Option, - /// Whether or not to expose the GraphQL playground on the API endpoint. + /// Whether the [GraphQL Playground](https://github.com/graphql/graphql-playground) is enabled + /// for the API. The Playground is accessible via the `/playground` endpoint + /// of the address set using the `bind` parameter. Note that the `playground` + /// endpoint will only be enabled if the `graphql` endpoint is also enabled. #[serde(default = "default_playground")] + #[configurable(metadata(docs::common = false, docs::required = false))] pub playground: bool, - /// Whether or not the GraphQL endpoint is enabled + /// Whether the endpoint for receiving and processing GraphQL queries is + /// enabled for the API. The endpoint is accessible via the `/graphql` + /// endpoint of the address set using the `bind` parameter. #[serde(default = "default_graphql", skip_serializing_if = "is_true")] + #[configurable(metadata(docs::common = true, docs::required = false))] pub graphql: bool, } +impl_generate_config_from_default!(Options); + impl Default for Options { fn default() -> Self { Self { @@ -56,7 +71,7 @@ pub fn default_address() -> Option { /// Default GraphQL API address pub fn default_graphql_url() -> Url { let addr = default_address().unwrap(); - Url::parse(&format!("http://{}/graphql", addr)) + Url::parse(&format!("http://{addr}/graphql")) .expect("Couldn't parse default API URL. Please report this.") } @@ -80,9 +95,7 @@ impl Options { // Prefer non default address (Some(a), Some(b)) => { match (Some(a) == default_address(), Some(b) == default_address()) { - (false, false) => { - return Err(format!("Conflicting `api` address: {}, {} .", a, b)) - } + (false, false) => return Err(format!("Conflicting `api` address: {a}, {b} .")), (false, true) => Some(a), (true, _) => Some(b), } diff --git a/src/config/builder.rs b/src/config/builder.rs index 3a0413e38a..b71e9de747 100644 --- a/src/config/builder.rs +++ b/src/config/builder.rs @@ -9,9 +9,8 @@ use crate::{enrichment_tables::EnrichmentTables, providers::Providers, secrets:: #[cfg(feature = "api")] use super::api; use super::{ - compiler, schema, BoxedSink, BoxedSource, BoxedTransform, ComponentKey, Config, - EnrichmentTableOuter, HealthcheckOptions, SinkOuter, SourceOuter, TestDefinition, - TransformOuter, + BoxedSink, BoxedSource, BoxedTransform, ComponentKey, Config, EnrichmentTableOuter, + HealthcheckOptions, SinkOuter, SourceOuter, TestDefinition, TransformOuter, compiler, schema, }; /// A complete Vector configuration. @@ -38,7 +37,7 @@ pub struct ConfigBuilder { /// All configured enrichment tables. #[serde(default)] - pub enrichment_tables: IndexMap, + pub enrichment_tables: IndexMap>, /// All configured sources. #[serde(default)] @@ -106,6 +105,11 @@ impl From for ConfigBuilder { .map(|(key, sink)| (key, sink.map_inputs(ToString::to_string))) .collect(); + let enrichment_tables = enrichment_tables + .into_iter() + .map(|(key, table)| (key, table.map_inputs(ToString::to_string))) + .collect(); + let tests = tests.into_iter().map(TestDefinition::stringify).collect(); ConfigBuilder { @@ -145,11 +149,16 @@ impl ConfigBuilder { pub fn add_enrichment_table, E: Into>( &mut self, key: K, + inputs: &[&str], enrichment_table: E, ) { + let inputs = inputs + .iter() + .map(|value| value.to_string()) + .collect::>(); self.enrichment_tables.insert( ComponentKey::from(key.into()), - EnrichmentTableOuter::new(enrichment_table), + EnrichmentTableOuter::new(inputs, enrichment_table), ); } @@ -221,22 +230,22 @@ impl ConfigBuilder { with.enrichment_tables.keys().for_each(|k| { if self.enrichment_tables.contains_key(k) { - errors.push(format!("duplicate enrichment_table name found: {}", k)); + errors.push(format!("duplicate enrichment_table name found: {k}")); } }); with.sources.keys().for_each(|k| { if self.sources.contains_key(k) { - errors.push(format!("duplicate source id found: {}", k)); + errors.push(format!("duplicate source id found: {k}")); } }); with.sinks.keys().for_each(|k| { if self.sinks.contains_key(k) { - errors.push(format!("duplicate sink id found: {}", k)); + errors.push(format!("duplicate sink id found: {k}")); } }); with.transforms.keys().for_each(|k| { if self.transforms.contains_key(k) { - errors.push(format!("duplicate transform id found: {}", k)); + errors.push(format!("duplicate transform id found: {k}")); } }); with.tests.iter().for_each(|wt| { @@ -246,7 +255,7 @@ impl ConfigBuilder { }); with.secret.keys().for_each(|k| { if self.secret.contains_key(k) { - errors.push(format!("duplicate secret id found: {}", k)); + errors.push(format!("duplicate secret id found: {k}")); } }); if !errors.is_empty() { diff --git a/src/config/cmd.rs b/src/config/cmd.rs index a76d7a4a08..e0b9f7389e 100644 --- a/src/config/cmd.rs +++ b/src/config/cmd.rs @@ -3,7 +3,7 @@ use std::path::PathBuf; use clap::Parser; use serde_json::Value; -use super::{load_builder_from_paths, load_source_from_paths, process_paths, ConfigBuilder}; +use super::{ConfigBuilder, load_builder_from_paths, load_source_from_paths, process_paths}; use crate::cli::handle_config_errors; use crate::config; @@ -78,7 +78,7 @@ impl Opts { /// Helper to merge JSON. Handles objects and array concatenation. fn merge_json(a: &mut Value, b: Value) { match (a, b) { - (Value::Object(ref mut a), Value::Object(b)) => { + (Value::Object(a), Value::Object(b)) => { for (k, v) in b { merge_json(a.entry(k).or_insert(Value::Null), v); } @@ -92,7 +92,7 @@ fn merge_json(a: &mut Value, b: Value) { /// Helper to sort array values. fn sort_json_array_values(json: &mut Value) { match json { - Value::Array(ref mut arr) => { + Value::Array(arr) => { for v in arr.iter_mut() { sort_json_array_values(v); } @@ -113,7 +113,7 @@ fn sort_json_array_values(json: &mut Value) { .map(|v| serde_json::from_str(v.as_str()).unwrap()) .collect::>(); } - Value::Object(ref mut json) => { + Value::Object(json) => { for (_, v) in json { sort_json_array_values(v); } @@ -196,8 +196,8 @@ mod tests { use proptest::{num, prelude::*, sample}; use rand::{ - prelude::{SliceRandom, StdRng}, SeedableRng, + prelude::{SliceRandom, StdRng}, }; use serde_json::json; use similar_asserts::assert_eq; @@ -207,9 +207,9 @@ mod tests { use crate::config::Format; use crate::{ - config::{cmd::serialize_to_json, vars, ConfigBuilder}, + config::{ConfigBuilder, cmd::serialize_to_json, vars}, generate, - generate::{generate_example, TransformInputsStrategy}, + generate::{TransformInputsStrategy, generate_example}, }; use super::merge_json; @@ -242,13 +242,12 @@ mod tests { r#" [sources.in] type = "demo_logs" - format = "${{{}}}" + format = "${{{env_var}}}" [sinks.out] type = "blackhole" - inputs = ["${{{}}}"] - "#, - env_var, env_var_in_arr + inputs = ["${{{env_var_in_arr}}}"] + "# ); let interpolated_config_source = vars::interpolate( config_source.as_ref(), @@ -319,18 +318,18 @@ mod tests { "{}/{}/{}", sources .iter() - .map(|source| format!("{}:{}", source, source)) + .map(|source| format!("{source}:{source}")) .collect::>() .join(","), transforms .iter() - .map(|transform| format!("{}:{}", transform, transform)) + .map(|transform| format!("{transform}:{transform}")) .chain(vec!["manually-added-remap:remap".to_string()]) .collect::>() .join(","), sinks .iter() - .map(|sink| format!("{}:{}", sink, sink)) + .map(|sink| format!("{sink}:{sink}")) .collect::>() .join(","), ); diff --git a/src/config/compiler.rs b/src/config/compiler.rs index 28015e01ab..bc194baa59 100644 --- a/src/config/compiler.rs +++ b/src/config/compiler.rs @@ -1,9 +1,9 @@ use super::{ - builder::ConfigBuilder, graph::Graph, transform::get_transform_output_ids, validation, Config, - OutputId, + Config, OutputId, builder::ConfigBuilder, graph::Graph, transform::get_transform_output_ids, + validation, }; -use indexmap::IndexSet; +use indexmap::{IndexMap, IndexSet}; use vector_lib::id::Inputs; pub fn compile(mut builder: ConfigBuilder) -> Result<(Config, Vec), Vec> { @@ -52,8 +52,32 @@ pub fn compile(mut builder: ConfigBuilder) -> Result<(Config, Vec), Vec< graceful_shutdown_duration, allow_empty: _, } = builder; + let all_sinks = sinks + .clone() + .into_iter() + .chain( + enrichment_tables + .iter() + .filter_map(|(key, table)| table.as_sink(key)), + ) + .collect::>(); + let sources_and_table_sources = sources + .clone() + .into_iter() + .chain( + enrichment_tables + .iter() + .filter_map(|(key, table)| table.as_source(key)), + ) + .collect::>(); - let graph = match Graph::new(&sources, &transforms, &sinks, schema) { + let graph = match Graph::new( + &sources_and_table_sources, + &transforms, + &all_sinks, + schema, + global.wildcard_matching.unwrap_or_default(), + ) { Ok(graph) => graph, Err(graph_errors) => { errors.extend(graph_errors); @@ -85,6 +109,13 @@ pub fn compile(mut builder: ConfigBuilder) -> Result<(Config, Vec), Vec< (key, transform.with_inputs(inputs)) }) .collect(); + let enrichment_tables = enrichment_tables + .into_iter() + .map(|(key, table)| { + let inputs = graph.inputs_for(&key); + (key, table.with_inputs(inputs)) + }) + .collect(); let tests = tests .into_iter() .map(|test| test.resolve_outputs(&graph)) diff --git a/src/config/diff.rs b/src/config/diff.rs index da5ed54fae..fa5f8cf09b 100644 --- a/src/config/diff.rs +++ b/src/config/diff.rs @@ -1,56 +1,70 @@ use std::collections::HashSet; use indexmap::IndexMap; +use vector_lib::config::OutputId; -use super::{ComponentKey, Config}; +use super::{ComponentKey, Config, EnrichmentTableOuter}; #[derive(Debug)] pub struct ConfigDiff { pub sources: Difference, pub transforms: Difference, pub sinks: Difference, + /// This difference does not only contain the actual enrichment_tables keys, but also keys that + /// may be used for their source and sink components (if available). pub enrichment_tables: Difference, + pub components_to_reload: HashSet, } impl ConfigDiff { pub fn initial(initial: &Config) -> Self { - Self::new(&Config::default(), initial) + Self::new(&Config::default(), initial, HashSet::new()) } - pub fn new(old: &Config, new: &Config) -> Self { + pub fn new(old: &Config, new: &Config, components_to_reload: HashSet) -> Self { ConfigDiff { - sources: Difference::new(&old.sources, &new.sources), - transforms: Difference::new(&old.transforms, &new.transforms), - sinks: Difference::new(&old.sinks, &new.sinks), - enrichment_tables: Difference::new(&old.enrichment_tables, &new.enrichment_tables), + sources: Difference::new(&old.sources, &new.sources, &components_to_reload), + transforms: Difference::new(&old.transforms, &new.transforms, &components_to_reload), + sinks: Difference::new(&old.sinks, &new.sinks, &components_to_reload), + enrichment_tables: Difference::new_tables( + &old.enrichment_tables, + &new.enrichment_tables, + ), + components_to_reload, } } /// Swaps removed with added in Differences. - pub fn flip(mut self) -> Self { + pub const fn flip(mut self) -> Self { self.sources.flip(); self.transforms.flip(); self.sinks.flip(); + self.enrichment_tables.flip(); self } - /// Checks whether or not the given component is present at all. + /// Checks whether the given component is present at all. pub fn contains(&self, key: &ComponentKey) -> bool { - self.sources.contains(key) || self.transforms.contains(key) || self.sinks.contains(key) + self.sources.contains(key) + || self.transforms.contains(key) + || self.sinks.contains(key) + || self.enrichment_tables.contains(key) } - /// Checks whether or not the given component is changed. + /// Checks whether the given component is changed. pub fn is_changed(&self, key: &ComponentKey) -> bool { self.sources.is_changed(key) || self.transforms.is_changed(key) || self.sinks.is_changed(key) + || self.enrichment_tables.contains(key) } - /// Checks whether or not the given component is removed. + /// Checks whether the given component is removed. pub fn is_removed(&self, key: &ComponentKey) -> bool { self.sources.is_removed(key) || self.transforms.is_removed(key) || self.sinks.is_removed(key) + || self.enrichment_tables.contains(key) } } @@ -62,13 +76,58 @@ pub struct Difference { } impl Difference { - fn new(old: &IndexMap, new: &IndexMap) -> Self + fn new( + old: &IndexMap, + new: &IndexMap, + need_change: &HashSet, + ) -> Self where C: serde::Serialize + serde::Deserialize<'static>, { let old_names = old.keys().cloned().collect::>(); let new_names = new.keys().cloned().collect::>(); + let to_change = old_names + .intersection(&new_names) + .filter(|&n| { + // This is a hack around the issue of comparing two + // trait objects. Json is used here over toml since + // toml does not support serializing `None` + // to_value is used specifically (instead of string) + // to avoid problems comparing serialized HashMaps, + // which can iterate in varied orders. + let old_value = serde_json::to_value(&old[n]).unwrap(); + let new_value = serde_json::to_value(&new[n]).unwrap(); + old_value != new_value || need_change.contains(n) + }) + .cloned() + .collect::>(); + + let to_remove = &old_names - &new_names; + let to_add = &new_names - &old_names; + + Self { + to_remove, + to_change, + to_add, + } + } + + fn new_tables( + old: &IndexMap>, + new: &IndexMap>, + ) -> Self { + let old_names = old + .iter() + .flat_map(|(k, t)| vec![t.as_source(k).map(|(k, _)| k), t.as_sink(k).map(|(k, _)| k)]) + .flatten() + .collect::>(); + let new_names = new + .iter() + .flat_map(|(k, t)| vec![t.as_source(k).map(|(k, _)| k), t.as_sink(k).map(|(k, _)| k)]) + .flatten() + .collect::>(); + let to_change = old_names .intersection(&new_names) .filter(|&n| { @@ -130,7 +189,7 @@ impl Difference { self.to_remove.contains(key) } - fn flip(&mut self) { + const fn flip(&mut self) { std::mem::swap(&mut self.to_remove, &mut self.to_add); } diff --git a/src/config/enrichment_table.rs b/src/config/enrichment_table.rs index 5e2cd72a00..a953dcd747 100644 --- a/src/config/enrichment_table.rs +++ b/src/config/enrichment_table.rs @@ -1,21 +1,110 @@ use enum_dispatch::enum_dispatch; +use serde::Serialize; use vector_lib::config::GlobalOptions; -use vector_lib::configurable::{configurable_component, NamedComponent}; +use vector_lib::configurable::{Configurable, NamedComponent, ToValue, configurable_component}; +use vector_lib::id::{ComponentKey, Inputs}; use crate::enrichment_tables::EnrichmentTables; +use super::dot_graph::GraphConfig; +use super::{SinkConfig, SinkOuter, SourceConfig, SourceOuter}; + /// Fully resolved enrichment table component. #[configurable_component] #[derive(Clone, Debug)] -pub struct EnrichmentTableOuter { +pub struct EnrichmentTableOuter +where + T: Configurable + Serialize + 'static + ToValue + Clone, +{ #[serde(flatten)] pub inner: EnrichmentTables, + #[configurable(derived)] + #[serde(default, skip_serializing_if = "vector_lib::serde::is_default")] + pub graph: GraphConfig, + #[configurable(derived)] + #[serde( + default = "Inputs::::default", + skip_serializing_if = "Inputs::is_empty" + )] + pub inputs: Inputs, } -impl EnrichmentTableOuter { - pub fn new>(inner: I) -> Self { +impl EnrichmentTableOuter +where + T: Configurable + Serialize + 'static + ToValue + Clone, +{ + pub fn new(inputs: I, inner: IET) -> Self + where + I: IntoIterator, + IET: Into, + { Self { inner: inner.into(), + graph: Default::default(), + inputs: Inputs::from_iter(inputs), + } + } + + // Components are currently built in a way that they match exactly one of the roles (source, + // transform, sink, enrichment table). Due to specific requirements of the "memory" enrichment + // table, it has to fulfill 2 of these roles (sink and enrichment table). To reduce the impact + // of this very specific requirement, any enrichment table can now be optionally mapped into a + // sink, but this will only work for a "memory" enrichment table, since other tables will not + // have a "sink_config" present. + // This is also not ideal, since `SinkOuter` is not meant to represent the actual configuration, + // but it should just be a representation of that config used for deserialization. + // In the future, if more such components come up, it would be good to limit such "Outer" + // components to deserialization and build up the components and the topology in a more granular + // way, with each having "modules" for inputs (making them valid as sinks), for healthchecks, + // for providing outputs, etc. + pub fn as_sink(&self, default_key: &ComponentKey) -> Option<(ComponentKey, SinkOuter)> { + self.inner.sink_config(default_key).map(|(key, sink)| { + ( + key, + SinkOuter { + graph: self.graph.clone(), + inputs: self.inputs.clone(), + healthcheck_uri: None, + healthcheck: Default::default(), + buffer: Default::default(), + proxy: Default::default(), + inner: sink, + }, + ) + }) + } + + pub fn as_source(&self, default_key: &ComponentKey) -> Option<(ComponentKey, SourceOuter)> { + self.inner.source_config(default_key).map(|(key, source)| { + ( + key, + SourceOuter { + graph: self.graph.clone(), + sink_acknowledgements: false, + proxy: Default::default(), + inner: source, + }, + ) + }) + } + + pub(super) fn map_inputs(self, f: impl Fn(&T) -> U) -> EnrichmentTableOuter + where + U: Configurable + Serialize + 'static + ToValue + Clone, + { + let inputs = self.inputs.iter().map(f).collect::>(); + self.with_inputs(inputs) + } + + pub(crate) fn with_inputs(self, inputs: I) -> EnrichmentTableOuter + where + I: IntoIterator, + U: Configurable + Serialize + 'static + ToValue + Clone, + { + EnrichmentTableOuter { + inputs: Inputs::from_iter(inputs), + inner: self.inner, + graph: self.graph, } } } @@ -36,4 +125,18 @@ pub trait EnrichmentTableConfig: NamedComponent + core::fmt::Debug + Send + Sync &self, globals: &GlobalOptions, ) -> crate::Result>; + + fn sink_config( + &self, + _default_key: &ComponentKey, + ) -> Option<(ComponentKey, Box)> { + None + } + + fn source_config( + &self, + _default_key: &ComponentKey, + ) -> Option<(ComponentKey, Box)> { + None + } } diff --git a/src/config/format.rs b/src/config/format.rs index 24e0b4059c..dd8ca028d5 100644 --- a/src/config/format.rs +++ b/src/config/format.rs @@ -6,13 +6,28 @@ use std::fmt; use std::path::Path; use std::str::FromStr; -use serde::de; +use serde::{Deserialize, Serialize, de}; +use vector_config_macros::Configurable; /// A type alias to better capture the semantics. pub type FormatHint = Option; /// The format used to represent the configuration data. -#[derive(Debug, Default, Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)] +#[derive( + Debug, + Default, + Copy, + Clone, + Eq, + PartialEq, + Ord, + PartialOrd, + Hash, + Serialize, + Deserialize, + Configurable, +)] +#[serde(rename_all = "snake_case")] pub enum Format { /// TOML format is used. #[default] @@ -31,7 +46,7 @@ impl FromStr for Format { "toml" => Ok(Format::Toml), "yaml" => Ok(Format::Yaml), "json" => Ok(Format::Json), - _ => Err(format!("Invalid format: {}", s)), + _ => Err(format!("Invalid format: {s}")), } } } @@ -43,7 +58,7 @@ impl fmt::Display for Format { Format::Json => "json", Format::Yaml => "yaml", }; - write!(f, "{}", format) + write!(f, "{format}") } } @@ -148,7 +163,7 @@ mod tests { for (input, expected) in cases { let output = Format::from_path(std::path::PathBuf::from(input)); - assert_eq!(expected, output.ok(), "{}", input) + assert_eq!(expected, output.ok(), "{input}") } } @@ -164,7 +179,7 @@ mod tests { use crate::config::ConfigBuilder; macro_rules! concat_with_newlines { - ($($e:expr,)*) => { concat!( $($e, "\n"),+ ) }; + ($($e:expr_2021,)*) => { concat!( $($e, "\n"),+ ) }; } const SAMPLE_TOML: &str = r#" @@ -301,23 +316,20 @@ mod tests { Ok(expected) => { #[allow(clippy::expect_fun_call)] // false positive let output: ConfigBuilder = output.expect(&format!( - "expected Ok, got Err with format {:?} and input {:?}", - format, input + "expected Ok, got Err with format {format:?} and input {input:?}" )); let output_json = serde_json::to_value(output).unwrap(); let expected_output: ConfigBuilder = deserialize(expected, Format::Toml) .expect("Invalid TOML passed as an expectation"); let expected_json = serde_json::to_value(expected_output).unwrap(); - assert_eq!(expected_json, output_json, "{}", input) + assert_eq!(expected_json, output_json, "{input}") } Err(expected) => assert_eq!( expected, output.expect_err(&format!( - "expected Err, got Ok with format {:?} and input {:?}", - format, input + "expected Err, got Ok with format {format:?} and input {input:?}" )), - "{}", - input + "{input}" ), } } diff --git a/src/config/graph.rs b/src/config/graph.rs index 70c59399ec..2f22013f4b 100644 --- a/src/config/graph.rs +++ b/src/config/graph.rs @@ -1,8 +1,8 @@ use super::{ - schema, ComponentKey, DataType, OutputId, SinkOuter, SourceOuter, SourceOutput, TransformOuter, - TransformOutput, + ComponentKey, DataType, OutputId, SinkOuter, SourceOuter, SourceOutput, TransformOuter, + TransformOutput, WildcardMatching, schema, }; -use indexmap::{set::IndexSet, IndexMap}; +use indexmap::{IndexMap, set::IndexSet}; use std::collections::{HashMap, HashSet, VecDeque}; use std::fmt; @@ -26,7 +26,7 @@ impl fmt::Display for Node { Node::Source { outputs } => { write!(f, "component_kind: source\n outputs:")?; for output in outputs { - write!(f, "\n {}", output)?; + write!(f, "\n {output}")?; } Ok(()) } @@ -36,7 +36,7 @@ impl fmt::Display for Node { "component_kind: source\n input_types: {in_ty}\n outputs:" )?; for output in outputs { - write!(f, "\n {}", output)?; + write!(f, "\n {output}")?; } Ok(()) } @@ -65,8 +65,9 @@ impl Graph { transforms: &IndexMap>, sinks: &IndexMap>, schema: schema::Options, + wildcard_matching: WildcardMatching, ) -> Result> { - Self::new_inner(sources, transforms, sinks, false, schema) + Self::new_inner(sources, transforms, sinks, false, schema, wildcard_matching) } pub fn new_unchecked( @@ -74,8 +75,10 @@ impl Graph { transforms: &IndexMap>, sinks: &IndexMap>, schema: schema::Options, + wildcard_matching: WildcardMatching, ) -> Self { - Self::new_inner(sources, transforms, sinks, true, schema).expect("errors ignored") + Self::new_inner(sources, transforms, sinks, true, schema, wildcard_matching) + .expect("errors ignored") } fn new_inner( @@ -84,6 +87,7 @@ impl Graph { sinks: &IndexMap>, ignore_errors: bool, schema: schema::Options, + wildcard_matching: WildcardMatching, ) -> Result> { let mut graph = Graph::default(); let mut errors = Vec::new(); @@ -112,7 +116,7 @@ impl Graph { ); } - for (id, config) in sinks.iter() { + for (id, config) in sinks { graph.nodes.insert( id.clone(), Node::Sink { @@ -127,15 +131,15 @@ impl Graph { for (id, config) in transforms.iter() { for input in config.inputs.iter() { - if let Err(e) = graph.add_input(input, id, &available_inputs) { + if let Err(e) = graph.add_input(input, id, &available_inputs, wildcard_matching) { errors.push(e); } } } - for (id, config) in sinks.iter() { + for (id, config) in sinks { for input in config.inputs.iter() { - if let Err(e) = graph.add_input(input, id, &available_inputs) { + if let Err(e) = graph.add_input(input, id, &available_inputs, wildcard_matching) { errors.push(e); } } @@ -153,6 +157,7 @@ impl Graph { from: &str, to: &ComponentKey, available_inputs: &HashMap, + wildcard_matching: WildcardMatching, ) -> Result<(), String> { if let Some(output_id) = available_inputs.get(from) { self.edges.push(Edge { @@ -166,11 +171,25 @@ impl Graph { Some(Node::Sink { .. }) => "sink", _ => panic!("only transforms and sinks have inputs"), }; + // allow empty result if relaxed wildcard matching is enabled + match wildcard_matching { + WildcardMatching::Relaxed => { + // using value != glob::Pattern::escape(value) to check if value is a glob + // TODO: replace with proper check when https://github.com/rust-lang/glob/issues/72 is resolved + if from != glob::Pattern::escape(from) { + info!( + "Input \"{from}\" for {output_type} \"{to}\" didn’t match any components, but this was ignored because `relaxed_wildcard_matching` is enabled." + ); + return Ok(()); + } + } + WildcardMatching::Strict => {} + } info!( "Available components:\n{}", self.nodes .iter() - .map(|(key, node)| format!("\"{}\":\n {}", key, node)) + .map(|(key, node)| format!("\"{key}\":\n {node}")) .collect::>() .join("\n") ); @@ -336,7 +355,7 @@ impl Graph { for id in self.valid_inputs() { if let Some(_other) = mapped.insert(id.to_string(), id.clone()) { - errors.insert(format!("Input specifier {} is ambiguous", id)); + errors.insert(format!("Input specifier {id} is ambiguous")); } } @@ -472,9 +491,14 @@ mod test { } } - fn test_add_input(&mut self, node: &str, input: &str) -> Result<(), String> { + fn test_add_input( + &mut self, + node: &str, + input: &str, + wildcard_matching: WildcardMatching, + ) -> Result<(), String> { let available_inputs = self.input_map().unwrap(); - self.add_input(input, &node.into(), &available_inputs) + self.add_input(input, &node.into(), &available_inputs, wildcard_matching) } } @@ -655,14 +679,22 @@ mod test { // make sure we're good with dotted paths assert_eq!( Ok(()), - graph.test_add_input("errored_log_sink", "log_to_log.errors") + graph.test_add_input( + "errored_log_sink", + "log_to_log.errors", + WildcardMatching::Strict + ) ); // make sure that we're not cool with an unknown dotted path let expected = "Input \"log_to_log.not_errors\" for sink \"bad_log_sink\" doesn't match any components.".to_string(); assert_eq!( Err(expected), - graph.test_add_input("bad_log_sink", "log_to_log.not_errors") + graph.test_add_input( + "bad_log_sink", + "log_to_log.not_errors", + WildcardMatching::Strict + ) ); } @@ -745,6 +777,40 @@ mod test { ); } + #[test] + fn wildcard_matching() { + let mut graph = Graph::default(); + graph.add_source("log_source", DataType::Log); + + // don't add inputs to these yet since they're not validated via these helpers + graph.add_sink("sink", DataType::Log, vec![]); + + // make sure we're not good with non existing inputs with relaxed wildcard matching disabled + let wildcard_matching = WildcardMatching::Strict; + let expected = + "Input \"bad_source-*\" for sink \"sink\" doesn't match any components.".to_string(); + assert_eq!( + Err(expected), + graph.test_add_input("sink", "bad_source-*", wildcard_matching) + ); + + // make sure we're good with non existing inputs with relaxed wildcard matching enabled + let wildcard_matching = WildcardMatching::Relaxed; + assert_eq!( + Ok(()), + graph.test_add_input("sink", "bad_source-*", wildcard_matching) + ); + + // make sure we're not good with non existing inputs that are not wildcards even when relaxed wildcard matching is enabled + let wildcard_matching = WildcardMatching::Relaxed; + let expected = + "Input \"bad_source-1\" for sink \"sink\" doesn't match any components.".to_string(); + assert_eq!( + Err(expected), + graph.test_add_input("sink", "bad_source-1", wildcard_matching) + ); + } + #[test] fn paths_to_sink_simple() { let mut graph = Graph::default(); diff --git a/src/config/loading/config_builder.rs b/src/config/loading/config_builder.rs index 1e40d38b98..609cf2f525 100644 --- a/src/config/loading/config_builder.rs +++ b/src/config/loading/config_builder.rs @@ -3,8 +3,8 @@ use std::{collections::HashMap, io::Read}; use indexmap::IndexMap; use toml::value::Table; -use super::{deserialize_table, loader, prepare_input, secret}; use super::{ComponentHint, Process}; +use super::{deserialize_table, loader, prepare_input, secret}; use crate::config::{ ComponentKey, ConfigBuilder, EnrichmentTableOuter, SinkOuter, SourceOuter, TestDefinition, TransformOuter, @@ -63,7 +63,7 @@ impl Process for ConfigBuilderLoader { } Some(ComponentHint::EnrichmentTable) => { self.builder.enrichment_tables.extend(deserialize_table::< - IndexMap, + IndexMap>, >(table)?); } Some(ComponentHint::Test) => { diff --git a/src/config/loading/loader.rs b/src/config/loading/loader.rs index 7bc9262d60..af53d11972 100644 --- a/src/config/loading/loader.rs +++ b/src/config/loading/loader.rs @@ -3,7 +3,7 @@ use std::path::{Path, PathBuf}; use serde_toml_merge::merge_into_table; use toml::value::{Table, Value}; -use super::{component_name, open_file, read_dir, Format}; +use super::{Format, component_name, open_file, read_dir}; use crate::config::format; /// Provides a hint to the loading system of the type of components that should be found @@ -100,8 +100,7 @@ pub(super) mod process { } Err(err) => { errors.push(format!( - "Could not read entry in config dir: {:?}, {}.", - path, err + "Could not read entry in config dir: {path:?}, {err}." )); } }; @@ -136,15 +135,15 @@ pub(super) mod process { // Only descend into folders if `recurse: true`. if recurse { for entry in folders { - if let Ok(name) = component_name(&entry) { - if !result.contains_key(&name) { - match self.load_dir(&entry, true) { - Ok(table) => { - result.insert(name, Value::Table(table)); - } - Err(errs) => { - errors.extend(errs); - } + if let Ok(name) = component_name(&entry) + && !result.contains_key(&name) + { + match self.load_dir(&entry, true) { + Ok(table) => { + result.insert(name, Value::Table(table)); + } + Err(errs) => { + errors.extend(errs); } } } @@ -164,10 +163,9 @@ pub(super) mod process { path: &Path, format: Format, ) -> Result, Vec> { - if let (Ok(name), Some(file)) = (component_name(path), open_file(path)) { - self.load(file, format).map(|value| Some((name, value))) - } else { - Ok(None) + match (component_name(path), open_file(path)) { + (Ok(name), Some(file)) => self.load(file, format).map(|value| Some((name, value))), + _ => Ok(None), } } @@ -179,10 +177,11 @@ pub(super) mod process { format: Format, ) -> Result, Vec> { if let Some((name, mut table)) = self.load_file(path, format)? { - if let Some(subdir) = path.parent().map(|p| p.join(&name)) { - if subdir.is_dir() && subdir.exists() { - self.load_dir_into(&subdir, &mut table, true)?; - } + if let Some(subdir) = path.parent().map(|p| p.join(&name)) + && subdir.is_dir() + && subdir.exists() + { + self.load_dir_into(&subdir, &mut table, true)?; } Ok(Some((name, table))) } else { diff --git a/src/config/loading/mod.rs b/src/config/loading/mod.rs index a4def7e197..e4070473d0 100644 --- a/src/config/loading/mod.rs +++ b/src/config/loading/mod.rs @@ -20,7 +20,7 @@ pub use source::*; use vector_lib::configurable::NamedComponent; use super::{ - builder::ConfigBuilder, format, validation, vars, Config, ConfigPath, Format, FormatHint, + Config, ConfigPath, Format, FormatHint, builder::ConfigBuilder, format, validation, vars, }; use crate::{config::ProviderConfig, signal}; @@ -276,11 +276,17 @@ pub fn prepare_input(mut input: R) -> Result>(); - if !vars.contains_key("HOSTNAME") { - if let Ok(hostname) = crate::get_hostname() { - vars.insert("HOSTNAME".into(), hostname); - } + let mut vars: HashMap = std::env::vars_os() + .filter_map(|(k, v)| match (k.into_string(), v.into_string()) { + (Ok(k), Ok(v)) => Some((k, v)), + _ => None, + }) + .collect(); + + if !vars.contains_key("HOSTNAME") + && let Ok(hostname) = crate::get_hostname() + { + vars.insert("HOSTNAME".into(), hostname); } vars::interpolate(&source_string, &vars) } @@ -336,15 +342,21 @@ mod tests { .join("success"); let configs = vec![ConfigPath::Dir(path)]; let builder = load_builder_from_paths(&configs).unwrap(); - assert!(builder - .transforms - .contains_key(&ComponentKey::from("apache_parser"))); - assert!(builder - .sources - .contains_key(&ComponentKey::from("apache_logs"))); - assert!(builder - .sinks - .contains_key(&ComponentKey::from("es_cluster"))); + assert!( + builder + .transforms + .contains_key(&ComponentKey::from("apache_parser")) + ); + assert!( + builder + .sources + .contains_key(&ComponentKey::from("apache_logs")) + ); + assert!( + builder + .sinks + .contains_key(&ComponentKey::from("es_cluster")) + ); assert_eq!(builder.tests.len(), 2); } diff --git a/src/config/loading/secret.rs b/src/config/loading/secret.rs index 2107ec6757..57fac29238 100644 --- a/src/config/loading/secret.rs +++ b/src/config/loading/secret.rs @@ -13,8 +13,8 @@ use vector_lib::config::ComponentKey; use crate::{ config::{ - loading::{deserialize_table, prepare_input, process::Process, ComponentHint, Loader}, SecretBackend, + loading::{ComponentHint, Loader, deserialize_table, prepare_input, process::Process}, }, secrets::SecretBackends, signal, @@ -28,7 +28,7 @@ use crate::{ // - "SECRET[secret_name]" will not match // - "SECRET[.secret.name]" will not match pub static COLLECTOR: LazyLock = - LazyLock::new(|| Regex::new(r"SECRET\[([[:word:]]+)\.([[:word:].]+)\]").unwrap()); + LazyLock::new(|| Regex::new(r"SECRET\[([[:word:]]+)\.([[:word:].-]+)\]").unwrap()); /// Helper type for specifically deserializing secrets backends. #[derive(Debug, Default, Deserialize, Serialize)] @@ -201,6 +201,7 @@ mod tests { collect_secret_keys( indoc! {r" SECRET[first_backend.secret_key] + SECRET[first_backend.secret-key] SECRET[first_backend.another_secret_key] SECRET[second_backend.secret_key] SECRET[second_backend.secret.key] @@ -216,8 +217,9 @@ mod tests { assert!(keys.contains_key("second_backend")); let first_backend_keys = keys.get("first_backend").unwrap(); - assert_eq!(first_backend_keys.len(), 4); + assert_eq!(first_backend_keys.len(), 5); assert!(first_backend_keys.contains("secret_key")); + assert!(first_backend_keys.contains("secret-key")); assert!(first_backend_keys.contains("another_secret_key")); assert!(first_backend_keys.contains("a_third.secret_key")); assert!(first_backend_keys.contains("..an_extra_secret_key")); diff --git a/src/config/loading/secret_backend_example.rs b/src/config/loading/secret_backend_example.rs index 2466d7da38..5b6d848c84 100644 --- a/src/config/loading/secret_backend_example.rs +++ b/src/config/loading/secret_backend_example.rs @@ -1,6 +1,6 @@ use std::{ collections::HashMap, - io::{stdin, BufReader}, + io::{BufReader, stdin}, }; use serde::{Deserialize, Serialize}; @@ -33,7 +33,7 @@ async fn main() { ( secret.clone(), ExecResponse { - value: format!("{}.retrieved", secret), + value: format!("{secret}.retrieved"), error: None, }, ) diff --git a/src/config/mod.rs b/src/config/mod.rs index 2326a8bc0e..5cee3f4047 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -2,20 +2,27 @@ use std::{ collections::{HashMap, HashSet}, fmt::{self, Display, Formatter}, + fs, hash::Hash, net::SocketAddr, path::PathBuf, time::Duration, }; -use crate::{conditions, event::Metric, secrets::SecretBackends, serde::OneOrMany}; +use crate::{ + conditions, + event::{Metric, Value}, + secrets::SecretBackends, + serde::OneOrMany, +}; + use indexmap::IndexMap; use serde::Serialize; use vector_config::configurable_component; pub use vector_lib::config::{ AcknowledgementsConfig, DataType, GlobalOptions, Input, LogNamespace, - SourceAcknowledgementsConfig, SourceOutput, TransformOutput, + SourceAcknowledgementsConfig, SourceOutput, TransformOutput, WildcardMatching, }; pub use vector_lib::configurable::component::{ GenerateConfig, SinkDescription, TransformDescription, @@ -26,7 +33,7 @@ mod builder; mod cmd; mod compiler; mod diff; -mod dot_graph; +pub mod dot_graph; mod enrichment_table; pub mod format; mod graph; @@ -43,33 +50,74 @@ mod vars; pub mod watcher; pub use builder::ConfigBuilder; -pub use cmd::{cmd, Opts}; +pub use cmd::{Opts, cmd}; pub use diff::ConfigDiff; pub use enrichment_table::{EnrichmentTableConfig, EnrichmentTableOuter}; pub use format::{Format, FormatHint}; pub use loading::{ - load, load_builder_from_paths, load_from_paths, load_from_paths_with_provider_and_secrets, - load_from_str, load_source_from_paths, merge_path_lists, process_paths, COLLECTOR, - CONFIG_PATHS, + COLLECTOR, CONFIG_PATHS, load, load_builder_from_paths, load_from_paths, + load_from_paths_with_provider_and_secrets, load_from_str, load_source_from_paths, + merge_path_lists, process_paths, }; pub use provider::ProviderConfig; pub use secret::SecretBackend; pub use sink::{BoxedSink, SinkConfig, SinkContext, SinkHealthcheckOptions, SinkOuter}; pub use source::{BoxedSource, SourceConfig, SourceContext, SourceOuter}; pub use transform::{ - get_transform_output_ids, BoxedTransform, TransformConfig, TransformContext, TransformOuter, + BoxedTransform, TransformConfig, TransformContext, TransformOuter, get_transform_output_ids, }; -pub use unit_test::{build_unit_tests, build_unit_tests_main, UnitTestResult}; +pub use unit_test::{UnitTestResult, build_unit_tests, build_unit_tests_main}; pub use validation::warnings; -pub use vars::{interpolate, ENVIRONMENT_VARIABLE_INTERPOLATION_REGEX}; +pub use vars::{ENVIRONMENT_VARIABLE_INTERPOLATION_REGEX, interpolate}; pub use vector_lib::{ config::{ - init_log_schema, init_telemetry, log_schema, proxy::ProxyConfig, telemetry, ComponentKey, - LogSchema, OutputId, + ComponentKey, LogSchema, OutputId, init_log_schema, init_telemetry, log_schema, + proxy::ProxyConfig, telemetry, }, id::Inputs, }; +#[derive(Debug, Clone, Ord, PartialOrd, Eq, PartialEq)] +// // This is not a comprehensive set; variants are added as needed. +pub enum ComponentType { + Transform, + Sink, + EnrichmentTable, +} + +#[derive(Debug, Clone, Ord, PartialOrd, Eq, PartialEq)] +pub struct ComponentConfig { + pub config_paths: Vec, + pub component_key: ComponentKey, + pub component_type: ComponentType, +} + +impl ComponentConfig { + pub fn new( + config_paths: Vec, + component_key: ComponentKey, + component_type: ComponentType, + ) -> Self { + let canonicalized_paths = config_paths + .into_iter() + .filter_map(|p| fs::canonicalize(p).ok()) + .collect(); + + Self { + config_paths: canonicalized_paths, + component_key, + component_type, + } + } + + pub fn contains(&self, config_paths: &[PathBuf]) -> Option<(ComponentKey, ComponentType)> { + if config_paths.iter().any(|p| self.config_paths.contains(p)) { + return Some((self.component_key.clone(), self.component_type.clone())); + } + None + } +} + #[derive(Debug, Clone, Ord, PartialOrd, Eq, PartialEq)] pub enum ConfigPath { File(PathBuf, FormatHint), @@ -104,7 +152,7 @@ pub struct Config { sources: IndexMap, sinks: IndexMap>, transforms: IndexMap>, - pub enrichment_tables: IndexMap, + pub enrichment_tables: IndexMap>, tests: Vec, secret: IndexMap, pub graceful_shutdown_duration: Option, @@ -143,11 +191,22 @@ impl Config { self.sinks.get(id) } + pub fn enrichment_tables( + &self, + ) -> impl Iterator)> { + self.enrichment_tables.iter() + } + + pub fn enrichment_table(&self, id: &ComponentKey) -> Option<&EnrichmentTableOuter> { + self.enrichment_tables.get(id) + } + pub fn inputs_for_node(&self, id: &ComponentKey) -> Option<&[OutputId]> { self.transforms .get(id) .map(|t| &t.inputs[..]) .or_else(|| self.sinks.get(id).map(|s| &s.inputs[..])) + .or_else(|| self.enrichment_tables.get(id).map(|s| &s.inputs[..])) } pub fn propagate_acknowledgements(&mut self) -> Result<(), Vec> { @@ -220,7 +279,7 @@ impl HealthcheckOptions { } } - fn merge(&mut self, other: Self) { + const fn merge(&mut self, other: Self) { self.enabled &= other.enabled; self.require_healthy |= other.require_healthy; } @@ -269,10 +328,10 @@ impl Resource { // Find equality based conflicts for (key, resources) in components { for resource in resources { - if let Resource::Port(address, protocol) = &resource { - if address.ip().is_unspecified() { - unspecified.push((key.clone(), *address, *protocol)); - } + if let Resource::Port(address, protocol) = &resource + && address.ip().is_unspecified() + { + unspecified.push((key.clone(), *address, *protocol)); } resource_map @@ -316,10 +375,10 @@ impl Display for Protocol { impl Display for Resource { fn fmt(&self, fmt: &mut Formatter<'_>) -> Result<(), fmt::Error> { match self { - Resource::Port(address, protocol) => write!(fmt, "{} {}", protocol, address), + Resource::Port(address, protocol) => write!(fmt, "{protocol} {address}"), Resource::SystemFdOffset(offset) => write!(fmt, "systemd {}th socket", offset + 1), - Resource::Fd(fd) => write!(fmt, "file descriptor: {}", fd), - Resource::DiskBuffer(name) => write!(fmt, "disk buffer {:?}", name), + Resource::Fd(fd) => write!(fmt, "file descriptor: {fd}"), + Resource::DiskBuffer(name) => write!(fmt, "disk buffer {name:?}"), } } } @@ -381,8 +440,7 @@ impl TestDefinition { outputs.push(output_id.clone()); } else { errors.push(format!( - r#"Invalid extract_from target in test '{}': '{}' does not exist"#, - name, from + r#"Invalid extract_from target in test '{name}': '{from}' does not exist"# )); } } @@ -404,8 +462,7 @@ impl TestDefinition { Some(output_id.clone()) } else { errors.push(format!( - r#"Invalid no_outputs_from target in test '{}': '{}' does not exist"#, - name, o + r#"Invalid no_outputs_from target in test '{name}': '{o}' does not exist"# )); None } @@ -462,24 +519,6 @@ impl TestDefinition { } } -/// Value for a log field. -#[configurable_component] -#[derive(Clone, Debug)] -#[serde(untagged)] -pub enum TestInputValue { - /// A string. - String(String), - - /// An integer. - Integer(i64), - - /// A floating-point number. - Float(f64), - - /// A boolean. - Boolean(bool), -} - /// A unit test input. /// /// An input describes not only the type of event to insert, but also which transform within the @@ -511,7 +550,7 @@ pub struct TestInput { /// The set of log fields to use when creating a log input event. /// /// Only relevant when `type` is `log`. - pub log_fields: Option>, + pub log_fields: Option>, /// The metric to use as an input event. /// @@ -545,7 +584,7 @@ mod tests { use crate::{config, topology}; use indoc::indoc; - use super::{builder::ConfigBuilder, format, load_from_str, ComponentKey, ConfigDiff, Format}; + use super::{ComponentKey, ConfigDiff, Format, builder::ConfigBuilder, format, load_from_str}; async fn load(config: &str, format: config::Format) -> Result, Vec> { match config::load_from_str(config, format) { @@ -1295,12 +1334,13 @@ mod resource_config_tests { use indoc::indoc; use vector_lib::configurable::schema::generate_root_schema; - use super::{load_from_str, Format}; + use super::{Format, load_from_str}; #[test] fn config_conflict_detected() { - assert!(load_from_str( - indoc! {r#" + assert!( + load_from_str( + indoc! {r#" [sources.in0] type = "stdin" @@ -1312,9 +1352,10 @@ mod resource_config_tests { inputs = ["in0","in1"] encoding.codec = "json" "#}, - Format::Toml, - ) - .is_err()); + Format::Toml, + ) + .is_err() + ); } #[test] @@ -1349,9 +1390,9 @@ mod resource_config_tests { let json = serde_json::to_string_pretty(&schema) .expect("rendering root schema to JSON should not fail"); - println!("{}", json); + println!("{json}"); } - Err(e) => eprintln!("error while generating schema: {:?}", e), + Err(e) => eprintln!("error while generating schema: {e:?}"), } } } diff --git a/src/config/schema.rs b/src/config/schema.rs index 2245e84e75..bed2c0f532 100644 --- a/src/config/schema.rs +++ b/src/config/schema.rs @@ -147,13 +147,12 @@ mod test { let mut errors = vec![]; a.append(b, &mut errors); if errors.is_empty() { - assert_eq!(Some(a), expected, "result mismatch: {}", test); + assert_eq!(Some(a), expected, "result mismatch: {test}"); } else { assert_eq!( errors.is_empty(), expected.is_some(), - "error mismatch: {}", - test + "error mismatch: {test}" ); } } diff --git a/src/config/sink.rs b/src/config/sink.rs index a322863f11..7d4f38de6e 100644 --- a/src/config/sink.rs +++ b/src/config/sink.rs @@ -1,13 +1,16 @@ use std::cell::RefCell; +use std::time::Duration; use async_trait::async_trait; use dyn_clone::DynClone; use serde::Serialize; +use serde_with::serde_as; +use std::path::PathBuf; use vector_lib::buffers::{BufferConfig, BufferType}; use vector_lib::configurable::attributes::CustomAttribute; use vector_lib::configurable::schema::{SchemaGenerator, SchemaObject}; use vector_lib::configurable::{ - configurable_component, Configurable, GenerateError, Metadata, NamedComponent, + Configurable, GenerateError, Metadata, NamedComponent, configurable_component, }; use vector_lib::{ config::{AcknowledgementsConfig, GlobalOptions, Input}, @@ -15,9 +18,9 @@ use vector_lib::{ sink::VectorSink, }; -use super::{dot_graph::GraphConfig, schema, ComponentKey, ProxyConfig, Resource}; +use super::{ComponentKey, ProxyConfig, Resource, dot_graph::GraphConfig, schema}; use crate::extra_context::ExtraContext; -use crate::sinks::{util::UriSerde, Healthcheck}; +use crate::sinks::{Healthcheck, util::UriSerde}; pub type BoxedSink = Box; @@ -34,8 +37,10 @@ impl Configurable for BoxedSink { metadata } - fn generate_schema(gen: &RefCell) -> Result { - vector_lib::configurable::component::SinkDescription::generate_schemas(gen) + fn generate_schema( + generator: &RefCell, + ) -> Result { + vector_lib::configurable::component::SinkDescription::generate_schemas(generator) } } @@ -65,11 +70,11 @@ where /// This must be a valid URI, which requires at least the scheme and host. All other /// components -- port, path, etc -- are allowed as well. #[configurable(deprecated, metadata(docs::hidden), validation(format = "uri"))] - healthcheck_uri: Option, + pub healthcheck_uri: Option, #[configurable(derived, metadata(docs::advanced))] #[serde(default, deserialize_with = "crate::serde::bool_or_struct")] - healthcheck: SinkHealthcheckOptions, + pub healthcheck: SinkHealthcheckOptions, #[configurable(derived)] #[serde(default, skip_serializing_if = "vector_lib::serde::is_default")] @@ -77,7 +82,7 @@ where #[configurable(derived)] #[serde(default, skip_serializing_if = "vector_lib::serde::is_default")] - proxy: ProxyConfig, + pub proxy: ProxyConfig, #[serde(flatten)] #[configurable(metadata(docs::hidden))] @@ -117,7 +122,9 @@ where pub fn healthcheck(&self) -> SinkHealthcheckOptions { if self.healthcheck_uri.is_some() && self.healthcheck.uri.is_some() { - warn!("Both `healthcheck.uri` and `healthcheck_uri` options are specified. Using value of `healthcheck.uri`.") + warn!( + "Both `healthcheck.uri` and `healthcheck_uri` options are specified. Using value of `healthcheck.uri`." + ) } else if self.healthcheck_uri.is_some() { warn!( "The `healthcheck_uri` option has been deprecated, use `healthcheck.uri` instead." @@ -163,6 +170,7 @@ where } /// Healthcheck configuration. +#[serde_as] #[configurable_component] #[derive(Clone, Debug)] #[serde(default)] @@ -170,6 +178,14 @@ pub struct SinkHealthcheckOptions { /// Whether or not to check the health of the sink when Vector starts up. pub enabled: bool, + /// Timeout duration for healthcheck in seconds. + #[serde_as(as = "serde_with::DurationSecondsWithFrac")] + #[serde( + default = "default_healthcheck_timeout", + skip_serializing_if = "is_default_healthcheck_timeout" + )] + pub timeout: Duration, + /// The full URI to make HTTP healthcheck requests to. /// /// This must be a valid URI, which requires at least the scheme and host. All other @@ -178,26 +194,38 @@ pub struct SinkHealthcheckOptions { pub uri: Option, } +const fn default_healthcheck_timeout() -> Duration { + Duration::from_secs(10) +} + +fn is_default_healthcheck_timeout(timeout: &Duration) -> bool { + timeout == &default_healthcheck_timeout() +} + impl Default for SinkHealthcheckOptions { fn default() -> Self { Self { enabled: true, uri: None, + timeout: default_healthcheck_timeout(), } } } impl From for SinkHealthcheckOptions { fn from(enabled: bool) -> Self { - Self { enabled, uri: None } + Self { + enabled, + ..Default::default() + } } } impl From for SinkHealthcheckOptions { fn from(uri: UriSerde) -> Self { Self { - enabled: true, uri: Some(uri), + ..Default::default() } } } @@ -220,6 +248,11 @@ pub trait SinkConfig: DynClone + NamedComponent + core::fmt::Debug + Send + Sync /// Gets the input configuration for this sink. fn input(&self) -> Input; + /// Gets the files to watch to trigger reload + fn files_to_watch(&self) -> Vec<&PathBuf> { + Vec::new() + } + /// Gets the list of resources, if any, used by this sink. /// /// Resources represent dependencies -- network ports, file descriptors, and so on -- that @@ -237,10 +270,11 @@ pub trait SinkConfig: DynClone + NamedComponent + core::fmt::Debug + Send + Sync dyn_clone::clone_trait_object!(SinkConfig); -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct SinkContext { pub healthcheck: SinkHealthcheckOptions, pub globals: GlobalOptions, + pub enrichment_tables: vector_lib::enrichment::TableRegistry, pub proxy: ProxyConfig, pub schema: schema::Options, pub app_name: String, @@ -256,6 +290,7 @@ impl Default for SinkContext { Self { healthcheck: Default::default(), globals: Default::default(), + enrichment_tables: Default::default(), proxy: Default::default(), schema: Default::default(), app_name: crate::get_app_name().to_string(), diff --git a/src/config/source.rs b/src/config/source.rs index 1ad0e46de3..1b47519273 100644 --- a/src/config/source.rs +++ b/src/config/source.rs @@ -15,8 +15,8 @@ use vector_lib::{ source::Source, }; -use super::{dot_graph::GraphConfig, schema, ComponentKey, ProxyConfig, Resource}; -use crate::{extra_context::ExtraContext, shutdown::ShutdownSignal, SourceSender}; +use super::{ComponentKey, ProxyConfig, Resource, dot_graph::GraphConfig, schema}; +use crate::{SourceSender, extra_context::ExtraContext, shutdown::ShutdownSignal}; pub type BoxedSource = Box; @@ -33,8 +33,10 @@ impl Configurable for BoxedSource { metadata } - fn generate_schema(gen: &RefCell) -> Result { - vector_lib::configurable::component::SourceDescription::generate_schemas(gen) + fn generate_schema( + generator: &RefCell, + ) -> Result { + vector_lib::configurable::component::SourceDescription::generate_schemas(generator) } } @@ -124,6 +126,7 @@ dyn_clone::clone_trait_object!(SourceConfig); pub struct SourceContext { pub key: ComponentKey, pub globals: GlobalOptions, + pub enrichment_tables: vector_lib::enrichment::TableRegistry, pub shutdown: ShutdownSignal, pub out: SourceSender, pub proxy: ProxyConfig, @@ -153,6 +156,7 @@ impl SourceContext { Self { key: key.clone(), globals: GlobalOptions::default(), + enrichment_tables: Default::default(), shutdown: shutdown_signal, out, proxy: Default::default(), @@ -173,6 +177,7 @@ impl SourceContext { Self { key: ComponentKey::from("default"), globals: GlobalOptions::default(), + enrichment_tables: Default::default(), shutdown: ShutdownSignal::noop(), out, proxy: Default::default(), diff --git a/src/config/transform.rs b/src/config/transform.rs index 4ab7e21f99..9f75caf6a9 100644 --- a/src/config/transform.rs +++ b/src/config/transform.rs @@ -1,14 +1,14 @@ use std::cell::RefCell; use std::collections::{HashMap, HashSet}; +use std::path::PathBuf; use async_trait::async_trait; use dyn_clone::DynClone; use serde::Serialize; use vector_lib::configurable::attributes::CustomAttribute; use vector_lib::configurable::{ - configurable_component, + Configurable, GenerateError, Metadata, NamedComponent, configurable_component, schema::{SchemaGenerator, SchemaObject}, - Configurable, GenerateError, Metadata, NamedComponent, }; use vector_lib::{ config::{GlobalOptions, Input, LogNamespace, TransformOutput}, @@ -17,10 +17,10 @@ use vector_lib::{ transform::Transform, }; -use super::dot_graph::GraphConfig; -use super::schema::Options as SchemaOptions; use super::ComponentKey; use super::OutputId; +use super::dot_graph::GraphConfig; +use super::schema::Options as SchemaOptions; use crate::extra_context::ExtraContext; pub type BoxedTransform = Box; @@ -38,8 +38,10 @@ impl Configurable for BoxedTransform { metadata } - fn generate_schema(gen: &RefCell) -> Result { - vector_lib::configurable::component::TransformDescription::generate_schemas(gen) + fn generate_schema( + generator: &RefCell, + ) -> Result { + vector_lib::configurable::component::TransformDescription::generate_schemas(generator) } } @@ -253,6 +255,11 @@ pub trait TransformConfig: DynClone + NamedComponent + core::fmt::Debug + Send + fn nestable(&self, _parents: &HashSet<&'static str>) -> bool { true } + + /// Gets the files to watch to trigger reload + fn files_to_watch(&self) -> Vec<&PathBuf> { + Vec::new() + } } dyn_clone::clone_trait_object!(TransformConfig); diff --git a/src/config/unit_test/mod.rs b/src/config/unit_test/mod.rs index 5354fbe316..1c5aa2d534 100644 --- a/src/config/unit_test/mod.rs +++ b/src/config/unit_test/mod.rs @@ -16,16 +16,15 @@ use std::{ sync::Arc, }; -use futures_util::{stream::FuturesUnordered, StreamExt}; +use futures_util::{StreamExt, stream::FuturesUnordered}; use indexmap::IndexMap; -use ordered_float::NotNan; use tokio::sync::{ - oneshot::{self, Receiver}, Mutex, + oneshot::{self, Receiver}, }; use uuid::Uuid; use vrl::{ - compiler::{state::RuntimeState, Context, TargetValue, TimeZone}, + compiler::{Context, TargetValue, TimeZone, state::RuntimeState}, diagnostic::Formatter, value, }; @@ -34,16 +33,16 @@ pub use self::unit_test_components::{ UnitTestSinkCheck, UnitTestSinkConfig, UnitTestSinkResult, UnitTestSourceConfig, UnitTestStreamSinkConfig, UnitTestStreamSourceConfig, }; -use super::{compiler::expand_globs, graph::Graph, transform::get_transform_output_ids, OutputId}; +use super::{OutputId, compiler::expand_globs, graph::Graph, transform::get_transform_output_ids}; use crate::{ conditions::Condition, config::{ - self, loading, ComponentKey, Config, ConfigBuilder, ConfigPath, SinkOuter, SourceOuter, - TestDefinition, TestInput, TestInputValue, TestOutput, + self, ComponentKey, Config, ConfigBuilder, ConfigPath, SinkOuter, SourceOuter, + TestDefinition, TestInput, TestOutput, loading, }, - event::{Event, EventMetadata, LogEvent, Value}, + event::{Event, EventMetadata, LogEvent}, signal, - topology::{builder::TopologyPieces, RunningTopology}, + topology::{RunningTopology, builder::TopologyPieces}, }; pub struct UnitTest { @@ -140,7 +139,7 @@ pub async fn build_unit_tests( let mut test_error = errors.join("\n"); // Indent all line breaks test_error = test_error.replace('\n', "\n "); - test_error.insert_str(0, &format!("Failed to build test '{}':\n ", test_name)); + test_error.insert_str(0, &format!("Failed to build test '{test_name}':\n ")); build_errors.push(test_error); } } @@ -386,6 +385,10 @@ async fn build_unit_test( &transform_only_config.transforms, &transform_only_config.sinks, transform_only_config.schema, + transform_only_config + .global + .wildcard_matching + .unwrap_or_default(), ); let test = test.resolve_outputs(&transform_only_graph)?; @@ -402,6 +405,7 @@ async fn build_unit_test( &config_builder.transforms, &config_builder.sinks, config_builder.schema, + config_builder.global.wildcard_matching.unwrap_or_default(), ); let mut valid_components = get_relevant_test_components( @@ -433,6 +437,7 @@ async fn build_unit_test( &config_builder.transforms, &config_builder.sinks, config_builder.schema, + config_builder.global.wildcard_matching.unwrap_or_default(), ); let valid_inputs = graph.input_map()?; for (_, transform) in config_builder.transforms.iter_mut() { @@ -563,8 +568,7 @@ fn build_outputs( match condition.build(&Default::default()) { Ok(condition) => conditions.push(condition), Err(error) => errors.push(format!( - "failed to create test condition '{}': {}", - index, error + "failed to create test condition '{index}': {error}" )), } } @@ -622,16 +626,8 @@ fn build_input_event(input: &TestInput) -> Result { if let Some(log_fields) = &input.log_fields { let mut event = LogEvent::from_str_legacy(""); for (path, value) in log_fields { - let value: Value = match value { - TestInputValue::String(s) => Value::from(s.to_owned()), - TestInputValue::Boolean(b) => Value::from(*b), - TestInputValue::Integer(i) => Value::from(*i), - TestInputValue::Float(f) => Value::from( - NotNan::new(*f).map_err(|_| "NaN value not supported".to_string())?, - ), - }; event - .parse_path_and_insert(path, value) + .parse_path_and_insert(path, value.clone()) .map_err(|e| e.to_string())?; } Ok(event.into()) diff --git a/src/config/unit_test/tests.rs b/src/config/unit_test/tests.rs index 2172a43bb1..938286ac9b 100644 --- a/src/config/unit_test/tests.rs +++ b/src/config/unit_test/tests.rs @@ -33,10 +33,12 @@ async fn parse_no_input() { let errs = build_unit_tests(config).await.err().unwrap(); assert_eq!( errs, - vec![indoc! {r" + vec![ + indoc! {r" Failed to build test 'broken test': inputs[0]: unable to locate target transform 'foo'"} - .to_owned(),] + .to_owned(), + ] ); let config: ConfigBuilder = toml::from_str(indoc! {r#" @@ -69,10 +71,12 @@ async fn parse_no_input() { let errs = build_unit_tests(config).await.err().unwrap(); assert_eq!( errs, - vec![indoc! {r" + vec![ + indoc! {r" Failed to build test 'broken test': inputs[1]: unable to locate target transform 'foo'"} - .to_owned(),] + .to_owned(), + ] ); } @@ -102,10 +106,12 @@ async fn parse_no_test_input() { let errs = build_unit_tests(config).await.err().unwrap(); assert_eq!( errs, - vec![indoc! {r" + vec![ + indoc! {r" Failed to build test 'broken test': must specify at least one input."} - .to_owned(),] + .to_owned(), + ] ); } @@ -133,10 +139,12 @@ async fn parse_no_outputs() { let errs = build_unit_tests(config).await.err().unwrap(); assert_eq!( errs, - vec![indoc! {r" + vec![ + indoc! {r" Failed to build test 'broken test': unit test must contain at least one of `outputs` or `no_outputs_from`."} - .to_owned(),] + .to_owned(), + ] ); } @@ -170,10 +178,12 @@ async fn parse_invalid_output_targets() { let errs = build_unit_tests(config).await.err().unwrap(); assert_eq!( errs, - vec![indoc! {r" + vec![ + indoc! {r" Failed to build test 'broken test': Invalid extract_from target in test 'broken test': 'nonexistent' does not exist"} - .to_owned(),] + .to_owned(), + ] ); let config: ConfigBuilder = toml::from_str(indoc! {r#" @@ -197,10 +207,12 @@ async fn parse_invalid_output_targets() { let errs = build_unit_tests(config).await.err().unwrap(); assert_eq!( errs, - vec![indoc! {r" + vec![ + indoc! {r" Failed to build test 'broken test': Invalid no_outputs_from target in test 'broken test': 'nonexistent' does not exist"} - .to_owned(),] + .to_owned(), + ] ); } @@ -338,10 +350,12 @@ async fn parse_bad_input_event() { let errs = build_unit_tests(config).await.err().unwrap(); assert_eq!( errs, - vec![indoc! {r" + vec![ + indoc! {r" Failed to build test 'broken test': unrecognized input type 'nah', expected one of: 'raw', 'log' or 'metric'"} - .to_owned(),] + .to_owned(), + ] ); } @@ -805,6 +819,9 @@ async fn test_log_input() { message = "this is the message" int_val = 5 bool_val = true + arr_val = [1, 2, "hi", false] + obj_val = { a = true, b = "b", c = 5 } + [[tests.outputs]] extract_from = "foo" @@ -816,6 +833,10 @@ async fn test_log_input() { assert_eq!(.message, "this is the message") assert!(.bool_val) assert_eq!(.int_val, 5) + assert_eq!(.arr_val, [1, 2, "hi", false]) + assert!(.obj_val.a) + assert_eq!(.obj_val.b, "b") + assert_eq!(.obj_val.c, 5) """ "#}) .unwrap(); diff --git a/src/config/unit_test/unit_test_components.rs b/src/config/unit_test/unit_test_components.rs index d478c80c87..5ba6fe4d18 100644 --- a/src/config/unit_test/unit_test_components.rs +++ b/src/config/unit_test/unit_test_components.rs @@ -1,8 +1,8 @@ use std::sync::Arc; -use futures::{stream, Sink, Stream}; -use futures_util::{future, stream::BoxStream, FutureExt, StreamExt}; -use tokio::sync::{oneshot, Mutex}; +use futures::{Sink, Stream, stream}; +use futures_util::{FutureExt, StreamExt, future, stream::BoxStream}; +use tokio::sync::{Mutex, oneshot}; use vector_lib::configurable::configurable_component; use vector_lib::{ config::{DataType, Input, LogNamespace}, @@ -220,8 +220,7 @@ impl StreamSink for UnitTestSink { break; } Err(error) => { - condition_errors - .push(format!(" condition[{}]: {}", j, error)); + condition_errors.push(format!(" condition[{j}]: {error}")); } } } @@ -262,10 +261,10 @@ impl StreamSink for UnitTestSink { UnitTestSinkCheck::NoOp => {} } - if let Some(tx) = self.result_tx { - if tx.send(result).is_err() { - error!(message = "Sending unit test results failed in unit test sink."); - } + if let Some(tx) = self.result_tx + && tx.send(result).is_err() + { + error!(message = "Sending unit test results failed in unit test sink."); } Ok(()) } diff --git a/src/config/validation.rs b/src/config/validation.rs index 56423e6aa7..8011ea4146 100644 --- a/src/config/validation.rs +++ b/src/config/validation.rs @@ -1,13 +1,13 @@ use crate::config::schema; -use futures_util::{stream, FutureExt, StreamExt, TryFutureExt, TryStreamExt}; +use futures_util::{FutureExt, StreamExt, TryFutureExt, TryStreamExt, stream}; use heim::{disk::Partition, units::information::byte}; use indexmap::IndexMap; use std::{collections::HashMap, path::PathBuf}; use vector_lib::{buffers::config::DiskUsage, internal_event::DEFAULT_OUTPUT}; use super::{ - builder::ConfigBuilder, transform::get_transform_output_ids, ComponentKey, Config, OutputId, - Resource, + ComponentKey, Config, OutputId, Resource, builder::ConfigBuilder, + transform::get_transform_output_ids, }; /// Check that provide + topology config aren't present in the same builder, which is an error. @@ -140,10 +140,7 @@ pub fn check_resources(config: &ConfigBuilder) -> Result<(), Vec> { Err(conflicting_components .into_iter() .map(|(resource, components)| { - format!( - "Resource `{}` is claimed by multiple components: {:?}", - resource, components - ) + format!("Resource `{resource}` is claimed by multiple components: {components:?}") }) .collect()) } @@ -325,20 +322,29 @@ async fn process_partitions(partitions: Vec) -> heim::Result Vec { let mut warnings = vec![]; - let source_ids = config.sources.iter().flat_map(|(key, source)| { - source - .inner - .outputs(config.schema.log_namespace()) - .iter() - .map(|output| { - if let Some(port) = &output.port { - ("source", OutputId::from((key, port.clone()))) - } else { - ("source", OutputId::from(key)) - } - }) - .collect::>() - }); + let table_sources = config + .enrichment_tables + .iter() + .filter_map(|(key, table)| table.as_source(key)) + .collect::>(); + let source_ids = config + .sources + .iter() + .chain(table_sources.iter().map(|(k, s)| (k, s))) + .flat_map(|(key, source)| { + source + .inner + .outputs(config.schema.log_namespace()) + .iter() + .map(|output| { + if let Some(port) = &output.port { + ("source", OutputId::from((key, port.clone()))) + } else { + ("source", OutputId::from(key)) + } + }) + .collect::>() + }); let transform_ids = config.transforms.iter().flat_map(|(key, transform)| { get_transform_output_ids( transform.inner.as_ref(), @@ -349,6 +355,11 @@ pub fn warnings(config: &Config) -> Vec { .collect::>() }); + let table_sinks = config + .enrichment_tables + .iter() + .filter_map(|(key, table)| table.as_sink(key)) + .collect::>(); for (input_type, id) in transform_ids.chain(source_ids) { if !config .transforms @@ -358,6 +369,9 @@ pub fn warnings(config: &Config) -> Vec { .sinks .iter() .any(|(_, sink)| sink.inputs.contains(&id)) + && !table_sinks + .iter() + .any(|(_, sink)| sink.inputs.contains(&id)) { warnings.push(format!( "{} \"{}\" has no consumers", diff --git a/src/config/watcher.rs b/src/config/watcher.rs index 269b127f89..26d0133890 100644 --- a/src/config/watcher.rs +++ b/src/config/watcher.rs @@ -1,13 +1,15 @@ +use crate::config::{ComponentConfig, ComponentType}; +use std::collections::HashMap; use std::{ path::{Path, PathBuf}, time::Duration, }; use std::{ - sync::mpsc::{channel, Receiver}, + sync::mpsc::{Receiver, channel}, thread, }; -use notify::{recommended_watcher, EventKind, RecursiveMode}; +use notify::{EventKind, RecursiveMode, recommended_watcher}; use crate::Error; @@ -59,7 +61,7 @@ impl Watcher { } } -/// Sends a ReloadFromDisk on config_path changes. +/// Sends a ReloadFromDisk or ReloadEnrichmentTables on config_path changes. /// Accumulates file changes until no change for given duration has occurred. /// Has best effort guarantee of detecting all file changes from the end of /// this function until the main thread stops. @@ -67,9 +69,18 @@ pub fn spawn_thread<'a>( watcher_conf: WatcherConfig, signal_tx: crate::signal::SignalTx, config_paths: impl IntoIterator + 'a, + component_configs: Vec, delay: impl Into>, ) -> Result<(), Error> { - let config_paths: Vec<_> = config_paths.into_iter().cloned().collect(); + let mut config_paths: Vec<_> = config_paths.into_iter().cloned().collect(); + let mut component_config_paths: Vec<_> = component_configs + .clone() + .into_iter() + .flat_map(|p| p.config_paths.clone()) + .collect(); + + config_paths.append(&mut component_config_paths); + let delay = delay.into().unwrap_or(CONFIG_WATCH_DELAY); // Create watcher now so not to miss any changes happening between @@ -78,53 +89,86 @@ pub fn spawn_thread<'a>( info!("Watching configuration files."); - thread::spawn(move || loop { - if let Some((mut watcher, receiver)) = watcher.take() { - while let Ok(Ok(event)) = receiver.recv() { - if matches!( - event.kind, - EventKind::Create(_) | EventKind::Remove(_) | EventKind::Modify(_) - ) { - debug!(message = "Configuration file change detected.", event = ?event); - - // Consume events until delay amount of time has passed since the latest event. - while receiver.recv_timeout(delay).is_ok() {} - - debug!(message = "Consumed file change events for delay.", delay = ?delay); - - // We need to read paths to resolve any inode changes that may have happened. - // And we need to do it before raising sighup to avoid missing any change. - if let Err(error) = watcher.add_paths(&config_paths) { - error!(message = "Failed to read files to watch.", %error); - break; + thread::spawn(move || { + loop { + if let Some((mut watcher, receiver)) = watcher.take() { + while let Ok(Ok(event)) = receiver.recv() { + if matches!( + event.kind, + EventKind::Create(_) | EventKind::Remove(_) | EventKind::Modify(_) + ) { + debug!(message = "Configuration file change detected.", event = ?event); + + // Consume events until delay amount of time has passed since the latest event. + while receiver.recv_timeout(delay).is_ok() {} + + debug!(message = "Consumed file change events for delay.", delay = ?delay); + + let changed_components: HashMap<_, _> = component_configs + .clone() + .into_iter() + .flat_map(|p| p.contains(&event.paths)) + .collect(); + + // We need to read paths to resolve any inode changes that may have happened. + // And we need to do it before raising sighup to avoid missing any change. + if let Err(error) = watcher.add_paths(&config_paths) { + error!(message = "Failed to read files to watch.", %error); + break; + } + + debug!(message = "Reloaded paths."); + + info!("Configuration file changed."); + if !changed_components.is_empty() { + info!( + internal_log_rate_limit = true, + "Component {:?} configuration changed.", + changed_components.keys() + ); + if changed_components + .iter() + .all(|(_, t)| *t == ComponentType::EnrichmentTable) + { + info!( + internal_log_rate_limit = true, + "Only enrichment tables have changed." + ); + _ = signal_tx.send(crate::signal::SignalTo::ReloadEnrichmentTables).map_err(|error| { + error!(message = "Unable to reload enrichment tables.", cause = %error, internal_log_rate_limit = true) + }); + } else { + _ = signal_tx.send(crate::signal::SignalTo::ReloadComponents(changed_components.into_keys().collect())).map_err(|error| { + error!(message = "Unable to reload component configuration. Restart Vector to reload it.", cause = %error, internal_log_rate_limit = true) + }); + } + } else { + _ = signal_tx.send(crate::signal::SignalTo::ReloadFromDisk) + .map_err(|error| { + error!(message = "Unable to reload configuration file. Restart Vector to reload it.", cause = %error, internal_log_rate_limit = true) + }); + } + } else { + debug!(message = "Ignoring event.", event = ?event) } - - debug!(message = "Reloaded paths."); - - info!("Configuration file changed."); - _ = signal_tx.send(crate::signal::SignalTo::ReloadFromDisk).map_err(|error| { - error!(message = "Unable to reload configuration file. Restart Vector to reload it.", cause = %error) - }); - } else { - debug!(message = "Ignoring event.", event = ?event) } } - } - thread::sleep(RETRY_TIMEOUT); + thread::sleep(RETRY_TIMEOUT); - watcher = create_watcher(&watcher_conf, &config_paths) - .map_err(|error| error!(message = "Failed to create file watcher.", %error)) - .ok(); + watcher = create_watcher(&watcher_conf, &config_paths) + .map_err(|error| error!(message = "Failed to create file watcher.", %error)) + .ok(); - if watcher.is_some() { - // Config files could have changed while we weren't watching, - // so for a good measure raise SIGHUP and let reload logic - // determine if anything changed. - info!("Speculating that configuration files have changed."); - _ = signal_tx.send(crate::signal::SignalTo::ReloadFromDisk).map_err(|error| { + if watcher.is_some() { + // Config files could have changed while we weren't watching, + // so for a good measure raise SIGHUP and let reload logic + // determine if anything changed. + info!("Speculating that configuration files have changed."); + _ = signal_tx.send(crate::signal::SignalTo::ReloadFromDisk).map_err(|error| { error!(message = "Unable to reload configuration file. Restart Vector to reload it.", cause = %error) }); + } } }); @@ -158,22 +202,89 @@ fn create_watcher( mod tests { use super::*; use crate::{ + config::ComponentKey, signal::SignalRx, test_util::{temp_dir, temp_file, trace_init}, }; - use std::{fs::File, io::Write, time::Duration}; + use std::{collections::HashSet, fs::File, io::Write, time::Duration}; use tokio::sync::broadcast; - async fn test(file: &mut File, timeout: Duration, mut receiver: SignalRx) -> bool { + async fn test_signal( + file: &mut File, + expected_signal: crate::signal::SignalTo, + timeout: Duration, + mut receiver: SignalRx, + ) -> bool { file.write_all(&[0]).unwrap(); file.sync_all().unwrap(); - matches!( - tokio::time::timeout(timeout, receiver.recv()).await, - Ok(Ok(crate::signal::SignalTo::ReloadFromDisk)) - ) + match tokio::time::timeout(timeout, receiver.recv()).await { + Ok(Ok(signal)) => signal == expected_signal, + _ => false, + } } + #[tokio::test] + async fn component_update() { + trace_init(); + + let delay = Duration::from_secs(3); + let dir = temp_dir().to_path_buf(); + let watcher_conf = WatcherConfig::RecommendedWatcher; + let component_file_path = vec![dir.join("tls.cert"), dir.join("tls.key")]; + let http_component = ComponentKey::from("http"); + + std::fs::create_dir(&dir).unwrap(); + + let mut component_files: Vec = component_file_path + .iter() + .map(|file| File::create(file).unwrap()) + .collect(); + let component_config = ComponentConfig::new( + component_file_path.clone(), + http_component.clone(), + ComponentType::Sink, + ); + + let (signal_tx, signal_rx) = broadcast::channel(128); + spawn_thread( + watcher_conf, + signal_tx, + &[dir], + vec![component_config], + delay, + ) + .unwrap(); + + let signal_rx = signal_rx.resubscribe(); + let signal_rx2 = signal_rx.resubscribe(); + + if !test_signal( + &mut component_files[0], + crate::signal::SignalTo::ReloadComponents(HashSet::from_iter(vec![ + http_component.clone(), + ])), + delay * 5, + signal_rx, + ) + .await + { + panic!("Test timed out"); + } + + if !test_signal( + &mut component_files[1], + crate::signal::SignalTo::ReloadComponents(HashSet::from_iter(vec![ + http_component.clone(), + ])), + delay * 5, + signal_rx2, + ) + .await + { + panic!("Test timed out"); + } + } #[tokio::test] async fn file_directory_update() { trace_init(); @@ -187,9 +298,16 @@ mod tests { let mut file = File::create(&file_path).unwrap(); let (signal_tx, signal_rx) = broadcast::channel(128); - spawn_thread(watcher_conf, signal_tx, &[dir], delay).unwrap(); + spawn_thread(watcher_conf, signal_tx, &[dir], vec![], delay).unwrap(); - if !test(&mut file, delay * 5, signal_rx).await { + if !test_signal( + &mut file, + crate::signal::SignalTo::ReloadFromDisk, + delay * 5, + signal_rx, + ) + .await + { panic!("Test timed out"); } } @@ -204,9 +322,16 @@ mod tests { let watcher_conf = WatcherConfig::RecommendedWatcher; let (signal_tx, signal_rx) = broadcast::channel(128); - spawn_thread(watcher_conf, signal_tx, &[file_path], delay).unwrap(); + spawn_thread(watcher_conf, signal_tx, &[file_path], vec![], delay).unwrap(); - if !test(&mut file, delay * 5, signal_rx).await { + if !test_signal( + &mut file, + crate::signal::SignalTo::ReloadFromDisk, + delay * 5, + signal_rx, + ) + .await + { panic!("Test timed out"); } } @@ -225,9 +350,16 @@ mod tests { let watcher_conf = WatcherConfig::RecommendedWatcher; let (signal_tx, signal_rx) = broadcast::channel(128); - spawn_thread(watcher_conf, signal_tx, &[sym_file], delay).unwrap(); + spawn_thread(watcher_conf, signal_tx, &[sym_file], vec![], delay).unwrap(); - if !test(&mut file, delay * 5, signal_rx).await { + if !test_signal( + &mut file, + crate::signal::SignalTo::ReloadFromDisk, + delay * 5, + signal_rx, + ) + .await + { panic!("Test timed out"); } } @@ -246,9 +378,16 @@ mod tests { let mut file = File::create(&file_path).unwrap(); let (signal_tx, signal_rx) = broadcast::channel(128); - spawn_thread(watcher_conf, signal_tx, &[sub_dir], delay).unwrap(); + spawn_thread(watcher_conf, signal_tx, &[sub_dir], vec![], delay).unwrap(); - if !test(&mut file, delay * 5, signal_rx).await { + if !test_signal( + &mut file, + crate::signal::SignalTo::ReloadFromDisk, + delay * 5, + signal_rx, + ) + .await + { panic!("Test timed out"); } } diff --git a/src/convert_config.rs b/src/convert_config.rs index ec80424c12..703b41db11 100644 --- a/src/convert_config.rs +++ b/src/convert_config.rs @@ -1,4 +1,4 @@ -use crate::config::{format, ConfigBuilder, Format}; +use crate::config::{ConfigBuilder, Format, format}; use clap::Parser; use colored::*; use std::fs; @@ -58,12 +58,12 @@ pub(crate) fn cmd(opts: &Opts) -> exitcode::ExitCode { } if opts.input_path.is_file() && opts.output_path.extension().is_some() { - if let Some(base_dir) = opts.output_path.parent() { - if !base_dir.exists() { - fs::create_dir_all(base_dir).unwrap_or_else(|_| { - panic!("Failed to create output dir(s): {:?}", &opts.output_path) - }); - } + if let Some(base_dir) = opts.output_path.parent() + && !base_dir.exists() + { + fs::create_dir_all(base_dir).unwrap_or_else(|_| { + panic!("Failed to create output dir(s): {:?}", &opts.output_path) + }); } match convert_config(&opts.input_path, &opts.output_path, opts.output_format) { @@ -207,8 +207,8 @@ fn walk_dir_and_convert( feature = "sinks-console" ))] mod tests { - use crate::config::{format, ConfigBuilder, Format}; - use crate::convert_config::{check_paths, walk_dir_and_convert, Opts}; + use crate::config::{ConfigBuilder, Format, format}; + use crate::convert_config::{Opts, check_paths, walk_dir_and_convert}; use std::path::{Path, PathBuf}; use std::str::FromStr; use std::{env, fs}; @@ -271,7 +271,7 @@ mod tests { let input_path = test_data_dir(); let output_dir = tempdir() .expect("Unable to create tempdir for config") - .into_path(); + .keep(); walk_dir_and_convert(&input_path, &output_dir, Format::Yaml).unwrap(); let mut count: usize = 0; diff --git a/src/dns.rs b/src/dns.rs index 5886b4a6a9..0d6aa3eeae 100644 --- a/src/dns.rs +++ b/src/dns.rs @@ -4,7 +4,7 @@ use std::{ task::{Context, Poll}, }; -use futures::{future::BoxFuture, FutureExt}; +use futures::{FutureExt, future::BoxFuture}; use hyper::client::connect::dns::Name; use snafu::ResultExt; use tokio::task::spawn_blocking; diff --git a/src/docker.rs b/src/docker.rs index 58e9c1819f..3780405056 100644 --- a/src/docker.rs +++ b/src/docker.rs @@ -2,11 +2,14 @@ use std::{collections::HashMap, env, path::PathBuf}; use bollard::{ - container::{Config, CreateContainerOptions}, + API_DEFAULT_VERSION, Docker, errors::Error as DockerError, - image::{CreateImageOptions, ListImagesOptions}, models::HostConfig, - Docker, API_DEFAULT_VERSION, + query_parameters::{ + CreateContainerOptionsBuilder, CreateImageOptionsBuilder, ListImagesOptionsBuilder, + RemoveContainerOptions, StartContainerOptions, StopContainerOptions, + }, + secret::ContainerCreateBody, }; use futures::StreamExt; use http::uri::Uri; @@ -45,7 +48,7 @@ pub fn docker(host: Option, tls: Option) -> crate::Resu let host = host.or_else(|| env::var("DOCKER_HOST").ok()); match host { - None => Docker::connect_with_local_defaults().map_err(Into::into), + None => Docker::connect_with_defaults().map_err(Into::into), Some(host) => { let scheme = host .parse::() @@ -74,19 +77,9 @@ pub fn docker(host: Option, tls: Option) -> crate::Resu .map_err(Into::into) } Some("unix") | Some("npipe") | None => { - // TODO: Use `connect_with_local` on all platforms. - // - // Named pipes are currently disabled in Tokio. Tracking issue: - // https://github.com/fussybeaver/bollard/pull/138 - if cfg!(windows) { - warn!("Named pipes are currently not available on Windows, trying to connecting to Docker with default HTTP settings instead."); - Docker::connect_with_http_defaults().map_err(Into::into) - } else { - Docker::connect_with_local(&host, DEFAULT_TIMEOUT, API_DEFAULT_VERSION) - .map_err(Into::into) - } + Docker::connect_with_defaults().map_err(Into::into) } - Some(scheme) => Err(format!("Unknown scheme: {}", scheme).into()), + Some(scheme) => Err(format!("Unknown scheme: {scheme}").into()), } } } @@ -120,26 +113,24 @@ async fn pull_image(docker: &Docker, image: &str, tag: &str) { vec![format!("{}:{}", image, tag)], ); - let options = Some(ListImagesOptions { - filters, - ..Default::default() - }); + let options = Some(ListImagesOptionsBuilder::new().filters(&filters).build()); let images = docker.list_images(options).await.unwrap(); if images.is_empty() { // If not found, pull it - let options = Some(CreateImageOptions { - from_image: image, - tag, - ..Default::default() - }); + let options = Some( + CreateImageOptionsBuilder::new() + .from_image(image) + .tag(tag) + .build(), + ); docker .create_image(options, None, None) .for_each(|item| async move { let info = item.unwrap(); if let Some(error) = info.error { - panic!("{:?}", error); + panic!("{error:?}"); } }) .await @@ -150,7 +141,7 @@ async fn remove_container(docker: &Docker, id: &str) { trace!("Stopping container."); _ = docker - .stop_container(id, None) + .stop_container(id, None::) .await .map_err(|e| error!(%e)); @@ -158,7 +149,7 @@ async fn remove_container(docker: &Docker, id: &str) { // Don't panic, as this is unrelated to the test _ = docker - .remove_container(id, None) + .remove_container(id, None::) .await .map_err(|e| error!(%e)); } @@ -181,7 +172,7 @@ impl Container { } pub fn bind(mut self, src: impl std::fmt::Display, dst: &str) -> Self { - let bind = format!("{}:{}", src, dst); + let bind = format!("{src}:{dst}"); self.binds.get_or_insert_with(Vec::new).push(bind); self } @@ -196,12 +187,11 @@ impl Container { pull_image(&docker, self.image, self.tag).await; - let options = Some(CreateContainerOptions { - name: format!("vector_test_{}", uuid::Uuid::new_v4()), - platform: None, - }); + let options = CreateContainerOptionsBuilder::new() + .name(&format!("vector_test_{}", uuid::Uuid::new_v4())) + .build(); - let config = Config { + let config = ContainerCreateBody { image: Some(format!("{}:{}", &self.image, &self.tag)), cmd: self.cmd, host_config: Some(HostConfig { @@ -213,10 +203,13 @@ impl Container { ..Default::default() }; - let container = docker.create_container(options, config).await.unwrap(); + let container = docker + .create_container(Some(options), config) + .await + .unwrap(); docker - .start_container::(&container.id, None) + .start_container(&container.id, None::) .await .unwrap(); diff --git a/src/encoding_transcode.rs b/src/encoding_transcode.rs index 5cc1c6882b..986c8657b8 100644 --- a/src/encoding_transcode.rs +++ b/src/encoding_transcode.rs @@ -84,10 +84,7 @@ impl Decoder { // processing, we handle it centrally here. Also, the BOM does not serve // any more use for us, since the source encoding is already pre-identified // as part of decoder initialization. - if output - .get(..BOM_UTF8_LEN) - .map_or(false, |start| start == BOM_UTF8) - { + if output.get(..BOM_UTF8_LEN) == Some(BOM_UTF8) { emit!(DecoderBomRemoval { from_encoding: self.inner.encoding().name() }); @@ -189,9 +186,9 @@ mod tests { use std::char::REPLACEMENT_CHARACTER; use bytes::Bytes; - use encoding_rs::{SHIFT_JIS, UTF_16BE, UTF_16LE, UTF_8}; + use encoding_rs::{SHIFT_JIS, UTF_8, UTF_16BE, UTF_16LE}; - use super::{Decoder, Encoder, BOM_UTF8}; + use super::{BOM_UTF8, Decoder, Encoder}; // BOM unicode character (U+FEFF) expressed in utf-16 // http://unicode.org/faq/utf_bom.html#bom4 @@ -293,10 +290,7 @@ mod tests { assert_eq!( d.decode_to_utf8(Bytes::from(problematic_input)), - Bytes::from(format!( - "{}{}123", - REPLACEMENT_CHARACTER, REPLACEMENT_CHARACTER - )) + Bytes::from(format!("{REPLACEMENT_CHARACTER}{REPLACEMENT_CHARACTER}123")) ); } diff --git a/src/enrichment_tables/file.rs b/src/enrichment_tables/file.rs index deadc6f940..f20331297a 100644 --- a/src/enrichment_tables/file.rs +++ b/src/enrichment_tables/file.rs @@ -5,7 +5,7 @@ use bytes::Bytes; use tracing::trace; use vector_lib::configurable::configurable_component; use vector_lib::enrichment::{Case, Condition, IndexHandle, Table}; -use vector_lib::{conversion::Conversion, TimeZone}; +use vector_lib::{TimeZone, conversion::Conversion}; use vrl::value::{ObjectMap, Value}; use crate::config::EnrichmentTableConfig; @@ -14,6 +14,7 @@ use crate::config::EnrichmentTableConfig; #[configurable_component] #[derive(Clone, Debug, Eq, PartialEq)] #[serde(tag = "type", rename_all = "snake_case")] +#[configurable(metadata(docs::enum_tag_description = "File encoding type."))] pub enum Encoding { /// Decodes the file as a [CSV][csv] (comma-separated values) file. /// @@ -76,7 +77,7 @@ pub struct FileConfig { /// 1. One of the built-in-formats listed in the `Timestamp Formats` table below. /// 2. The [time format specifiers][chrono_fmt] from Rust’s `chrono` library. /// - /// ### Types + /// Types /// /// - **`bool`** /// - **`string`** @@ -85,7 +86,7 @@ pub struct FileConfig { /// - **`date`** /// - **`timestamp`** (see the table below for formats) /// - /// ### Timestamp Formats + /// Timestamp Formats /// /// | Format | Description | Example | /// |----------------------|----------------------------------------------------------------------------------|----------------------------------| @@ -111,6 +112,9 @@ pub struct FileConfig { /// [rfc3339]: https://tools.ietf.org/html/rfc3339 /// [chrono_fmt]: https://docs.rs/chrono/latest/chrono/format/strftime/index.html#specifiers #[serde(default)] + #[configurable(metadata( + docs::additional_props_description = "Represents mapped log field names and types." + ))] pub schema: HashMap, } @@ -139,10 +143,7 @@ impl FileConfig { .from_utc_datetime( &chrono::NaiveDate::parse_from_str(value, "%Y-%m-%d") .map_err(|_| { - format!( - "unable to parse date {} found in row {}", - value, row - ) + format!("unable to parse date {value} found in row {row}") })? .and_hms_opt(0, 0, 0) .expect("invalid timestamp"), @@ -155,10 +156,7 @@ impl FileConfig { .from_utc_datetime( &chrono::NaiveDate::parse_from_str(value, format) .map_err(|_| { - format!( - "unable to parse date {} found in row {}", - value, row - ) + format!("unable to parse date {value} found in row {row}") })? .and_hms_opt(0, 0, 0) .expect("invalid timestamp"), @@ -170,9 +168,7 @@ impl FileConfig { Conversion::parse(format, timezone).map_err(|err| err.to_string())?; conversion .convert(Bytes::copy_from_slice(value.as_bytes())) - .map_err(|_| { - format!("unable to parse {} found in row {}", value, row) - })? + .map_err(|_| format!("unable to parse {value} found in row {row}"))? } } } @@ -192,6 +188,7 @@ impl FileConfig { .delimiter(delimiter as u8) .from_path(&self.file.path)?; + let first_row = reader.records().next(); let headers = if include_headers { reader .headers()? @@ -201,14 +198,15 @@ impl FileConfig { } else { // If there are no headers in the datafile we make headers as the numerical index of // the column. - match reader.records().next() { - Some(Ok(row)) => (0..row.len()).map(|idx| idx.to_string()).collect(), + match first_row { + Some(Ok(ref row)) => (0..row.len()).map(|idx| idx.to_string()).collect(), _ => Vec::new(), } }; - let data = reader - .records() + let data = first_row + .into_iter() + .chain(reader.records()) .map(|row| { Ok(row? .iter() @@ -289,20 +287,60 @@ impl File { } /// Does the given row match all the conditions specified? - fn row_equals(&self, case: Case, condition: &[Condition], row: &[Value]) -> bool { + fn row_equals( + &self, + case: Case, + condition: &[Condition], + row: &[Value], + wildcard: Option<&Value>, + ) -> bool { condition.iter().all(|condition| match condition { Condition::Equals { field, value } => match self.column_index(field) { None => false, - Some(idx) => match (case, &row[idx], value) { - (Case::Insensitive, Value::Bytes(bytes1), Value::Bytes(bytes2)) => { - match (std::str::from_utf8(bytes1), std::str::from_utf8(bytes2)) { - (Ok(s1), Ok(s2)) => s1.to_lowercase() == s2.to_lowercase(), - (Err(_), Err(_)) => bytes1 == bytes2, - _ => false, + Some(idx) => { + let current_row_value = &row[idx]; + + // Helper closure for comparing current_row_value with another value, + // respecting the specified case for Value::Bytes. + let compare_values = |val_to_compare: &Value| -> bool { + match (case, current_row_value, val_to_compare) { + ( + Case::Insensitive, + Value::Bytes(bytes_row), + Value::Bytes(bytes_cmp), + ) => { + // Perform case-insensitive comparison for byte strings. + // If both are valid UTF-8, compare their lowercase versions. + // If both are non-UTF-8 bytes, compare them directly. + // If one is UTF-8 and the other is not, they are considered not equal. + match ( + std::str::from_utf8(bytes_row), + std::str::from_utf8(bytes_cmp), + ) { + (Ok(s_row), Ok(s_cmp)) => { + s_row.to_lowercase() == s_cmp.to_lowercase() + } + (Err(_), Err(_)) => bytes_row == bytes_cmp, + _ => false, + } + } + // For Case::Sensitive, or for Case::Insensitive with non-Bytes types, + // perform a direct equality check. + _ => current_row_value == val_to_compare, } + }; + + // First, check if the row value matches the condition's value. + if compare_values(value) { + true + } else if let Some(wc_val) = wildcard { + // If not, and a wildcard is provided, check if the row value matches the wildcard. + compare_values(wc_val) + } else { + // Otherwise, no match. + false } - (_, value1, value2) => value1 == value2, - }, + } }, Condition::BetweenDates { field, from, to } => match self.column_index(field) { None => false, @@ -311,6 +349,20 @@ impl File { _ => false, }, }, + Condition::FromDate { field, from } => match self.column_index(field) { + None => false, + Some(idx) => match row[idx] { + Value::Timestamp(date) => from <= &date, + _ => false, + }, + }, + Condition::ToDate { field, to } => match self.column_index(field) { + None => false, + Some(idx) => match row[idx] { + Value::Timestamp(date) => &date <= to, + _ => false, + }, + }, }) } @@ -356,7 +408,7 @@ impl File { }) .collect::>() .join(", "); - Err(format!("field(s) '{}' missing from dataset", missing)) + Err(format!("field(s) '{missing}' missing from dataset")) } else { Ok(normalized) } @@ -402,12 +454,13 @@ impl File { case: Case, condition: &'a [Condition<'a>], select: Option<&'a [String]>, + wildcard: Option<&'a Value>, ) -> impl Iterator + 'a where I: Iterator> + 'a, { data.filter_map(move |row| { - if self.row_equals(case, condition, row) { + if self.row_equals(case, condition, row, wildcard) { Some(self.add_columns(select, row)) } else { None @@ -439,6 +492,32 @@ impl File { let IndexHandle(handle) = handle; Ok(self.indexes[handle].2.get(&key)) } + + fn indexed_with_wildcard<'a>( + &'a self, + case: Case, + wildcard: &'a Value, + condition: &'a [Condition<'a>], + handle: IndexHandle, + ) -> Result>, String> { + if let Some(result) = self.indexed(case, condition, handle)? { + return Ok(Some(result)); + } + + // If lookup fails and a wildcard is provided, compute hash for the wildcard + let mut wildcard_hash = seahash::SeaHasher::default(); + for header in self.headers.iter() { + if condition.iter().any( + |condition| matches!(condition, Condition::Equals { field, .. } if field == header), + ) { + hash_value(&mut wildcard_hash, case, wildcard)?; + } + } + + let wildcard_key = wildcard_hash.finish(); + let IndexHandle(handle) = handle; + Ok(self.indexes[handle].2.get(&wildcard_key)) + } } /// Adds the bytes from the given value to the hash. @@ -486,22 +565,26 @@ impl Table for File { case: Case, condition: &'a [Condition<'a>], select: Option<&'a [String]>, + wildcard: Option<&Value>, index: Option, ) -> Result { match index { None => { // No index has been passed so we need to do a Sequential Scan. - single_or_err(self.sequential(self.data.iter(), case, condition, select)) + single_or_err(self.sequential(self.data.iter(), case, condition, select, wildcard)) } Some(handle) => { - let result = self - .indexed(case, condition, handle)? - .ok_or_else(|| "no rows found in index".to_string())? - .iter() - .map(|idx| &self.data[*idx]); + let result = if let Some(wildcard) = wildcard { + self.indexed_with_wildcard(case, wildcard, condition, handle)? + } else { + self.indexed(case, condition, handle)? + } + .ok_or_else(|| "no rows found in index".to_string())? + .iter() + .map(|idx| &self.data[*idx]); // Perform a sequential scan over the indexed result. - single_or_err(self.sequential(result, case, condition, select)) + single_or_err(self.sequential(result, case, condition, select, wildcard)) } } } @@ -511,25 +594,33 @@ impl Table for File { case: Case, condition: &'a [Condition<'a>], select: Option<&'a [String]>, + wildcard: Option<&Value>, index: Option, ) -> Result, String> { match index { None => { // No index has been passed so we need to do a Sequential Scan. Ok(self - .sequential(self.data.iter(), case, condition, select) + .sequential(self.data.iter(), case, condition, select, wildcard) .collect()) } Some(handle) => { // Perform a sequential scan over the indexed result. + let indexed_result = if let Some(wildcard) = wildcard { + self.indexed_with_wildcard(case, wildcard, condition, handle)? + } else { + self.indexed(case, condition, handle)? + }; + Ok(self .sequential( - self.indexed(case, condition, handle)? + indexed_result .iter() .flat_map(|results| results.iter().map(|idx| &self.data[*idx])), case, condition, select, + wildcard, ) .collect()) } @@ -598,6 +689,64 @@ mod tests { use super::*; + #[test] + fn parse_file_with_headers() { + let dir = tempfile::tempdir().expect("Unable to create tempdir for enrichment table"); + let path = dir.path().join("table.csv"); + fs::write(path.clone(), "foo,bar\na,1\nb,2").expect("Failed to write enrichment table"); + + let config = FileConfig { + file: FileSettings { + path, + encoding: Encoding::Csv { + include_headers: true, + delimiter: default_delimiter(), + }, + }, + schema: HashMap::new(), + }; + let data = config + .load_file(Default::default()) + .expect("Failed to parse csv"); + assert_eq!(vec!["foo".to_string(), "bar".to_string()], data.headers); + assert_eq!( + vec![ + vec![Value::from("a"), Value::from("1")], + vec![Value::from("b"), Value::from("2")], + ], + data.data + ); + } + + #[test] + fn parse_file_no_headers() { + let dir = tempfile::tempdir().expect("Unable to create tempdir for enrichment table"); + let path = dir.path().join("table.csv"); + fs::write(path.clone(), "a,1\nb,2").expect("Failed to write enrichment table"); + + let config = FileConfig { + file: FileSettings { + path, + encoding: Encoding::Csv { + include_headers: false, + delimiter: default_delimiter(), + }, + }, + schema: HashMap::new(), + }; + let data = config + .load_file(Default::default()) + .expect("Failed to parse csv"); + assert_eq!(vec!["0".to_string(), "1".to_string()], data.headers); + assert_eq!( + vec![ + vec![Value::from("a"), Value::from("1")], + vec![Value::from("b"), Value::from("2")], + ], + data.data + ); + } + #[test] fn parse_column() { let mut schema = HashMap::new(); @@ -726,7 +875,37 @@ mod tests { ("field1".into(), Value::from("zirp")), ("field2".into(), Value::from("zurp")), ])), - file.find_table_row(Case::Sensitive, &[condition], None, None) + file.find_table_row(Case::Sensitive, &[condition], None, None, None) + ); + } + + #[test] + fn finds_row_with_wildcard() { + let file = File::new( + Default::default(), + FileData { + modified: SystemTime::now(), + data: vec![ + vec!["zip".into(), "zup".into()], + vec!["zirp".into(), "zurp".into()], + ], + headers: vec!["field1".to_string(), "field2".to_string()], + }, + ); + + let wildcard = Value::from("zirp"); + + let condition = Condition::Equals { + field: "field1", + value: Value::from("nonexistent"), + }; + + assert_eq!( + Ok(ObjectMap::from([ + ("field1".into(), Value::from("zirp")), + ("field2".into(), Value::from("zurp")), + ])), + file.find_table_row(Case::Sensitive, &[condition], None, Some(&wildcard), None) ); } @@ -800,7 +979,44 @@ mod tests { ("field1".into(), Value::from("zirp")), ("field2".into(), Value::from("zurp")), ])), - file.find_table_row(Case::Sensitive, &[condition], None, Some(handle)) + file.find_table_row(Case::Sensitive, &[condition], None, None, Some(handle)) + ); + } + + #[test] + fn finds_row_with_index_case_sensitive_and_wildcard() { + let mut file = File::new( + Default::default(), + FileData { + modified: SystemTime::now(), + data: vec![ + vec!["zip".into(), "zup".into()], + vec!["zirp".into(), "zurp".into()], + ], + headers: vec!["field1".to_string(), "field2".to_string()], + }, + ); + + let handle = file.add_index(Case::Sensitive, &["field1"]).unwrap(); + let wildcard = Value::from("zirp"); + + let condition = Condition::Equals { + field: "field1", + value: Value::from("nonexistent"), + }; + + assert_eq!( + Ok(ObjectMap::from([ + ("field1".into(), Value::from("zirp")), + ("field2".into(), Value::from("zurp")), + ])), + file.find_table_row( + Case::Sensitive, + &[condition], + None, + Some(&wildcard), + Some(handle) + ) ); } @@ -839,6 +1055,7 @@ mod tests { value: Value::from("zip"), }], None, + None, Some(handle) ) ); @@ -852,6 +1069,7 @@ mod tests { value: Value::from("ZiP"), }], None, + None, Some(handle) ) ); @@ -898,6 +1116,7 @@ mod tests { Case::Sensitive, &[condition], Some(&["field1".to_string(), "field3".to_string()]), + None, Some(handle) ) ); @@ -938,6 +1157,7 @@ mod tests { value: Value::from("zip"), }], None, + None, Some(handle) ) ); @@ -960,13 +1180,78 @@ mod tests { value: Value::from("ZiP"), }], None, + None, Some(handle) ) ); } #[test] - fn finds_row_with_dates() { + fn finds_rows_with_index_case_insensitive_and_wildcard() { + let mut file = File::new( + Default::default(), + FileData { + modified: SystemTime::now(), + data: vec![ + vec!["zip".into(), "zup".into()], + vec!["zirp".into(), "zurp".into()], + vec!["zip".into(), "zoop".into()], + ], + headers: vec!["field1".to_string(), "field2".to_string()], + }, + ); + + let handle = file.add_index(Case::Insensitive, &["field1"]).unwrap(); + + assert_eq!( + Ok(vec![ + ObjectMap::from([ + ("field1".into(), Value::from("zip")), + ("field2".into(), Value::from("zup")), + ]), + ObjectMap::from([ + ("field1".into(), Value::from("zip")), + ("field2".into(), Value::from("zoop")), + ]), + ]), + file.find_table_rows( + Case::Insensitive, + &[Condition::Equals { + field: "field1", + value: Value::from("nonexistent"), + }], + None, + Some(&Value::from("zip")), + Some(handle) + ) + ); + + assert_eq!( + Ok(vec![ + ObjectMap::from([ + ("field1".into(), Value::from("zip")), + ("field2".into(), Value::from("zup")), + ]), + ObjectMap::from([ + ("field1".into(), Value::from("zip")), + ("field2".into(), Value::from("zoop")), + ]), + ]), + file.find_table_rows( + Case::Insensitive, + &[Condition::Equals { + field: "field1", + value: Value::from("ZiP"), + }], + None, + Some(&Value::from("ZiP")), + Some(handle) + ) + ); + } + + #[test] + fn finds_row_between_dates() { let mut file = File::new( Default::default(), FileData { @@ -1028,7 +1313,133 @@ mod tests { ) ) ])), - file.find_table_row(Case::Sensitive, &conditions, None, Some(handle)) + file.find_table_row(Case::Sensitive, &conditions, None, None, Some(handle)) + ); + } + + #[test] + fn finds_row_from_date() { + let mut file = File::new( + Default::default(), + FileData { + modified: SystemTime::now(), + data: vec![ + vec![ + "zip".into(), + Value::Timestamp( + chrono::Utc + .with_ymd_and_hms(2015, 12, 7, 0, 0, 0) + .single() + .expect("invalid timestamp"), + ), + ], + vec![ + "zip".into(), + Value::Timestamp( + chrono::Utc + .with_ymd_and_hms(2016, 12, 7, 0, 0, 0) + .single() + .expect("invalid timestamp"), + ), + ], + ], + headers: vec!["field1".to_string(), "field2".to_string()], + }, + ); + + let handle = file.add_index(Case::Sensitive, &["field1"]).unwrap(); + + let conditions = [ + Condition::Equals { + field: "field1", + value: "zip".into(), + }, + Condition::FromDate { + field: "field2", + from: chrono::Utc + .with_ymd_and_hms(2016, 1, 1, 0, 0, 0) + .single() + .expect("invalid timestamp"), + }, + ]; + + assert_eq!( + Ok(ObjectMap::from([ + ("field1".into(), Value::from("zip")), + ( + "field2".into(), + Value::Timestamp( + chrono::Utc + .with_ymd_and_hms(2016, 12, 7, 0, 0, 0) + .single() + .expect("invalid timestamp") + ) + ) + ])), + file.find_table_row(Case::Sensitive, &conditions, None, None, Some(handle)) + ); + } + + #[test] + fn finds_row_to_date() { + let mut file = File::new( + Default::default(), + FileData { + modified: SystemTime::now(), + data: vec![ + vec![ + "zip".into(), + Value::Timestamp( + chrono::Utc + .with_ymd_and_hms(2015, 12, 7, 0, 0, 0) + .single() + .expect("invalid timestamp"), + ), + ], + vec![ + "zip".into(), + Value::Timestamp( + chrono::Utc + .with_ymd_and_hms(2016, 12, 7, 0, 0, 0) + .single() + .expect("invalid timestamp"), + ), + ], + ], + headers: vec!["field1".to_string(), "field2".to_string()], + }, + ); + + let handle = file.add_index(Case::Sensitive, &["field1"]).unwrap(); + + let conditions = [ + Condition::Equals { + field: "field1", + value: "zip".into(), + }, + Condition::ToDate { + field: "field2", + to: chrono::Utc + .with_ymd_and_hms(2016, 1, 1, 0, 0, 0) + .single() + .expect("invalid timestamp"), + }, + ]; + + assert_eq!( + Ok(ObjectMap::from([ + ("field1".into(), Value::from("zip")), + ( + "field2".into(), + Value::Timestamp( + chrono::Utc + .with_ymd_and_hms(2015, 12, 7, 0, 0, 0) + .single() + .expect("invalid timestamp") + ) + ) + ])), + file.find_table_row(Case::Sensitive, &conditions, None, None, Some(handle)) ); } @@ -1053,7 +1464,7 @@ mod tests { assert_eq!( Err("no rows found".to_string()), - file.find_table_row(Case::Sensitive, &[condition], None, None) + file.find_table_row(Case::Sensitive, &[condition], None, None, None) ); } @@ -1080,7 +1491,41 @@ mod tests { assert_eq!( Err("no rows found in index".to_string()), - file.find_table_row(Case::Sensitive, &[condition], None, Some(handle)) + file.find_table_row(Case::Sensitive, &[condition], None, None, Some(handle)) + ); + } + + #[test] + fn doesnt_find_row_with_index_and_wildcard() { + let mut file = File::new( + Default::default(), + FileData { + modified: SystemTime::now(), + data: vec![ + vec!["zip".into(), "zup".into()], + vec!["zirp".into(), "zurp".into()], + ], + headers: vec!["field1".to_string(), "field2".to_string()], + }, + ); + + let handle = file.add_index(Case::Sensitive, &["field1"]).unwrap(); + let wildcard = Value::from("nonexistent"); + + let condition = Condition::Equals { + field: "field1", + value: Value::from("zorp"), + }; + + assert_eq!( + Err("no rows found in index".to_string()), + file.find_table_row( + Case::Sensitive, + &[condition], + None, + Some(&wildcard), + Some(handle) + ) ); } } diff --git a/src/enrichment_tables/geoip.rs b/src/enrichment_tables/geoip.rs index 0d6b46601d..3758331b82 100644 --- a/src/enrichment_tables/geoip.rs +++ b/src/enrichment_tables/geoip.rs @@ -4,11 +4,11 @@ //! //! [maxmind]: https://dev.maxmind.com/geoip/geoip2/downloadable //! [geolite]: https://dev.maxmind.com/geoip/geoip2/geolite2/#Download_Access -use std::{collections::BTreeMap, fs, net::IpAddr, sync::Arc, time::SystemTime}; +use std::{collections::BTreeMap, fs, net::IpAddr, path::PathBuf, sync::Arc, time::SystemTime}; use maxminddb::{ + Reader, geoip2::{AnonymousIp, City, ConnectionType, Isp}, - MaxMindDBError, Reader, }; use ordered_float::NotNan; use vector_lib::configurable::configurable_component; @@ -56,7 +56,7 @@ pub struct GeoipConfig { /// /// [geoip2]: https://dev.maxmind.com/geoip/geoip2/downloadable /// [geolite2]: https://dev.maxmind.com/geoip/geoip2/geolite2/#Download_Access - pub path: String, + pub path: PathBuf, /// The locale to use when querying the database. /// @@ -87,7 +87,7 @@ fn default_locale() -> String { impl GenerateConfig for GeoipConfig { fn generate_config() -> toml::Value { toml::Value::try_from(Self { - path: "/path/to/GeoLite2-City.mmdb".to_string(), + path: "/path/to/GeoLite2-City.mmdb".into(), locale: default_locale(), }) .unwrap() @@ -115,7 +115,7 @@ pub struct Geoip { impl Geoip { /// Creates a new GeoIP struct from the provided config. pub fn new(config: GeoipConfig) -> crate::Result { - let dbreader = Arc::new(Reader::open_readfile(config.path.clone())?); + let dbreader = Arc::new(Reader::open_readfile(&config.path)?); let dbkind = DatabaseKind::try_from(dbreader.metadata.database_type.as_str()).map_err(|_| { format!( @@ -134,7 +134,7 @@ impl Geoip { }; match result { - Ok(_) | Err(MaxMindDBError::AddressNotFoundError(_)) => Ok(Geoip { + Ok(_) => Ok(Geoip { last_modified: fs::metadata(&config.path)?.modified()?, dbreader, dbkind, @@ -156,14 +156,14 @@ impl Geoip { }; macro_rules! add_field { - ($k:expr, $v:expr) => { + ($k:expr_2021, $v:expr_2021) => { add_field($k, $v.map(Into::into)) }; } match self.dbkind { DatabaseKind::Asn | DatabaseKind::Isp => { - let data = self.dbreader.lookup::(ip).ok()?; + let data = self.dbreader.lookup::(ip).ok()??; add_field!("autonomous_system_number", data.autonomous_system_number); add_field!( @@ -174,7 +174,7 @@ impl Geoip { add_field!("organization", data.organization); } DatabaseKind::City => { - let data = self.dbreader.lookup::(ip).ok()?; + let data = self.dbreader.lookup::(ip).ok()??; add_field!( "city_name", @@ -224,12 +224,12 @@ impl Geoip { add_field!("postal_code", data.postal.and_then(|p| p.code)); } DatabaseKind::ConnectionType => { - let data = self.dbreader.lookup::(ip).ok()?; + let data = self.dbreader.lookup::(ip).ok()??; add_field!("connection_type", data.connection_type); } DatabaseKind::AnonymousIp => { - let data = self.dbreader.lookup::(ip).ok()?; + let data = self.dbreader.lookup::(ip).ok()??; add_field!("is_anonymous", data.is_anonymous); add_field!("is_anonymous_vpn", data.is_anonymous_vpn); @@ -264,9 +264,10 @@ impl Table for Geoip { case: Case, condition: &'a [Condition<'a>], select: Option<&[String]>, + wildcard: Option<&Value>, index: Option, ) -> Result { - let mut rows = self.find_table_rows(case, condition, select, index)?; + let mut rows = self.find_table_rows(case, condition, select, wildcard, index)?; match rows.pop() { Some(row) if rows.is_empty() => Ok(row), @@ -283,6 +284,7 @@ impl Table for Geoip { _: Case, condition: &'a [Condition<'a>], select: Option<&[String]>, + _wildcard: Option<&Value>, _: Option, ) -> Result, String> { match condition.first() { @@ -333,7 +335,8 @@ impl std::fmt::Debug for Geoip { write!( f, "Geoip {} database {})", - self.config.locale, self.config.path + self.config.locale, + self.config.path.display() ) } } @@ -468,7 +471,7 @@ mod tests { #[test] fn custom_mmdb_type_error() { let result = Geoip::new(GeoipConfig { - path: "tests/data/custom-type.mmdb".to_string(), + path: "tests/data/custom-type.mmdb".into(), locale: default_locale(), }); @@ -502,7 +505,7 @@ mod tests { fn find_select(ip: &str, database: &str, select: Option<&[String]>) -> Option { Geoip::new(GeoipConfig { - path: database.to_string(), + path: database.into(), locale: default_locale(), }) .unwrap() @@ -514,6 +517,7 @@ mod tests { }], select, None, + None, ) .unwrap() .pop() diff --git a/src/enrichment_tables/memory/config.rs b/src/enrichment_tables/memory/config.rs new file mode 100644 index 0000000000..ea3db29159 --- /dev/null +++ b/src/enrichment_tables/memory/config.rs @@ -0,0 +1,203 @@ +use std::num::NonZeroU64; +use std::sync::Arc; + +use crate::sinks::Healthcheck; +use crate::sources::Source; +use crate::{config::SinkContext, enrichment_tables::memory::Memory}; +use async_trait::async_trait; +use futures::{FutureExt, future}; +use tokio::sync::Mutex; +use vector_lib::config::{AcknowledgementsConfig, DataType, Input, LogNamespace}; +use vector_lib::enrichment::Table; +use vector_lib::id::ComponentKey; +use vector_lib::schema::{self}; +use vector_lib::{configurable::configurable_component, sink::VectorSink}; +use vrl::path::OwnedTargetPath; +use vrl::value::Kind; + +use crate::config::{EnrichmentTableConfig, SinkConfig, SourceConfig, SourceContext, SourceOutput}; + +use super::internal_events::InternalMetricsConfig; +use super::source::MemorySourceConfig; + +/// Configuration for the `memory` enrichment table. +#[configurable_component(enrichment_table("memory"))] +#[derive(Clone)] +pub struct MemoryConfig { + /// TTL (time-to-live in seconds) is used to limit the lifetime of data stored in the cache. + /// When TTL expires, data behind a specific key in the cache is removed. + /// TTL is reset when the key is replaced. + #[serde(default = "default_ttl")] + pub ttl: u64, + /// The scan interval used to look for expired records. This is provided + /// as an optimization to ensure that TTL is updated, but without doing + /// too many cache scans. + #[serde(default = "default_scan_interval")] + pub scan_interval: NonZeroU64, + /// The interval used for making writes visible in the table. + /// Longer intervals might get better performance, + /// but there is a longer delay before the data is visible in the table. + /// Since every TTL scan makes its changes visible, only use this value + /// if it is shorter than the `scan_interval`. + /// + /// By default, all writes are made visible immediately. + #[serde(skip_serializing_if = "vector_lib::serde::is_default")] + pub flush_interval: Option, + /// Maximum size of the table in bytes. All insertions that make + /// this table bigger than the maximum size are rejected. + /// + /// By default, there is no size limit. + #[serde(skip_serializing_if = "vector_lib::serde::is_default")] + pub max_byte_size: Option, + /// The namespace to use for logs. This overrides the global setting. + #[configurable(metadata(docs::hidden))] + #[serde(default)] + pub log_namespace: Option, + /// Configuration of internal metrics + #[configurable(derived)] + #[serde(default)] + pub internal_metrics: InternalMetricsConfig, + /// Configuration for source functionality. + #[configurable(derived)] + #[serde(skip_serializing_if = "vector_lib::serde::is_default")] + pub source_config: Option, + + #[serde(skip)] + memory: Arc>>>, +} + +impl PartialEq for MemoryConfig { + fn eq(&self, other: &Self) -> bool { + self.ttl == other.ttl + && self.scan_interval == other.scan_interval + && self.flush_interval == other.flush_interval + } +} +impl Eq for MemoryConfig {} + +impl Default for MemoryConfig { + fn default() -> Self { + Self { + ttl: default_ttl(), + scan_interval: default_scan_interval(), + flush_interval: None, + memory: Arc::new(Mutex::new(None)), + max_byte_size: None, + log_namespace: None, + source_config: None, + internal_metrics: InternalMetricsConfig::default(), + } + } +} + +const fn default_ttl() -> u64 { + 600 +} + +const fn default_scan_interval() -> NonZeroU64 { + unsafe { NonZeroU64::new_unchecked(30) } +} + +impl MemoryConfig { + pub(super) async fn get_or_build_memory(&self) -> Memory { + let mut boxed_memory = self.memory.lock().await; + *boxed_memory + .get_or_insert_with(|| Box::new(Memory::new(self.clone()))) + .clone() + } +} + +impl EnrichmentTableConfig for MemoryConfig { + async fn build( + &self, + _globals: &crate::config::GlobalOptions, + ) -> crate::Result> { + Ok(Box::new(self.get_or_build_memory().await)) + } + + fn sink_config( + &self, + default_key: &ComponentKey, + ) -> Option<(ComponentKey, Box)> { + Some((default_key.clone(), Box::new(self.clone()))) + } + + fn source_config( + &self, + _default_key: &ComponentKey, + ) -> Option<(ComponentKey, Box)> { + let Some(source_config) = &self.source_config else { + return None; + }; + Some(( + source_config.source_key.clone().into(), + Box::new(self.clone()), + )) + } +} + +#[async_trait] +#[typetag::serde(name = "memory_enrichment_table")] +impl SinkConfig for MemoryConfig { + async fn build(&self, _cx: SinkContext) -> crate::Result<(VectorSink, Healthcheck)> { + let sink = VectorSink::from_event_streamsink(self.get_or_build_memory().await); + + Ok((sink, future::ok(()).boxed())) + } + + fn input(&self) -> Input { + Input::log() + } + + fn acknowledgements(&self) -> &AcknowledgementsConfig { + &AcknowledgementsConfig::DEFAULT + } +} + +#[async_trait] +#[typetag::serde(name = "memory_enrichment_table")] +impl SourceConfig for MemoryConfig { + async fn build(&self, cx: SourceContext) -> crate::Result { + let memory = self.get_or_build_memory().await; + + let log_namespace = cx.log_namespace(self.log_namespace); + + Ok(Box::pin( + memory.as_source(cx.shutdown, cx.out, log_namespace).run(), + )) + } + + fn outputs(&self, global_log_namespace: LogNamespace) -> Vec { + let log_namespace = global_log_namespace.merge(self.log_namespace); + let schema_definition = match log_namespace { + LogNamespace::Legacy => schema::Definition::default_legacy_namespace(), + LogNamespace::Vector => { + schema::Definition::new_with_default_metadata(Kind::any_object(), [log_namespace]) + .with_meaning(OwnedTargetPath::event_root(), "message") + } + } + .with_standard_vector_source_metadata(); + + vec![SourceOutput::new_maybe_logs( + DataType::Log, + schema_definition, + )] + } + + fn can_acknowledge(&self) -> bool { + false + } +} + +impl std::fmt::Debug for MemoryConfig { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MemoryConfig") + .field("ttl", &self.ttl) + .field("scan_interval", &self.scan_interval) + .field("flush_interval", &self.flush_interval) + .field("max_byte_size", &self.max_byte_size) + .finish() + } +} + +impl_generate_config_from_default!(MemoryConfig); diff --git a/src/enrichment_tables/memory/internal_events.rs b/src/enrichment_tables/memory/internal_events.rs new file mode 100644 index 0000000000..7a95438938 --- /dev/null +++ b/src/enrichment_tables/memory/internal_events.rs @@ -0,0 +1,154 @@ +use metrics::{counter, gauge}; +use vector_lib::configurable::configurable_component; +use vector_lib::internal_event::InternalEvent; + +/// Configuration of internal metrics for enrichment memory table. +#[configurable_component] +#[derive(Clone, Debug, PartialEq, Eq, Default)] +#[serde(deny_unknown_fields)] +pub struct InternalMetricsConfig { + /// Determines whether to include the key tag on internal metrics. + /// + /// This is useful for distinguishing between different keys while monitoring. However, the tag's + /// cardinality is unbounded. + #[serde(default = "crate::serde::default_false")] + pub include_key_tag: bool, +} + +#[derive(Debug)] +pub(crate) struct MemoryEnrichmentTableRead<'a> { + pub key: &'a str, + pub include_key_metric_tag: bool, +} + +impl InternalEvent for MemoryEnrichmentTableRead<'_> { + fn emit(self) { + if self.include_key_metric_tag { + counter!( + "memory_enrichment_table_reads_total", + "key" => self.key.to_owned() + ) + .increment(1); + } else { + counter!("memory_enrichment_table_reads_total",).increment(1); + } + } + + fn name(&self) -> Option<&'static str> { + Some("MemoryEnrichmentTableRead") + } +} + +#[derive(Debug)] +pub(crate) struct MemoryEnrichmentTableInserted<'a> { + pub key: &'a str, + pub include_key_metric_tag: bool, +} + +impl InternalEvent for MemoryEnrichmentTableInserted<'_> { + fn emit(self) { + if self.include_key_metric_tag { + counter!( + "memory_enrichment_table_insertions_total", + "key" => self.key.to_owned() + ) + .increment(1); + } else { + counter!("memory_enrichment_table_insertions_total",).increment(1); + } + } + + fn name(&self) -> Option<&'static str> { + Some("MemoryEnrichmentTableInserted") + } +} + +#[derive(Debug)] +pub(crate) struct MemoryEnrichmentTableFlushed { + pub new_objects_count: usize, + pub new_byte_size: usize, +} + +impl InternalEvent for MemoryEnrichmentTableFlushed { + fn emit(self) { + counter!("memory_enrichment_table_flushes_total",).increment(1); + gauge!("memory_enrichment_table_objects_count",).set(self.new_objects_count as f64); + gauge!("memory_enrichment_table_byte_size",).set(self.new_byte_size as f64); + } + + fn name(&self) -> Option<&'static str> { + Some("MemoryEnrichmentTableFlushed") + } +} + +#[derive(Debug)] +pub(crate) struct MemoryEnrichmentTableTtlExpired<'a> { + pub key: &'a str, + pub include_key_metric_tag: bool, +} + +impl InternalEvent for MemoryEnrichmentTableTtlExpired<'_> { + fn emit(self) { + if self.include_key_metric_tag { + counter!( + "memory_enrichment_table_ttl_expirations", + "key" => self.key.to_owned() + ) + .increment(1); + } else { + counter!("memory_enrichment_table_ttl_expirations",).increment(1); + } + } + + fn name(&self) -> Option<&'static str> { + Some("MemoryEnrichmentTableTtlExpired") + } +} + +#[derive(Debug)] +pub(crate) struct MemoryEnrichmentTableReadFailed<'a> { + pub key: &'a str, + pub include_key_metric_tag: bool, +} + +impl InternalEvent for MemoryEnrichmentTableReadFailed<'_> { + fn emit(self) { + if self.include_key_metric_tag { + counter!( + "memory_enrichment_table_failed_reads", + "key" => self.key.to_owned() + ) + .increment(1); + } else { + counter!("memory_enrichment_table_failed_reads",).increment(1); + } + } + + fn name(&self) -> Option<&'static str> { + Some("MemoryEnrichmentTableReadFailed") + } +} + +#[derive(Debug)] +pub(crate) struct MemoryEnrichmentTableInsertFailed<'a> { + pub key: &'a str, + pub include_key_metric_tag: bool, +} + +impl InternalEvent for MemoryEnrichmentTableInsertFailed<'_> { + fn emit(self) { + if self.include_key_metric_tag { + counter!( + "memory_enrichment_table_failed_insertions", + "key" => self.key.to_owned() + ) + .increment(1); + } else { + counter!("memory_enrichment_table_failed_insertions",).increment(1); + } + } + + fn name(&self) -> Option<&'static str> { + Some("MemoryEnrichmentTableInsertFailed") + } +} diff --git a/src/enrichment_tables/memory/mod.rs b/src/enrichment_tables/memory/mod.rs new file mode 100644 index 0000000000..72b0986f9b --- /dev/null +++ b/src/enrichment_tables/memory/mod.rs @@ -0,0 +1,9 @@ +//! Handles enrichment tables for `type = memory`. + +mod config; +mod internal_events; +mod source; +mod table; + +pub use config::*; +pub use table::*; diff --git a/src/enrichment_tables/memory/source.rs b/src/enrichment_tables/memory/source.rs new file mode 100644 index 0000000000..2dfceeff55 --- /dev/null +++ b/src/enrichment_tables/memory/source.rs @@ -0,0 +1,138 @@ +use chrono::Utc; +use futures::StreamExt; +use std::{ + num::NonZeroU64, + time::{Duration, Instant}, +}; +use tokio::time::interval; +use tokio_stream::wrappers::IntervalStream; +use vector_lib::{ + ByteSizeOf, EstimatedJsonEncodedSizeOf, + config::LogNamespace, + configurable::configurable_component, + event::{Event, EventMetadata, LogEvent}, + internal_event::{ + ByteSize, BytesReceived, CountByteSize, EventsReceived, InternalEventHandle, Protocol, + }, + shutdown::ShutdownSignal, +}; + +use crate::{SourceSender, internal_events::StreamClosedError}; + +use super::{Memory, MemoryConfig}; + +/// Configuration for memory enrichment table source functionality. +#[configurable_component] +#[derive(Clone, Debug, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct MemorySourceConfig { + /// Interval for exporting all data from the table when used as a source. + pub export_interval: NonZeroU64, + /// Batch size for data exporting. Used to prevent exporting entire table at + /// once and blocking the system. + /// + /// By default, batches are not used and entire table is exported. + #[serde(skip_serializing_if = "vector_lib::serde::is_default")] + pub export_batch_size: Option, + /// If set to true, all data will be removed from cache after exporting. + /// Only valid if used as a source and export_interval > 0 + /// + /// By default, export will not remove data from cache + #[serde(default = "crate::serde::default_false")] + pub remove_after_export: bool, + /// Key to use for this component when used as a source. This must be different from the + /// component key. + pub source_key: String, +} + +/// A struct that represents Memory when used as a source. +pub(crate) struct MemorySource { + pub(super) memory: Memory, + pub(super) shutdown: ShutdownSignal, + pub(super) out: SourceSender, + pub(super) log_namespace: LogNamespace, +} + +impl MemorySource { + pub(crate) async fn run(mut self) -> Result<(), ()> { + let events_received = register!(EventsReceived); + let bytes_received = register!(BytesReceived::from(Protocol::INTERNAL)); + let source_config = self + .memory + .config + .source_config + .as_ref() + .expect("Unexpected missing source config in memory table used as a source."); + let mut interval = IntervalStream::new(interval(Duration::from_secs( + source_config.export_interval.into(), + ))) + .take_until(self.shutdown); + + while interval.next().await.is_some() { + let mut sent = 0_usize; + loop { + let mut events = Vec::new(); + { + let mut writer = self.memory.write_handle.lock().unwrap(); + if let Some(reader) = self.memory.get_read_handle().read() { + let now = Instant::now(); + let utc_now = Utc::now(); + events = reader + .iter() + .skip(if source_config.remove_after_export { + 0 + } else { + sent + }) + .take(if let Some(batch_size) = source_config.export_batch_size { + batch_size as usize + } else { + usize::MAX + }) + .filter_map(|(k, v)| { + if source_config.remove_after_export { + writer.write_handle.empty(k.clone()); + } + v.get_one().map(|v| (k, v)) + }) + .filter_map(|(k, v)| { + let mut event = Event::Log(LogEvent::from_map( + v.as_object_map(now, self.memory.config.ttl, k).ok()?, + EventMetadata::default(), + )); + let log = event.as_mut_log(); + self.log_namespace.insert_standard_vector_source_metadata( + log, + MemoryConfig::NAME, + utc_now, + ); + + Some(event) + }) + .collect::>(); + if source_config.remove_after_export { + writer.write_handle.refresh(); + } + } + } + let count = events.len(); + let byte_size = events.size_of(); + let json_size = events.estimated_json_encoded_size_of(); + bytes_received.emit(ByteSize(byte_size)); + events_received.emit(CountByteSize(count, json_size)); + if self.out.send_batch(events).await.is_err() { + emit!(StreamClosedError { count }); + } + + sent += count; + match source_config.export_batch_size { + None => break, + Some(export_batch_size) if count < export_batch_size as usize => break, + _ => {} + } + } + } + + Ok(()) + } +} diff --git a/src/enrichment_tables/memory/table.rs b/src/enrichment_tables/memory/table.rs new file mode 100644 index 0000000000..0d97c7ebbc --- /dev/null +++ b/src/enrichment_tables/memory/table.rs @@ -0,0 +1,901 @@ +#![allow(unsafe_op_in_unsafe_fn)] // TODO review ShallowCopy usage code and fix properly. + +use crate::SourceSender; +use crate::enrichment_tables::memory::MemoryConfig; +use crate::enrichment_tables::memory::internal_events::{ + MemoryEnrichmentTableFlushed, MemoryEnrichmentTableInsertFailed, MemoryEnrichmentTableInserted, + MemoryEnrichmentTableRead, MemoryEnrichmentTableReadFailed, MemoryEnrichmentTableTtlExpired, +}; +use std::sync::{Arc, Mutex, MutexGuard}; +use std::time::{Duration, Instant}; + +use evmap::shallow_copy::CopyValue; +use evmap::{self}; +use evmap_derive::ShallowCopy; +use futures::StreamExt; +use thread_local::ThreadLocal; +use tokio::time::interval; +use tokio_stream::wrappers::IntervalStream; +use vector_lib::config::LogNamespace; +use vector_lib::shutdown::ShutdownSignal; +use vector_lib::{ByteSizeOf, EstimatedJsonEncodedSizeOf}; + +use async_trait::async_trait; +use bytes::Bytes; +use futures::stream::BoxStream; +use vector_lib::enrichment::{Case, Condition, IndexHandle, Table}; +use vector_lib::event::{Event, EventStatus, Finalizable}; +use vector_lib::internal_event::{ + ByteSize, BytesSent, CountByteSize, EventsSent, InternalEventHandle, Output, Protocol, +}; +use vector_lib::sink::StreamSink; +use vrl::value::{KeyString, ObjectMap, Value}; + +use super::source::MemorySource; + +/// Single memory entry containing the value and TTL +#[derive(Clone, Eq, PartialEq, Hash, ShallowCopy)] +pub struct MemoryEntry { + value: String, + update_time: CopyValue, +} + +impl ByteSizeOf for MemoryEntry { + fn allocated_bytes(&self) -> usize { + self.value.size_of() + } +} + +impl MemoryEntry { + pub(super) fn as_object_map( + &self, + now: Instant, + total_ttl: u64, + key: &str, + ) -> Result { + let ttl = total_ttl.saturating_sub(now.duration_since(*self.update_time).as_secs()); + Ok(ObjectMap::from([ + ( + KeyString::from("key"), + Value::Bytes(Bytes::copy_from_slice(key.as_bytes())), + ), + ( + KeyString::from("value"), + serde_json::from_str::(&self.value) + .map_err(|_| "Failed to read value from memory!")?, + ), + ( + KeyString::from("ttl"), + Value::Integer(ttl.try_into().unwrap_or(i64::MAX)), + ), + ])) + } + + fn expired(&self, now: Instant, ttl: u64) -> bool { + now.duration_since(*self.update_time).as_secs() > ttl + } +} + +#[derive(Default)] +struct MemoryMetadata { + byte_size: u64, +} + +// Used to ensure that these 2 are locked together +pub(super) struct MemoryWriter { + pub(super) write_handle: evmap::WriteHandle, + metadata: MemoryMetadata, +} + +/// A struct that implements [vector_lib::enrichment::Table] to handle loading enrichment data from a memory structure. +pub struct Memory { + pub(super) read_handle_factory: evmap::ReadHandleFactory, + pub(super) read_handle: ThreadLocal>, + pub(super) write_handle: Arc>, + pub(super) config: MemoryConfig, +} + +impl Memory { + /// Creates a new [Memory] based on the provided config. + pub fn new(config: MemoryConfig) -> Self { + let (read_handle, write_handle) = evmap::new(); + Self { + config, + read_handle_factory: read_handle.factory(), + read_handle: ThreadLocal::new(), + write_handle: Arc::new(Mutex::new(MemoryWriter { + write_handle, + metadata: MemoryMetadata::default(), + })), + } + } + + pub(super) fn get_read_handle(&self) -> &evmap::ReadHandle { + self.read_handle + .get_or(|| self.read_handle_factory.handle()) + } + + fn handle_value(&self, value: ObjectMap) { + let mut writer = self.write_handle.lock().expect("mutex poisoned"); + let now = Instant::now(); + + for (k, v) in value.into_iter() { + let new_entry_key = String::from(k); + let Ok(v) = serde_json::to_string(&v) else { + emit!(MemoryEnrichmentTableInsertFailed { + key: &new_entry_key, + include_key_metric_tag: self.config.internal_metrics.include_key_tag + }); + continue; + }; + let new_entry = MemoryEntry { + value: v, + update_time: now.into(), + }; + let new_entry_size = new_entry_key.size_of() + new_entry.size_of(); + if let Some(max_byte_size) = self.config.max_byte_size + && writer + .metadata + .byte_size + .saturating_add(new_entry_size as u64) + > max_byte_size + { + // Reject new entries + emit!(MemoryEnrichmentTableInsertFailed { + key: &new_entry_key, + include_key_metric_tag: self.config.internal_metrics.include_key_tag + }); + continue; + } + writer.metadata.byte_size = writer + .metadata + .byte_size + .saturating_add(new_entry_size as u64); + emit!(MemoryEnrichmentTableInserted { + key: &new_entry_key, + include_key_metric_tag: self.config.internal_metrics.include_key_tag + }); + writer.write_handle.update(new_entry_key, new_entry); + } + + if self.config.flush_interval.is_none() { + self.flush(writer); + } + } + + fn scan_and_mark_for_deletion(&self, writer: &mut MutexGuard<'_, MemoryWriter>) -> bool { + let now = Instant::now(); + + let mut needs_flush = false; + // Since evmap holds 2 separate maps for the data, we are free to directly remove + // elements via the writer, while we are iterating the reader + // Refresh will happen only after we manually invoke it after iteration + if let Some(reader) = self.get_read_handle().read() { + for (k, v) in reader.iter() { + if let Some(entry) = v.get_one() + && entry.expired(now, self.config.ttl) + { + // Byte size is not reduced at this point, because the actual deletion + // will only happen at refresh time + writer.write_handle.empty(k.clone()); + emit!(MemoryEnrichmentTableTtlExpired { + key: k, + include_key_metric_tag: self.config.internal_metrics.include_key_tag + }); + needs_flush = true; + } + } + }; + + needs_flush + } + + fn scan(&self, mut writer: MutexGuard<'_, MemoryWriter>) { + let needs_flush = self.scan_and_mark_for_deletion(&mut writer); + if needs_flush { + self.flush(writer); + } + } + + fn flush(&self, mut writer: MutexGuard<'_, MemoryWriter>) { + writer.write_handle.refresh(); + if let Some(reader) = self.get_read_handle().read() { + let mut byte_size = 0; + for (k, v) in reader.iter() { + byte_size += k.size_of() + v.get_one().size_of(); + } + writer.metadata.byte_size = byte_size as u64; + emit!(MemoryEnrichmentTableFlushed { + new_objects_count: reader.len(), + new_byte_size: byte_size + }); + } + } + + pub(crate) fn as_source( + &self, + shutdown: ShutdownSignal, + out: SourceSender, + log_namespace: LogNamespace, + ) -> MemorySource { + MemorySource { + memory: self.clone(), + shutdown, + out, + log_namespace, + } + } +} + +impl Clone for Memory { + fn clone(&self) -> Self { + Self { + read_handle_factory: self.read_handle_factory.clone(), + read_handle: ThreadLocal::new(), + write_handle: Arc::clone(&self.write_handle), + config: self.config.clone(), + } + } +} + +impl Table for Memory { + fn find_table_row<'a>( + &self, + case: Case, + condition: &'a [Condition<'a>], + select: Option<&'a [String]>, + wildcard: Option<&Value>, + index: Option, + ) -> Result { + let mut rows = self.find_table_rows(case, condition, select, wildcard, index)?; + + match rows.pop() { + Some(row) if rows.is_empty() => Ok(row), + Some(_) => Err("More than 1 row found".to_string()), + None => Err("Key not found".to_string()), + } + } + + fn find_table_rows<'a>( + &self, + _case: Case, + condition: &'a [Condition<'a>], + _select: Option<&'a [String]>, + _wildcard: Option<&Value>, + _index: Option, + ) -> Result, String> { + match condition.first() { + Some(_) if condition.len() > 1 => Err("Only one condition is allowed".to_string()), + Some(Condition::Equals { value, .. }) => { + let key = value.to_string_lossy(); + match self.get_read_handle().get_one(key.as_ref()) { + Some(row) => { + emit!(MemoryEnrichmentTableRead { + key: &key, + include_key_metric_tag: self.config.internal_metrics.include_key_tag + }); + row.as_object_map(Instant::now(), self.config.ttl, &key) + .map(|r| vec![r]) + } + None => { + emit!(MemoryEnrichmentTableReadFailed { + key: &key, + include_key_metric_tag: self.config.internal_metrics.include_key_tag + }); + Ok(Default::default()) + } + } + } + Some(_) => Err("Only equality condition is allowed".to_string()), + None => Err("Key condition must be specified".to_string()), + } + } + + fn add_index(&mut self, _case: Case, fields: &[&str]) -> Result { + match fields.len() { + 0 => Err("Key field is required".to_string()), + 1 => Ok(IndexHandle(0)), + _ => Err("Only one field is allowed".to_string()), + } + } + + /// Returns a list of the field names that are in each index + fn index_fields(&self) -> Vec<(Case, Vec)> { + Vec::new() + } + + /// Doesn't need reload, data is written directly + fn needs_reload(&self) -> bool { + false + } +} + +impl std::fmt::Debug for Memory { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Memory {} row(s)", self.get_read_handle().len()) + } +} + +#[async_trait] +impl StreamSink for Memory { + async fn run(mut self: Box, mut input: BoxStream<'_, Event>) -> Result<(), ()> { + let events_sent = register!(EventsSent::from(Output(None))); + let bytes_sent = register!(BytesSent::from(Protocol("memory_enrichment_table".into(),))); + let mut flush_interval = IntervalStream::new(interval( + self.config + .flush_interval + .map(Duration::from_secs) + .unwrap_or(Duration::MAX), + )); + let mut scan_interval = IntervalStream::new(interval(Duration::from_secs( + self.config.scan_interval.into(), + ))); + + loop { + tokio::select! { + event = input.next() => { + let mut event = if let Some(event) = event { + event + } else { + break; + }; + let event_byte_size = event.estimated_json_encoded_size_of(); + + let finalizers = event.take_finalizers(); + + // Panic: This sink only accepts Logs, so this should never panic + let log = event.into_log(); + + if let (Value::Object(map), _) = log.into_parts() { + self.handle_value(map) + }; + + finalizers.update_status(EventStatus::Delivered); + events_sent.emit(CountByteSize(1, event_byte_size)); + bytes_sent.emit(ByteSize(event_byte_size.get())); + } + + Some(_) = flush_interval.next() => { + let writer = self.write_handle.lock().expect("mutex poisoned"); + self.flush(writer); + } + + Some(_) = scan_interval.next() => { + let writer = self.write_handle.lock().expect("mutex poisoned"); + self.scan(writer); + } + } + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use futures::{StreamExt, future::ready}; + use futures_util::stream; + use std::slice::from_ref; + use std::{num::NonZeroU64, time::Duration}; + use tokio::time; + + use vector_lib::{ + event::{EventContainer, MetricValue}, + metrics::Controller, + sink::VectorSink, + }; + + use super::*; + use crate::{ + enrichment_tables::memory::{ + internal_events::InternalMetricsConfig, source::MemorySourceConfig, + }, + event::{Event, LogEvent}, + test_util::components::{ + SINK_TAGS, SOURCE_TAGS, run_and_assert_sink_compliance, + run_and_assert_source_compliance, + }, + }; + + fn build_memory_config(modfn: impl Fn(&mut MemoryConfig)) -> MemoryConfig { + let mut config = MemoryConfig::default(); + modfn(&mut config); + config + } + + #[test] + fn finds_row() { + let memory = Memory::new(Default::default()); + memory.handle_value(ObjectMap::from([("test_key".into(), Value::from(5))])); + + let condition = Condition::Equals { + field: "key", + value: Value::from("test_key"), + }; + + assert_eq!( + Ok(ObjectMap::from([ + ("key".into(), Value::from("test_key")), + ("ttl".into(), Value::from(memory.config.ttl)), + ("value".into(), Value::from(5)), + ])), + memory.find_table_row(Case::Sensitive, &[condition], None, None, None) + ); + } + + #[test] + fn calculates_ttl() { + let ttl = 100; + let secs_to_subtract = 10; + let memory = Memory::new(build_memory_config(|c| c.ttl = ttl)); + { + let mut handle = memory.write_handle.lock().unwrap(); + handle.write_handle.update( + "test_key".to_string(), + MemoryEntry { + value: "5".to_string(), + update_time: (Instant::now() - Duration::from_secs(secs_to_subtract)).into(), + }, + ); + handle.write_handle.refresh(); + } + + let condition = Condition::Equals { + field: "key", + value: Value::from("test_key"), + }; + + assert_eq!( + Ok(ObjectMap::from([ + ("key".into(), Value::from("test_key")), + ("ttl".into(), Value::from(ttl - secs_to_subtract)), + ("value".into(), Value::from(5)), + ])), + memory.find_table_row(Case::Sensitive, &[condition], None, None, None) + ); + } + + #[test] + fn removes_expired_records_on_scan_interval() { + let ttl = 100; + let memory = Memory::new(build_memory_config(|c| { + c.ttl = ttl; + })); + { + let mut handle = memory.write_handle.lock().unwrap(); + handle.write_handle.update( + "test_key".to_string(), + MemoryEntry { + value: "5".to_string(), + update_time: (Instant::now() - Duration::from_secs(ttl + 10)).into(), + }, + ); + handle.write_handle.refresh(); + } + + // Finds the value before scan + let condition = Condition::Equals { + field: "key", + value: Value::from("test_key"), + }; + assert_eq!( + Ok(ObjectMap::from([ + ("key".into(), Value::from("test_key")), + ("ttl".into(), Value::from(0)), + ("value".into(), Value::from(5)), + ])), + memory.find_table_row(Case::Sensitive, from_ref(&condition), None, None, None) + ); + + // Force scan + let writer = memory.write_handle.lock().unwrap(); + memory.scan(writer); + + // The value is not present anymore + assert!( + memory + .find_table_rows(Case::Sensitive, &[condition], None, None, None) + .unwrap() + .pop() + .is_none() + ); + } + + #[test] + fn does_not_show_values_before_flush_interval() { + let ttl = 100; + let memory = Memory::new(build_memory_config(|c| { + c.ttl = ttl; + c.flush_interval = Some(10); + })); + memory.handle_value(ObjectMap::from([("test_key".into(), Value::from(5))])); + + let condition = Condition::Equals { + field: "key", + value: Value::from("test_key"), + }; + + assert!( + memory + .find_table_rows(Case::Sensitive, &[condition], None, None, None) + .unwrap() + .pop() + .is_none() + ); + } + + #[test] + fn updates_ttl_on_value_replacement() { + let ttl = 100; + let memory = Memory::new(build_memory_config(|c| c.ttl = ttl)); + { + let mut handle = memory.write_handle.lock().unwrap(); + handle.write_handle.update( + "test_key".to_string(), + MemoryEntry { + value: "5".to_string(), + update_time: (Instant::now() - Duration::from_secs(ttl / 2)).into(), + }, + ); + handle.write_handle.refresh(); + } + let condition = Condition::Equals { + field: "key", + value: Value::from("test_key"), + }; + + assert_eq!( + Ok(ObjectMap::from([ + ("key".into(), Value::from("test_key")), + ("ttl".into(), Value::from(ttl / 2)), + ("value".into(), Value::from(5)), + ])), + memory.find_table_row(Case::Sensitive, from_ref(&condition), None, None, None) + ); + + memory.handle_value(ObjectMap::from([("test_key".into(), Value::from(5))])); + + assert_eq!( + Ok(ObjectMap::from([ + ("key".into(), Value::from("test_key")), + ("ttl".into(), Value::from(ttl)), + ("value".into(), Value::from(5)), + ])), + memory.find_table_row(Case::Sensitive, &[condition], None, None, None) + ); + } + + #[test] + fn ignores_all_values_over_byte_size_limit() { + let memory = Memory::new(build_memory_config(|c| { + c.max_byte_size = Some(1); + })); + memory.handle_value(ObjectMap::from([("test_key".into(), Value::from(5))])); + + let condition = Condition::Equals { + field: "key", + value: Value::from("test_key"), + }; + + assert!( + memory + .find_table_rows(Case::Sensitive, &[condition], None, None, None) + .unwrap() + .pop() + .is_none() + ); + } + + #[test] + fn ignores_values_when_byte_size_limit_is_reached() { + let ttl = 100; + let memory = Memory::new(build_memory_config(|c| { + c.ttl = ttl; + c.max_byte_size = Some(150); + })); + memory.handle_value(ObjectMap::from([("test_key".into(), Value::from(5))])); + memory.handle_value(ObjectMap::from([("rejected_key".into(), Value::from(5))])); + + assert_eq!( + Ok(ObjectMap::from([ + ("key".into(), Value::from("test_key")), + ("ttl".into(), Value::from(ttl)), + ("value".into(), Value::from(5)), + ])), + memory.find_table_row( + Case::Sensitive, + &[Condition::Equals { + field: "key", + value: Value::from("test_key") + }], + None, + None, + None + ) + ); + + assert!( + memory + .find_table_rows( + Case::Sensitive, + &[Condition::Equals { + field: "key", + value: Value::from("rejected_key") + }], + None, + None, + None + ) + .unwrap() + .pop() + .is_none() + ); + } + + #[test] + fn missing_key() { + let memory = Memory::new(Default::default()); + + let condition = Condition::Equals { + field: "key", + value: Value::from("test_key"), + }; + + assert!( + memory + .find_table_rows(Case::Sensitive, &[condition], None, None, None) + .unwrap() + .pop() + .is_none() + ); + } + + #[tokio::test] + async fn sink_spec_compliance() { + let event = Event::Log(LogEvent::from(ObjectMap::from([( + "test_key".into(), + Value::from(5), + )]))); + + let memory = Memory::new(Default::default()); + + run_and_assert_sink_compliance( + VectorSink::from_event_streamsink(memory), + stream::once(ready(event)), + &SINK_TAGS, + ) + .await; + } + + #[tokio::test] + async fn flush_metrics_without_interval() { + let event = Event::Log(LogEvent::from(ObjectMap::from([( + "test_key".into(), + Value::from(5), + )]))); + + let memory = Memory::new(Default::default()); + + run_and_assert_sink_compliance( + VectorSink::from_event_streamsink(memory), + stream::once(ready(event)), + &SINK_TAGS, + ) + .await; + + let metrics = Controller::get().unwrap().capture_metrics(); + let insertions_counter = metrics + .iter() + .find(|m| { + matches!(m.value(), MetricValue::Counter { .. }) + && m.name() == "memory_enrichment_table_insertions_total" + }) + .expect("Insertions metric is missing!"); + let MetricValue::Counter { + value: insertions_count, + } = insertions_counter.value() + else { + unreachable!(); + }; + let flushes_counter = metrics + .iter() + .find(|m| { + matches!(m.value(), MetricValue::Counter { .. }) + && m.name() == "memory_enrichment_table_flushes_total" + }) + .expect("Flushes metric is missing!"); + let MetricValue::Counter { + value: flushes_count, + } = flushes_counter.value() + else { + unreachable!(); + }; + let object_count_gauge = metrics + .iter() + .find(|m| { + matches!(m.value(), MetricValue::Gauge { .. }) + && m.name() == "memory_enrichment_table_objects_count" + }) + .expect("Object count metric is missing!"); + let MetricValue::Gauge { + value: object_count, + } = object_count_gauge.value() + else { + unreachable!(); + }; + let byte_size_gauge = metrics + .iter() + .find(|m| { + matches!(m.value(), MetricValue::Gauge { .. }) + && m.name() == "memory_enrichment_table_byte_size" + }) + .expect("Byte size metric is missing!"); + assert_eq!(*insertions_count, 1.0); + assert_eq!(*flushes_count, 1.0); + assert_eq!(*object_count, 1.0); + assert!(!byte_size_gauge.is_empty()); + } + + #[tokio::test] + async fn flush_metrics_with_interval() { + let event = Event::Log(LogEvent::from(ObjectMap::from([( + "test_key".into(), + Value::from(5), + )]))); + + let memory = Memory::new(build_memory_config(|c| { + c.flush_interval = Some(1); + })); + + run_and_assert_sink_compliance( + VectorSink::from_event_streamsink(memory), + stream::iter(vec![event.clone(), event]).flat_map(|e| { + stream::once(async move { + tokio::time::sleep(Duration::from_millis(600)).await; + e + }) + }), + &SINK_TAGS, + ) + .await; + + let metrics = Controller::get().unwrap().capture_metrics(); + let insertions_counter = metrics + .iter() + .find(|m| { + matches!(m.value(), MetricValue::Counter { .. }) + && m.name() == "memory_enrichment_table_insertions_total" + }) + .expect("Insertions metric is missing!"); + let MetricValue::Counter { + value: insertions_count, + } = insertions_counter.value() + else { + unreachable!(); + }; + let flushes_counter = metrics + .iter() + .find(|m| { + matches!(m.value(), MetricValue::Counter { .. }) + && m.name() == "memory_enrichment_table_flushes_total" + }) + .expect("Flushes metric is missing!"); + let MetricValue::Counter { + value: flushes_count, + } = flushes_counter.value() + else { + unreachable!(); + }; + let object_count_gauge = metrics + .iter() + .find(|m| { + matches!(m.value(), MetricValue::Gauge { .. }) + && m.name() == "memory_enrichment_table_objects_count" + }) + .expect("Object count metric is missing!"); + let MetricValue::Gauge { + value: object_count, + } = object_count_gauge.value() + else { + unreachable!(); + }; + let byte_size_gauge = metrics + .iter() + .find(|m| { + matches!(m.value(), MetricValue::Gauge { .. }) + && m.name() == "memory_enrichment_table_byte_size" + }) + .expect("Byte size metric is missing!"); + + assert_eq!(*insertions_count, 2.0); + // One is done right away and the next one after the interval + assert_eq!(*flushes_count, 2.0); + assert_eq!(*object_count, 1.0); + assert!(!byte_size_gauge.is_empty()); + } + + #[tokio::test] + async fn flush_metrics_with_key() { + let event = Event::Log(LogEvent::from(ObjectMap::from([( + "test_key".into(), + Value::from(5), + )]))); + + let memory = Memory::new(build_memory_config(|c| { + c.internal_metrics = InternalMetricsConfig { + include_key_tag: true, + }; + })); + + run_and_assert_sink_compliance( + VectorSink::from_event_streamsink(memory), + stream::once(ready(event)), + &SINK_TAGS, + ) + .await; + + let metrics = Controller::get().unwrap().capture_metrics(); + let insertions_counter = metrics + .iter() + .find(|m| { + matches!(m.value(), MetricValue::Counter { .. }) + && m.name() == "memory_enrichment_table_insertions_total" + }) + .expect("Insertions metric is missing!"); + + assert!(insertions_counter.tag_matches("key", "test_key")); + } + + #[tokio::test] + async fn flush_metrics_without_key() { + let event = Event::Log(LogEvent::from(ObjectMap::from([( + "test_key".into(), + Value::from(5), + )]))); + + let memory = Memory::new(Default::default()); + + run_and_assert_sink_compliance( + VectorSink::from_event_streamsink(memory), + stream::once(ready(event)), + &SINK_TAGS, + ) + .await; + + let metrics = Controller::get().unwrap().capture_metrics(); + let insertions_counter = metrics + .iter() + .find(|m| { + matches!(m.value(), MetricValue::Counter { .. }) + && m.name() == "memory_enrichment_table_insertions_total" + }) + .expect("Insertions metric is missing!"); + + assert!(insertions_counter.tag_value("key").is_none()); + } + + #[tokio::test] + async fn source_spec_compliance() { + let mut memory_config = MemoryConfig::default(); + memory_config.source_config = Some(MemorySourceConfig { + export_interval: NonZeroU64::try_from(1).unwrap(), + export_batch_size: None, + remove_after_export: false, + source_key: "test".to_string(), + }); + let memory = memory_config.get_or_build_memory().await; + memory.handle_value(ObjectMap::from([("test_key".into(), Value::from(5))])); + + let mut events: Vec = run_and_assert_source_compliance( + memory_config, + time::Duration::from_secs(5), + &SOURCE_TAGS, + ) + .await; + + assert!(!events.is_empty()); + let event = events.remove(0); + let log = event.as_log(); + + assert!(!log.value().is_empty()); + } +} diff --git a/src/enrichment_tables/mmdb.rs b/src/enrichment_tables/mmdb.rs index d4a40197d4..91b8d22a99 100644 --- a/src/enrichment_tables/mmdb.rs +++ b/src/enrichment_tables/mmdb.rs @@ -2,9 +2,10 @@ //! Enrichment data is loaded from any database in [MaxMind][maxmind] format. //! //! [maxmind]: https://maxmind.com +use std::path::PathBuf; use std::{fs, net::IpAddr, sync::Arc, time::SystemTime}; -use maxminddb::{MaxMindDBError, Reader}; +use maxminddb::Reader; use vector_lib::configurable::configurable_component; use vector_lib::enrichment::{Case, Condition, IndexHandle, Table}; use vrl::value::{ObjectMap, Value}; @@ -18,13 +19,13 @@ pub struct MmdbConfig { /// Path to the [MaxMind][maxmind] database /// /// [maxmind]: https://maxmind.com - pub path: String, + pub path: PathBuf, } impl GenerateConfig for MmdbConfig { fn generate_config() -> toml::Value { toml::Value::try_from(Self { - path: "/path/to/GeoLite2-City.mmdb".to_string(), + path: "/path/to/GeoLite2-City.mmdb".into(), }) .unwrap() } @@ -50,14 +51,14 @@ pub struct Mmdb { impl Mmdb { /// Creates a new Mmdb struct from the provided config. pub fn new(config: MmdbConfig) -> crate::Result { - let dbreader = Arc::new(Reader::open_readfile(config.path.clone())?); + let dbreader = Arc::new(Reader::open_readfile(&config.path)?); // Check if we can read database with dummy Ip. let ip = IpAddr::V4(std::net::Ipv4Addr::UNSPECIFIED); let result = dbreader.lookup::(ip).map(|_| ()); match result { - Ok(_) | Err(MaxMindDBError::AddressNotFoundError(_)) => Ok(Mmdb { + Ok(_) => Ok(Mmdb { last_modified: fs::metadata(&config.path)?.modified()?, dbreader, config, @@ -67,7 +68,7 @@ impl Mmdb { } fn lookup(&self, ip: IpAddr, select: Option<&[String]>) -> Option { - let data = self.dbreader.lookup::(ip).ok()?; + let data = self.dbreader.lookup::(ip).ok()??; if let Some(fields) = select { let mut filtered = Value::from(ObjectMap::new()); @@ -98,9 +99,10 @@ impl Table for Mmdb { case: Case, condition: &'a [Condition<'a>], select: Option<&[String]>, + wildcard: Option<&Value>, index: Option, ) -> Result { - let mut rows = self.find_table_rows(case, condition, select, index)?; + let mut rows = self.find_table_rows(case, condition, select, wildcard, index)?; match rows.pop() { Some(row) if rows.is_empty() => Ok(row), @@ -117,6 +119,7 @@ impl Table for Mmdb { _: Case, condition: &'a [Condition<'a>], select: Option<&[String]>, + _wildcard: Option<&Value>, _: Option, ) -> Result, String> { match condition.first() { @@ -164,7 +167,7 @@ impl Table for Mmdb { impl std::fmt::Debug for Mmdb { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "Maxmind database {})", self.config.path) + write!(f, "Maxmind database {})", self.config.path.display()) } } @@ -259,7 +262,7 @@ mod tests { fn find_select(ip: &str, database: &str, select: Option<&[String]>) -> Option { Mmdb::new(MmdbConfig { - path: database.to_string(), + path: database.into(), }) .unwrap() .find_table_rows( @@ -270,6 +273,7 @@ mod tests { }], select, None, + None, ) .unwrap() .pop() diff --git a/src/enrichment_tables/mod.rs b/src/enrichment_tables/mod.rs index 97a93b0059..f814e175f5 100644 --- a/src/enrichment_tables/mod.rs +++ b/src/enrichment_tables/mod.rs @@ -1,27 +1,58 @@ //! Functionality to handle enrichment tables. +use std::path::PathBuf; + use enum_dispatch::enum_dispatch; -use vector_lib::configurable::{configurable_component, NamedComponent}; +use vector_lib::configurable::configurable_component; pub use vector_lib::enrichment::{Condition, IndexHandle, Table}; -use crate::config::{EnrichmentTableConfig, GlobalOptions}; +use crate::config::{ + ComponentKey, EnrichmentTableConfig, GenerateConfig, GlobalOptions, SinkConfig, SourceConfig, +}; pub mod file; +#[cfg(feature = "enrichment-tables-memory")] +pub mod memory; + #[cfg(feature = "enrichment-tables-geoip")] pub mod geoip; #[cfg(feature = "enrichment-tables-mmdb")] pub mod mmdb; -/// Configurable enrichment tables. -#[configurable_component] +/// Configuration options for an [enrichment table](https://vector.dev/docs/reference/glossary/#enrichment-tables) to be used in a +/// [`remap`](https://vector.dev/docs/reference/configuration/transforms/remap/) transform. Currently supported are: +/// +/// * [CSV](https://en.wikipedia.org/wiki/Comma-separated_values) files +/// * [MaxMind](https://www.maxmind.com/en/home) databases +/// * In-memory storage +/// +/// For the lookup in the enrichment tables to be as performant as possible, the data is indexed according +/// to the fields that are used in the search. Note that indices can only be created for fields for which an +/// exact match is used in the condition. For range searches, an index isn't used and the enrichment table +/// drops back to a sequential scan of the data. A sequential scan shouldn't impact performance +/// significantly provided that there are only a few possible rows returned by the exact matches in the +/// condition. We don't recommend using a condition that uses only date range searches. +/// +/// +#[configurable_component(global_option("enrichment_tables"))] #[derive(Clone, Debug)] #[serde(tag = "type", rename_all = "snake_case")] #[enum_dispatch(EnrichmentTableConfig)] +#[configurable(metadata( + docs::enum_tag_description = "enrichment table type", + docs::common = false, + docs::required = false, +))] pub enum EnrichmentTables { /// Exposes data from a static file as an enrichment table. File(file::FileConfig), + /// Exposes data from a memory cache as an enrichment table. The cache can be written to using + /// a sink. + #[cfg(feature = "enrichment-tables-memory")] + Memory(memory::MemoryConfig), + /// Exposes data from a [MaxMind][maxmind] [GeoIP2][geoip2] database as an enrichment table. /// /// [maxmind]: https://www.maxmind.com/ @@ -36,17 +67,30 @@ pub enum EnrichmentTables { Mmdb(mmdb::MmdbConfig), } -// TODO: Use `enum_dispatch` here. -impl NamedComponent for EnrichmentTables { - fn get_component_name(&self) -> &'static str { +impl GenerateConfig for EnrichmentTables { + fn generate_config() -> toml::Value { + toml::Value::try_from(Self::File(file::FileConfig { + file: file::FileSettings { + path: "path/to/file".into(), + encoding: file::Encoding::default(), + }, + schema: Default::default(), + })) + .unwrap() + } +} + +impl EnrichmentTables { + /// Gets the files to watch to trigger reload + pub fn files_to_watch(&self) -> Vec<&PathBuf> { match self { - Self::File(config) => config.get_component_name(), + EnrichmentTables::File(file_config) => vec![&file_config.file.path], + #[cfg(feature = "enrichment-tables-memory")] + EnrichmentTables::Memory(_) => vec![], #[cfg(feature = "enrichment-tables-geoip")] - Self::Geoip(config) => config.get_component_name(), + EnrichmentTables::Geoip(geoip_config) => vec![&geoip_config.path], #[cfg(feature = "enrichment-tables-mmdb")] - Self::Mmdb(config) => config.get_component_name(), - #[allow(unreachable_patterns)] - _ => unimplemented!(), + EnrichmentTables::Mmdb(mmdb_config) => vec![&mmdb_config.path], } } } diff --git a/src/expiring_hash_map.rs b/src/expiring_hash_map.rs index 5993ee4351..0345e1d483 100644 --- a/src/expiring_hash_map.rs +++ b/src/expiring_hash_map.rs @@ -10,7 +10,7 @@ use std::{ }; use futures::StreamExt; -use tokio_util::time::{delay_queue, DelayQueue}; +use tokio_util::time::{DelayQueue, delay_queue}; /// An expired item, holding the value and the key with an expiration information. pub type ExpiredItem = (V, delay_queue::Expired); diff --git a/src/extra_context.rs b/src/extra_context.rs index 025093b2ad..b80586bc71 100644 --- a/src/extra_context.rs +++ b/src/extra_context.rs @@ -8,7 +8,7 @@ use std::{ /// Structure containing any extra data. /// The data is held in an [`Arc`] so is cheap to clone. -#[derive(Clone, Default)] +#[derive(Clone, Debug, Default)] pub struct ExtraContext(Arc>); type ContextItem = Box; diff --git a/src/gcp.rs b/src/gcp.rs index 30d2120d0a..bafa33381d 100644 --- a/src/gcp.rs +++ b/src/gcp.rs @@ -4,14 +4,14 @@ use std::{ time::Duration, }; -use base64::prelude::{Engine as _, BASE64_URL_SAFE}; +use base64::prelude::{BASE64_URL_SAFE, Engine as _}; pub use goauth::scopes::Scope; use goauth::{ + GoErr, auth::{JwtClaims, Token, TokenErr}, credentials::Credentials, - GoErr, }; -use http::{uri::PathAndQuery, Uri}; +use http::{Uri, uri::PathAndQuery}; use hyper::header::AUTHORIZATION; use smpl_jwt::Jwt; use snafu::{ResultExt, Snafu}; @@ -198,33 +198,34 @@ impl GcpAuthenticator { async fn token_regenerator(self, sender: watch::Sender<()>) { match self { Self::Credentials(inner) => { - let expires_in = inner.token.read().unwrap().expires_in() as u64; - let mut deadline = - Duration::from_secs(expires_in - METADATA_TOKEN_EXPIRY_MARGIN_SECS); + let mut expires_in = inner.token.read().unwrap().expires_in() as u64; loop { + let deadline = Duration::from_secs( + expires_in + .saturating_sub(METADATA_TOKEN_EXPIRY_MARGIN_SECS) + .max(METADATA_TOKEN_ERROR_RETRY_SECS), + ); + debug!( + deadline = deadline.as_secs(), + "Sleeping before refreshing GCP authentication token.", + ); tokio::time::sleep(deadline).await; - debug!("Renewing GCP authentication token."); match inner.regenerate_token().await { Ok(()) => { sender.send_replace(()); - let expires_in = inner.token.read().unwrap().expires_in() as u64; + debug!("GCP authentication token renewed."); // Rather than an expected fresh token, the Metadata Server may return // the same (cached) token during the last 300 seconds of its lifetime. // This scenario is handled by retrying the token refresh after the // METADATA_TOKEN_ERROR_RETRY_SECS period when a fresh token is expected - let new_deadline = if expires_in <= METADATA_TOKEN_EXPIRY_MARGIN_SECS { - METADATA_TOKEN_ERROR_RETRY_SECS - } else { - expires_in - METADATA_TOKEN_EXPIRY_MARGIN_SECS - }; - deadline = Duration::from_secs(new_deadline); + expires_in = inner.token.read().unwrap().expires_in() as u64; } Err(error) => { error!( message = "Failed to update GCP authentication token.", %error ); - deadline = Duration::from_secs(METADATA_TOKEN_ERROR_RETRY_SECS); + expires_in = METADATA_TOKEN_EXPIRY_MARGIN_SECS; } } } @@ -256,7 +257,13 @@ impl InnerCreds { } async fn fetch_token(creds: &Credentials, scope: &Scope) -> crate::Result { - let claims = JwtClaims::new(creds.iss(), scope, creds.token_uri(), None, None); + let claims = JwtClaims::new( + creds.iss(), + std::slice::from_ref(scope), + creds.token_uri(), + None, + None, + ); let rsa_key = creds.rsa_key().context(InvalidRsaKeySnafu)?; let jwt = Jwt::new(claims, rsa_key, None); diff --git a/src/generate.rs b/src/generate.rs index 745de1e6dd..d634c5fbe8 100644 --- a/src/generate.rs +++ b/src/generate.rs @@ -1,6 +1,6 @@ #![allow(missing_docs)] use std::{ - fs::{create_dir_all, File}, + fs::{File, create_dir_all}, io::Write, path::{Path, PathBuf}, }; @@ -9,13 +9,13 @@ use clap::Parser; use colored::*; use indexmap::IndexMap; use serde::Serialize; -use toml::{map::Map, Value}; +use toml::{Value, map::Map}; use vector_lib::configurable::component::{ SinkDescription, SourceDescription, TransformDescription, }; use vector_lib::{buffers::BufferConfig, config::GlobalOptions, default_data_dir}; -use crate::config::{format, Format, SinkHealthcheckOptions}; +use crate::config::{Format, SinkHealthcheckOptions, format}; #[derive(Parser, Debug)] #[command(rename_all = "kebab-case")] @@ -140,8 +140,7 @@ pub(crate) fn generate_example( let (name, source_type) = if let Some(c_index) = source_expr.find(':') { if c_index == 0 { errs.push(format!( - "failed to generate source '{}': empty name is not allowed", - source_expr + "failed to generate source '{source_expr}': empty name is not allowed" )); continue; } @@ -151,17 +150,14 @@ pub(crate) fn generate_example( chopped_expr.drain(1..).collect(), ) } else { - (format!("source{}", i), source_expr.clone()) + (format!("source{i}"), source_expr.clone()) }; source_names.push(name.clone()); let mut example = match SourceDescription::example(&source_type) { Ok(example) => example, Err(err) => { - errs.push(format!( - "failed to generate source '{}': {}", - source_type, err - )); + errs.push(format!("failed to generate source '{source_type}': {err}")); Value::Table(Map::new()) } }; @@ -186,8 +182,7 @@ pub(crate) fn generate_example( let (name, transform_type) = if let Some(c_index) = transform_expr.find(':') { if c_index == 0 { errs.push(format!( - "failed to generate transform '{}': empty name is not allowed", - transform_expr + "failed to generate transform '{transform_expr}': empty name is not allowed" )); continue; } @@ -197,7 +192,7 @@ pub(crate) fn generate_example( chopped_expr.drain(1..).collect(), ) } else { - (format!("transform{}", i), transform_expr.clone()) + (format!("transform{i}"), transform_expr.clone()) }; transform_names.push(name.clone()); @@ -206,10 +201,12 @@ pub(crate) fn generate_example( if i == 0 { source_names.clone() } else { - vec![transform_names - .get(i - 1) - .unwrap_or(&"component-id".to_owned()) - .to_owned()] + vec![ + transform_names + .get(i - 1) + .unwrap_or(&"component-id".to_owned()) + .to_owned(), + ] } } #[cfg(test)] @@ -220,8 +217,7 @@ pub(crate) fn generate_example( Ok(example) => example, Err(err) => { errs.push(format!( - "failed to generate transform '{}': {}", - transform_type, err + "failed to generate transform '{transform_type}': {err}" )); Value::Table(Map::new()) } @@ -252,8 +248,7 @@ pub(crate) fn generate_example( let (name, sink_type) = if let Some(c_index) = sink_expr.find(':') { if c_index == 0 { errs.push(format!( - "failed to generate sink '{}': empty name is not allowed", - sink_expr + "failed to generate sink '{sink_expr}': empty name is not allowed" )); continue; } @@ -263,13 +258,13 @@ pub(crate) fn generate_example( chopped_expr.drain(1..).collect(), ) } else { - (format!("sink{}", i), sink_expr.clone()) + (format!("sink{i}"), sink_expr.clone()) }; let mut example = match SinkDescription::example(&sink_type) { Ok(example) => example, Err(err) => { - errs.push(format!("failed to generate sink '{}': {}", sink_type, err)); + errs.push(format!("failed to generate sink '{sink_type}': {err}")); Value::Table(Map::new()) } }; @@ -329,9 +324,9 @@ pub(crate) fn generate_example( }; let file = opts.file.as_ref(); - if file.is_some() { - #[allow(clippy::print_stdout)] - match write_config(file.as_ref().unwrap(), &builder) { + if let Some(path) = file { + match write_config(path, &builder) { + #[allow(clippy::print_stdout)] Ok(_) => { println!( "Config file written to {:?}", @@ -354,7 +349,7 @@ pub fn cmd(opts: &Opts) -> exitcode::ExitCode { Ok(s) => { #[allow(clippy::print_stdout)] { - println!("{}", s); + println!("{s}"); } exitcode::OK } @@ -411,15 +406,15 @@ mod tests { #[test] fn generate_all(#[case] format: Format) { for name in SourceDescription::types() { - generate_and_deserialize(format!("{}//", name), format); + generate_and_deserialize(format!("{name}//"), format); } for name in TransformDescription::types() { - generate_and_deserialize(format!("/{}/", name), format); + generate_and_deserialize(format!("/{name}/"), format); } for name in SinkDescription::types() { - generate_and_deserialize(format!("//{}", name), format); + generate_and_deserialize(format!("//{name}"), format); } } @@ -672,13 +667,13 @@ mod tests { sources: source0: count: 9223372036854775807 - format: json - interval: 1.0 - type: demo_logs decoding: codec: bytes + format: json framing: method: bytes + interval: 1.0 + type: demo_logs transforms: transform0: inputs: @@ -693,12 +688,12 @@ mod tests { sink0: inputs: - transform0 - target: stdout - type: console encoding: codec: json json: pretty: false + target: stdout + type: console healthcheck: enabled: true uri: null @@ -732,15 +727,15 @@ mod tests { "sources": { "source0": { "count": 9223372036854775807, - "format": "json", - "interval": 1.0, - "type": "demo_logs", "decoding": { "codec": "bytes" }, + "format": "json", "framing": { "method": "bytes" - } + }, + "interval": 1.0, + "type": "demo_logs" } }, "transforms": { @@ -761,14 +756,14 @@ mod tests { "inputs": [ "transform0" ], - "target": "stdout", - "type": "console", "encoding": { "codec": "json", "json": { "pretty": false } }, + "target": "stdout", + "type": "console", "healthcheck": { "enabled": true, "uri": null diff --git a/src/generate_schema.rs b/src/generate_schema.rs index ab8e830d77..f20a71480c 100644 --- a/src/generate_schema.rs +++ b/src/generate_schema.rs @@ -1,25 +1,52 @@ -#![allow(missing_docs)] +//! Vector `generate-schema` command implementation. + +use clap::Parser; +use std::fs; +use std::path::PathBuf; use vector_lib::configurable::schema::generate_root_schema; use crate::config::ConfigBuilder; -pub fn cmd() -> exitcode::ExitCode { +#[derive(Parser, Debug)] +#[command(rename_all = "kebab-case")] +/// Command line options for the `generate-schema` command. +pub struct Opts { + /// File path to + #[arg(short, long)] + pub(crate) output_path: Option, +} + +/// Execute the `generate-schema` command. +#[allow(clippy::print_stdout, clippy::print_stderr)] +pub fn cmd(opts: &Opts) -> exitcode::ExitCode { match generate_root_schema::() { Ok(schema) => { let json = serde_json::to_string_pretty(&schema) .expect("rendering root schema to JSON should not fail"); - #[allow(clippy::print_stdout)] - { - println!("{}", json); + if let Some(output_path) = &opts.output_path { + if output_path.exists() { + eprintln!("Error: Output file {output_path:?} already exists"); + return exitcode::CANTCREAT; + } + + return match fs::write(output_path, json) { + Ok(_) => { + println!("Schema successfully written to {output_path:?}"); + exitcode::OK + } + Err(e) => { + eprintln!("Error writing to file {output_path:?}: {e:?}"); + exitcode::IOERR + } + }; + } else { + println!("{json}"); } exitcode::OK } Err(e) => { - #[allow(clippy::print_stderr)] - { - eprintln!("error while generating schema: {:?}", e); - } + eprintln!("error while generating schema: {e:?}"); exitcode::SOFTWARE } } diff --git a/src/graph.rs b/src/graph.rs index 8e5b225801..5d7ad0da03 100644 --- a/src/graph.rs +++ b/src/graph.rs @@ -47,6 +47,19 @@ pub struct Opts { value_delimiter(',') )] pub config_dirs: Vec, + + /// Set the output format + /// + /// See https://mermaid.js.org/syntax/flowchart.html#styling-and-classes for + /// information on the `mermaid` format. + #[arg(id = "format", long, default_value = "dot")] + pub format: OutputFormat, +} + +#[derive(clap::ValueEnum, Debug, Clone, Copy, PartialEq, Eq)] +pub enum OutputFormat { + Dot, + Mermaid, } impl Opts { @@ -72,10 +85,7 @@ fn node_attributes_to_string(attributes: &HashMap, default_shape if !attrs.contains_key("shape") { attrs.insert("shape".to_string(), default_shape.to_string()); } - attrs - .iter() - .map(|(k, v)| format!("{}=\"{}\"", k, v)) - .join(" ") + attrs.iter().map(|(k, v)| format!("{k}=\"{v}\"")).join(" ") } pub(crate) fn cmd(opts: &Opts) -> exitcode::ExitCode { @@ -90,12 +100,20 @@ pub(crate) fn cmd(opts: &Opts) -> exitcode::ExitCode { Err(errs) => { #[allow(clippy::print_stderr)] for err in errs { - eprintln!("{}", err); + eprintln!("{err}"); } return exitcode::CONFIG; } }; + let format = opts.format; + match format { + OutputFormat::Dot => render_dot(config), + OutputFormat::Mermaid => render_mermaid(config), + } +} + +fn render_dot(config: config::Config) -> exitcode::ExitCode { let mut dot = String::from("digraph {\n"); for (id, source) in config.sources() { @@ -126,8 +144,7 @@ pub(crate) fn cmd(opts: &Opts) -> exitcode::ExitCode { ) .expect("write to String never fails"); } else { - writeln!(dot, " \"{}\" -> \"{}\"", input, id) - .expect("write to String never fails"); + writeln!(dot, " \"{input}\" -> \"{id}\"").expect("write to String never fails"); } } } @@ -150,8 +167,7 @@ pub(crate) fn cmd(opts: &Opts) -> exitcode::ExitCode { ) .expect("write to String never fails"); } else { - writeln!(dot, " \"{}\" -> \"{}\"", input, id) - .expect("write to String never fails"); + writeln!(dot, " \"{input}\" -> \"{id}\"").expect("write to String never fails"); } } } @@ -160,7 +176,49 @@ pub(crate) fn cmd(opts: &Opts) -> exitcode::ExitCode { #[allow(clippy::print_stdout)] { - println!("{}", dot); + println!("{dot}"); + } + + exitcode::OK +} + +fn render_mermaid(config: config::Config) -> exitcode::ExitCode { + let mut mermaid = String::from("flowchart TD;\n"); + + writeln!(mermaid, "\n %% Sources").unwrap(); + for (id, _) in config.sources() { + writeln!(mermaid, " {id}[/{id}/]").unwrap(); + } + + writeln!(mermaid, "\n %% Transforms").unwrap(); + for (id, transform) in config.transforms() { + writeln!(mermaid, " {id}{{{id}}}").unwrap(); + + for input in transform.inputs.iter() { + if let Some(port) = &input.port { + writeln!(mermaid, " {0} -->|{port}| {id}", input.component).unwrap(); + } else { + writeln!(mermaid, " {0} --> {id}", input.component).unwrap(); + } + } + } + + writeln!(mermaid, "\n %% Sinks").unwrap(); + for (id, sink) in config.sinks() { + writeln!(mermaid, " {id}[\\{id}\\]").unwrap(); + + for input in &sink.inputs { + if let Some(port) = &input.port { + writeln!(mermaid, " {0} -->|{port}| {id}", input.component).unwrap(); + } else { + writeln!(mermaid, " {0} --> {id}", input.component).unwrap(); + } + } + } + + #[allow(clippy::print_stdout)] + { + println!("{mermaid}"); } exitcode::OK diff --git a/src/http.rs b/src/http.rs index 9faaee12ed..ad485e6df4 100644 --- a/src/http.rs +++ b/src/http.rs @@ -2,8 +2,8 @@ use futures::future::BoxFuture; use headers::{Authorization, HeaderMapExt}; use http::{ - header::HeaderValue, request::Builder, uri::InvalidUri, HeaderMap, Request, Response, Uri, - Version, + HeaderMap, Request, Response, Uri, Version, header::HeaderValue, request::Builder, + uri::InvalidUri, }; use hyper::{ body::{Body, HttpBody}, @@ -16,6 +16,7 @@ use rand::Rng; use serde_with::serde_as; use snafu::{ResultExt, Snafu}; use std::{ + collections::HashMap, fmt, net::SocketAddr, task::{Context, Poll}, @@ -31,10 +32,13 @@ use tracing::{Instrument, Span}; use vector_lib::configurable::configurable_component; use vector_lib::sensitive_string::SensitiveString; +#[cfg(feature = "aws-core")] +use crate::aws::AwsAuthentication; + use crate::{ config::ProxyConfig, - internal_events::{http_client, HttpServerRequestReceived, HttpServerResponseSent}, - tls::{tls_connector_builder, MaybeTlsSettings, TlsError}, + internal_events::{HttpServerRequestReceived, HttpServerResponseSent, http_client}, + tls::{MaybeTlsSettings, TlsError, tls_connector_builder}, }; pub mod status { @@ -101,7 +105,7 @@ where let app_name = crate::get_app_name(); let version = crate::get_version(); - let user_agent = HeaderValue::from_str(&format!("{}/{}", app_name, version)) + let user_agent = HeaderValue::from_str(&format!("{app_name}/{version}")) .expect("Invalid header value for user-agent!"); Ok(HttpClient { @@ -273,7 +277,7 @@ impl fmt::Debug for HttpClient { pub enum Auth { /// Basic authentication. /// - /// The username and password are concatenated and encoded via [base64][base64]. + /// The username and password are concatenated and encoded using [base64][base64]. /// /// [base64]: https://en.wikipedia.org/wiki/Base64 Basic { @@ -295,6 +299,16 @@ pub enum Auth { /// The bearer authentication token. token: SensitiveString, }, + + #[cfg(feature = "aws-core")] + /// AWS authentication. + Aws { + /// The AWS authentication configuration. + auth: AwsAuthentication, + + /// The AWS service name to use for signing. + service: String, + }, } pub trait MaybeAuth: Sized { @@ -333,6 +347,8 @@ impl Auth { Ok(auth) => map.typed_insert(auth), Err(error) => error!(message = "Invalid bearer token.", token = %token, %error), }, + #[cfg(feature = "aws-core")] + _ => {} } } } @@ -347,7 +363,7 @@ pub fn get_http_scheme_from_uri(uri: &Uri) -> &'static str { // it also supports arbitrary schemes, which is where we bomb out down here, since we can't generate a static // string for an arbitrary input string... and anything other than "http" and "https" makes no sense for an HTTP // client anyways. - s => panic!("invalid URI scheme for HTTP client: {}", s), + s => panic!("invalid URI scheme for HTTP client: {s}"), }) } @@ -461,8 +477,8 @@ impl MaxConnectionAgeLayer { // Ensure the jitter_factor is between 0.0 and 1.0 let jitter_factor = jitter_factor.clamp(0.0, 1.0); // Generate a random jitter factor between `1 - jitter_factor`` and `1 + jitter_factor`. - let mut rng = rand::thread_rng(); - let random_jitter_factor = rng.gen_range(-jitter_factor..=jitter_factor) + 1.; + let mut rng = rand::rng(); + let random_jitter_factor = rng.random_range(-jitter_factor..=jitter_factor) + 1.; duration.mul_f64(random_jitter_factor) } } @@ -553,11 +569,120 @@ where } } +/// The type of a query parameter's value, determines if it's treated as a plain string or a VRL expression. +#[configurable_component] +#[derive(Clone, Debug, Default, Eq, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum ParamType { + /// The parameter value is a plain string. + #[default] + String, + /// The parameter value is a VRL expression that will be evaluated before each request. + Vrl, +} + +impl ParamType { + fn is_default(&self) -> bool { + *self == Self::default() + } +} + +/// Represents a query parameter value, which can be a simple string or a typed object +/// indicating whether the value is a string or a VRL expression. +#[configurable_component] +#[derive(Clone, Debug, Eq, PartialEq)] +#[serde(untagged)] +pub enum ParameterValue { + /// A simple string value. For backwards compatibility. + String(String), + /// A value with an explicit type. + Typed { + /// The raw value of the parameter. + value: String, + /// The type of the parameter, indicating how the `value` should be treated. + #[serde( + default, + skip_serializing_if = "ParamType::is_default", + rename = "type" + )] + r#type: ParamType, + }, +} + +impl ParameterValue { + /// Returns true if the parameter is a VRL expression. + pub const fn is_vrl(&self) -> bool { + match self { + ParameterValue::String(_) => false, + ParameterValue::Typed { r#type, .. } => matches!(r#type, ParamType::Vrl), + } + } + + /// Returns the raw string value of the parameter. + #[allow(clippy::missing_const_for_fn)] + pub fn value(&self) -> &str { + match self { + ParameterValue::String(s) => s, + ParameterValue::Typed { value, .. } => value, + } + } + + /// Consumes the `ParameterValue` and returns the owned raw string value. + pub fn into_value(self) -> String { + match self { + ParameterValue::String(s) => s, + ParameterValue::Typed { value, .. } => value, + } + } +} + +/// Configuration of the query parameter value for HTTP requests. +#[configurable_component] +#[derive(Clone, Debug, Eq, PartialEq)] +#[serde(untagged)] +#[configurable(metadata(docs::enum_tag_description = "Query parameter value"))] +pub enum QueryParameterValue { + /// Query parameter with single value + SingleParam(ParameterValue), + /// Query parameter with multiple values + MultiParams(Vec), +} + +impl QueryParameterValue { + /// Returns an iterator over the contained `ParameterValue`s. + pub fn iter(&self) -> impl Iterator { + match self { + QueryParameterValue::SingleParam(param) => std::slice::from_ref(param).iter(), + QueryParameterValue::MultiParams(params) => params.iter(), + } + } + + /// Convert to `Vec` for owned iteration. + fn into_vec(self) -> Vec { + match self { + QueryParameterValue::SingleParam(param) => vec![param], + QueryParameterValue::MultiParams(params) => params, + } + } +} + +// Implement IntoIterator for owned QueryParameterValue +impl IntoIterator for QueryParameterValue { + type Item = ParameterValue; + type IntoIter = std::vec::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.into_vec().into_iter() + } +} + +pub type QueryParameters = HashMap; + #[cfg(test)] mod tests { use std::convert::Infallible; - use hyper::{server::conn::AddrStream, service::make_service_fn, Server}; + use hyper::{Server, server::conn::AddrStream, service::make_service_fn}; use proptest::prelude::*; use tower::ServiceBuilder; @@ -777,13 +902,13 @@ mod tests { // Responses generated before the client's max connection age has elapsed do not // include a `Connection: close` header in the response. - let req = Request::get(format!("http://{}/", addr)) + let req = Request::get(format!("http://{addr}/")) .body(Body::empty()) .unwrap(); let response = client.send(req).await.unwrap(); assert_eq!(response.headers().get("Connection"), None); - let req = Request::get(format!("http://{}/", addr)) + let req = Request::get(format!("http://{addr}/")) .body(Body::empty()) .unwrap(); let response = client.send(req).await.unwrap(); @@ -792,7 +917,7 @@ mod tests { // The first response generated after the client's max connection age has elapsed should // include the `Connection: close` header. tokio::time::sleep(Duration::from_secs(1)).await; - let req = Request::get(format!("http://{}/", addr)) + let req = Request::get(format!("http://{addr}/")) .body(Body::empty()) .unwrap(); let response = client.send(req).await.unwrap(); @@ -804,7 +929,7 @@ mod tests { // The next request should establish a new connection. // Importantly, this also confirms that each connection has its own independent // connection age timer. - let req = Request::get(format!("http://{}/", addr)) + let req = Request::get(format!("http://{addr}/")) .body(Body::empty()) .unwrap(); let response = client.send(req).await.unwrap(); diff --git a/src/internal_events/adaptive_concurrency.rs b/src/internal_events/adaptive_concurrency.rs index 03836f5d85..f815f7d4e2 100644 --- a/src/internal_events/adaptive_concurrency.rs +++ b/src/internal_events/adaptive_concurrency.rs @@ -1,6 +1,6 @@ use std::time::Duration; -use metrics::{histogram, Histogram}; +use metrics::{Histogram, histogram}; #[derive(Clone, Copy)] pub struct AdaptiveConcurrencyLimitData { @@ -25,9 +25,9 @@ registered_event! { fn emit(&self, data: AdaptiveConcurrencyLimitData) { self.limit.record(data.concurrency as f64); - let reached_limit = data.reached_limit.then_some(1.0).unwrap_or_default(); + let reached_limit = if data.reached_limit { 1.0 } else { Default::default() }; self.reached_limit.record(reached_limit); - let back_pressure = data.had_back_pressure.then_some(1.0).unwrap_or_default(); + let back_pressure = if data.had_back_pressure { 1.0 } else { Default::default() }; self.back_pressure.record(back_pressure); self.past_rtt_mean.record(data.past_rtt); // past_rtt_deviation is unrecorded diff --git a/src/internal_events/aws_cloudwatch_logs.rs b/src/internal_events/aws_cloudwatch_logs.rs index d4bfe0328a..b64253f8cc 100644 --- a/src/internal_events/aws_cloudwatch_logs.rs +++ b/src/internal_events/aws_cloudwatch_logs.rs @@ -1,6 +1,6 @@ use metrics::counter; use vector_lib::internal_event::InternalEvent; -use vector_lib::internal_event::{error_stage, error_type, ComponentEventsDropped, UNINTENTIONAL}; +use vector_lib::internal_event::{ComponentEventsDropped, UNINTENTIONAL, error_stage, error_type}; #[derive(Debug)] pub struct AwsCloudwatchLogsMessageSizeError { diff --git a/src/internal_events/aws_ecs_metrics.rs b/src/internal_events/aws_ecs_metrics.rs index 1a2f713309..37286c9bfb 100644 --- a/src/internal_events/aws_ecs_metrics.rs +++ b/src/internal_events/aws_ecs_metrics.rs @@ -2,7 +2,7 @@ use std::borrow::Cow; use metrics::counter; use vector_lib::{ - internal_event::{error_stage, error_type, InternalEvent}, + internal_event::{InternalEvent, error_stage, error_type}, json_size::JsonSize, }; diff --git a/src/internal_events/aws_kinesis.rs b/src/internal_events/aws_kinesis.rs index 332f55d08b..adf6a5c0ee 100644 --- a/src/internal_events/aws_kinesis.rs +++ b/src/internal_events/aws_kinesis.rs @@ -1,7 +1,7 @@ /// Used in both `aws_kinesis_streams` and `aws_kinesis_firehose` sinks use metrics::counter; use vector_lib::internal_event::InternalEvent; -use vector_lib::internal_event::{error_stage, error_type, ComponentEventsDropped, UNINTENTIONAL}; +use vector_lib::internal_event::{ComponentEventsDropped, UNINTENTIONAL, error_stage, error_type}; #[derive(Debug)] pub struct AwsKinesisStreamNoPartitionKeyError<'a> { diff --git a/src/internal_events/aws_sqs.rs b/src/internal_events/aws_sqs.rs index 2de4fd7f74..34aaf4d4a3 100644 --- a/src/internal_events/aws_sqs.rs +++ b/src/internal_events/aws_sqs.rs @@ -1,3 +1,5 @@ +#![allow(dead_code)] // TODO requires optional feature compilation + use metrics::counter; #[cfg(feature = "sources-aws_s3")] pub use s3::*; @@ -9,6 +11,7 @@ use vector_lib::internal_event::{error_stage, error_type}; mod s3 { use aws_sdk_sqs::types::{ BatchResultErrorEntry, DeleteMessageBatchRequestEntry, DeleteMessageBatchResultEntry, + SendMessageBatchRequestEntry, SendMessageBatchResultEntry, }; use super::*; @@ -114,6 +117,80 @@ mod s3 { .increment(1); } } + + #[derive(Debug)] + pub struct SqsMessageSentSucceeded { + pub message_ids: Vec, + } + + impl InternalEvent for SqsMessageSentSucceeded { + fn emit(self) { + trace!(message = "Deferred SQS message(s).", + message_ids = %self.message_ids.iter() + .map(|x| x.id.as_str()) + .collect::>() + .join(", ")); + counter!("sqs_message_defer_succeeded_total").increment(self.message_ids.len() as u64); + } + } + + #[derive(Debug)] + pub struct SqsMessageSentPartialError { + pub entries: Vec, + } + + impl InternalEvent for SqsMessageSentPartialError { + fn emit(self) { + error!( + message = "Sending of deferred SQS message(s) failed.", + message_ids = %self.entries.iter() + .map(|x| format!("{}/{}", x.id, x.code)) + .collect::>() + .join(", "), + error_code = "failed_deferring_some_sqs_messages", + error_type = error_type::ACKNOWLEDGMENT_FAILED, + stage = error_stage::PROCESSING, + internal_log_rate_limit = true, + ); + counter!( + "component_errors_total", + "error_code" => "failed_deferring_some_sqs_messages", + "error_type" => error_type::ACKNOWLEDGMENT_FAILED, + "stage" => error_stage::PROCESSING, + ) + .increment(1); + } + } + + #[derive(Debug)] + pub struct SqsMessageSendBatchError { + pub entries: Vec, + pub error: E, + } + + impl InternalEvent for SqsMessageSendBatchError { + fn emit(self) { + error!( + message = "Sending of deferred SQS message(s) failed.", + message_ids = %self.entries.iter() + .map(|x| x.id.as_str()) + .collect::>() + .join(", "), + error = %self.error, + error_code = "failed_deferring_all_sqs_messages", + error_type = error_type::ACKNOWLEDGMENT_FAILED, + stage = error_stage::PROCESSING, + internal_log_rate_limit = true, + ); + counter!( + "component_errors_total", + "error_code" => "failed_deferring_all_sqs_messages", + "error_type" => error_type::ACKNOWLEDGMENT_FAILED, + "stage" => error_stage::PROCESSING, + ) + .increment(1); + } + } } #[derive(Debug)] diff --git a/src/internal_events/batch.rs b/src/internal_events/batch.rs index 816afa4398..3a9be8fca2 100644 --- a/src/internal_events/batch.rs +++ b/src/internal_events/batch.rs @@ -1,6 +1,6 @@ use metrics::counter; use vector_lib::internal_event::InternalEvent; -use vector_lib::internal_event::{error_stage, error_type, ComponentEventsDropped, UNINTENTIONAL}; +use vector_lib::internal_event::{ComponentEventsDropped, UNINTENTIONAL, error_stage, error_type}; #[derive(Debug)] pub struct LargeEventDroppedError { diff --git a/src/internal_events/codecs.rs b/src/internal_events/codecs.rs index 50251bf089..2182b215cf 100644 --- a/src/internal_events/codecs.rs +++ b/src/internal_events/codecs.rs @@ -1,6 +1,6 @@ use metrics::counter; use vector_lib::internal_event::InternalEvent; -use vector_lib::internal_event::{error_stage, error_type, ComponentEventsDropped, UNINTENTIONAL}; +use vector_lib::internal_event::{ComponentEventsDropped, UNINTENTIONAL, error_stage, error_type}; #[derive(Debug)] pub struct DecoderFramingError { diff --git a/src/internal_events/common.rs b/src/internal_events/common.rs index 78971840d7..c91e980e96 100644 --- a/src/internal_events/common.rs +++ b/src/internal_events/common.rs @@ -3,7 +3,7 @@ use std::time::Instant; use metrics::{counter, histogram}; pub use vector_lib::internal_event::EventsReceived; use vector_lib::internal_event::InternalEvent; -use vector_lib::internal_event::{error_stage, error_type, ComponentEventsDropped, UNINTENTIONAL}; +use vector_lib::internal_event::{ComponentEventsDropped, UNINTENTIONAL, error_stage, error_type}; #[derive(Debug)] pub struct EndpointBytesReceived<'a> { diff --git a/src/internal_events/datadog_metrics.rs b/src/internal_events/datadog_metrics.rs index 8875b29281..90fe5177a7 100644 --- a/src/internal_events/datadog_metrics.rs +++ b/src/internal_events/datadog_metrics.rs @@ -1,6 +1,6 @@ use metrics::counter; use vector_lib::internal_event::InternalEvent; -use vector_lib::internal_event::{error_stage, error_type, ComponentEventsDropped, UNINTENTIONAL}; +use vector_lib::internal_event::{ComponentEventsDropped, UNINTENTIONAL, error_stage, error_type}; #[derive(Debug)] pub struct DatadogMetricsEncodingError<'a> { diff --git a/src/internal_events/datadog_traces.rs b/src/internal_events/datadog_traces.rs index dd1df213cd..e88797ceef 100644 --- a/src/internal_events/datadog_traces.rs +++ b/src/internal_events/datadog_traces.rs @@ -1,6 +1,6 @@ use metrics::counter; use vector_lib::internal_event::InternalEvent; -use vector_lib::internal_event::{error_stage, error_type, ComponentEventsDropped, UNINTENTIONAL}; +use vector_lib::internal_event::{ComponentEventsDropped, UNINTENTIONAL, error_stage, error_type}; #[derive(Debug)] pub struct DatadogTracesEncodingError { diff --git a/src/internal_events/dedupe.rs b/src/internal_events/dedupe.rs index d88930e0b6..c4a781de2c 100644 --- a/src/internal_events/dedupe.rs +++ b/src/internal_events/dedupe.rs @@ -1,4 +1,4 @@ -use vector_lib::internal_event::{ComponentEventsDropped, InternalEvent, INTENTIONAL}; +use vector_lib::internal_event::{ComponentEventsDropped, INTENTIONAL, InternalEvent}; #[derive(Debug)] pub struct DedupeEventsDropped { diff --git a/src/internal_events/exec.rs b/src/internal_events/exec.rs index 539471719d..d743cbec05 100644 --- a/src/internal_events/exec.rs +++ b/src/internal_events/exec.rs @@ -4,7 +4,7 @@ use metrics::{counter, histogram}; use tokio::time::error::Elapsed; use vector_lib::internal_event::InternalEvent; use vector_lib::{ - internal_event::{error_stage, error_type, ComponentEventsDropped, UNINTENTIONAL}, + internal_event::{ComponentEventsDropped, UNINTENTIONAL, error_stage, error_type}, json_size::JsonSize, }; @@ -153,7 +153,7 @@ impl ExecFailedToSignalChild { match self { #[cfg(unix)] - SignalError(err) => format!("errno_{}", err), + SignalError(err) => format!("errno_{err}"), #[cfg(unix)] FailedToMarshalPid(_) => String::from("failed_to_marshal_pid"), #[cfg(unix)] @@ -170,9 +170,9 @@ impl std::fmt::Display for ExecFailedToSignalChild { match self { #[cfg(unix)] - SignalError(err) => write!(f, "errno: {}", err), + SignalError(err) => write!(f, "errno: {err}"), #[cfg(unix)] - FailedToMarshalPid(err) => write!(f, "failed to marshal pid to i32: {}", err), + FailedToMarshalPid(err) => write!(f, "failed to marshal pid to i32: {err}"), #[cfg(unix)] NoPid => write!(f, "child had no pid"), #[cfg(windows)] diff --git a/src/internal_events/expansion.rs b/src/internal_events/expansion.rs new file mode 100644 index 0000000000..cf24ebd042 --- /dev/null +++ b/src/internal_events/expansion.rs @@ -0,0 +1,46 @@ +use metrics::counter; +use vector_lib::internal_event::{ComponentEventsDropped, InternalEvent, UNINTENTIONAL}; +use vector_lib::internal_event::{error_stage, error_type}; + +pub struct PairExpansionError<'a> { + pub key: &'a str, + pub value: &'a str, + pub drop_event: bool, + pub error: serde_json::Error, +} + +impl InternalEvent for PairExpansionError<'_> { + fn emit(self) { + let message = format!("Failed to expand key: `{}`:`{}`", self.key, self.value); + + if self.drop_event { + error!( + message = %message, + error = %self.error, + error_type = error_type::PARSER_FAILED, + stage = error_stage::PROCESSING, + internal_log_rate_limit = true, + ); + + counter!( + "component_errors_total", + "error_type" => error_type::PARSER_FAILED, + "stage" => error_stage::PROCESSING, + ) + .increment(1); + + emit!(ComponentEventsDropped:: { + count: 1, + reason: &message, + }); + } else { + warn!( + message = %message, + error = %self.error, + error_type = error_type::PARSER_FAILED, + stage = error_stage::PROCESSING, + internal_log_rate_limit = true, + ); + } + } +} diff --git a/src/internal_events/file.rs b/src/internal_events/file.rs index fba86d6ad1..252cd7d795 100644 --- a/src/internal_events/file.rs +++ b/src/internal_events/file.rs @@ -1,3 +1,5 @@ +#![allow(dead_code)] // TODO requires optional feature compilation + use metrics::{counter, gauge}; use std::borrow::Cow; use vector_lib::{ @@ -106,8 +108,10 @@ impl InternalEvent for FileIoError<'_, P> { mod source { use std::{io::Error, path::Path, time::Duration}; + use bytes::BytesMut; use metrics::counter; - use vector_lib::file_source::FileSourceInternalEvents; + use vector_lib::file_source_common::internal_events::FileSourceInternalEvents; + use vector_lib::internal_event::{ComponentEventsDropped, INTENTIONAL}; use super::{FileOpen, InternalEvent}; use vector_lib::emit; @@ -496,6 +500,38 @@ mod source { } } + #[derive(Debug)] + pub struct FileLineTooBigError<'a> { + pub truncated_bytes: &'a BytesMut, + pub configured_limit: usize, + pub encountered_size_so_far: usize, + } + + impl InternalEvent for FileLineTooBigError<'_> { + fn emit(self) { + error!( + message = "Found line that exceeds max_line_bytes; discarding.", + truncated_bytes = ?self.truncated_bytes, + configured_limit = self.configured_limit, + encountered_size_so_far = self.encountered_size_so_far, + internal_log_rate_limit = true, + error_type = error_type::CONDITION_FAILED, + stage = error_stage::RECEIVING, + ); + counter!( + "component_errors_total", + "error_code" => "reading_line_from_file", + "error_type" => error_type::CONDITION_FAILED, + "stage" => error_stage::RECEIVING, + ) + .increment(1); + emit!(ComponentEventsDropped:: { + count: 1, + reason: "Found line that exceeds max_line_bytes; discarding.", + }); + } + } + #[derive(Clone)] pub struct FileSourceInternalEventsEmitter { pub include_file_metric_tag: bool, @@ -578,5 +614,18 @@ mod source { fn emit_path_globbing_failed(&self, path: &Path, error: &Error) { emit!(PathGlobbingError { path, error }); } + + fn emit_file_line_too_long( + &self, + truncated_bytes: &bytes::BytesMut, + configured_limit: usize, + encountered_size_so_far: usize, + ) { + emit!(FileLineTooBigError { + truncated_bytes, + configured_limit, + encountered_size_so_far + }); + } } } diff --git a/src/internal_events/filter.rs b/src/internal_events/filter.rs index cb54a36852..0c7fc73930 100644 --- a/src/internal_events/filter.rs +++ b/src/internal_events/filter.rs @@ -1,4 +1,4 @@ -use vector_lib::internal_event::{ComponentEventsDropped, Count, Registered, INTENTIONAL}; +use vector_lib::internal_event::{ComponentEventsDropped, Count, INTENTIONAL, Registered}; vector_lib::registered_event! ( FilterEventsDropped => { diff --git a/src/internal_events/http_client.rs b/src/internal_events/http_client.rs index a6fde2f1a6..8af9cc974a 100644 --- a/src/internal_events/http_client.rs +++ b/src/internal_events/http_client.rs @@ -1,10 +1,10 @@ use std::time::Duration; use http::{ - header::{self, HeaderMap, HeaderValue}, Request, Response, + header::{self, HeaderMap, HeaderValue}, }; -use hyper::{body::HttpBody, Error}; +use hyper::{Error, body::HttpBody}; use metrics::{counter, histogram}; use vector_lib::internal_event::InternalEvent; use vector_lib::internal_event::{error_stage, error_type}; @@ -103,13 +103,13 @@ impl std::fmt::Display for FormatBody<'_, B> { let size = self.0.size_hint(); match (size.lower(), size.upper()) { (0, None) => write!(fmt, "[unknown]"), - (lower, None) => write!(fmt, "[>={} bytes]", lower), + (lower, None) => write!(fmt, "[>={lower} bytes]"), (0, Some(0)) => write!(fmt, "[empty]"), - (0, Some(upper)) => write!(fmt, "[<={} bytes]", upper), + (0, Some(upper)) => write!(fmt, "[<={upper} bytes]"), - (lower, Some(upper)) if lower == upper => write!(fmt, "[{} bytes]", lower), - (lower, Some(upper)) => write!(fmt, "[{}..={} bytes]", lower, upper), + (lower, Some(upper)) if lower == upper => write!(fmt, "[{lower} bytes]"), + (lower, Some(upper)) => write!(fmt, "[{lower}..={upper} bytes]"), } } } diff --git a/src/internal_events/http_client_source.rs b/src/internal_events/http_client_source.rs index 376adf35ed..7bbcd1d72a 100644 --- a/src/internal_events/http_client_source.rs +++ b/src/internal_events/http_client_source.rs @@ -1,3 +1,5 @@ +#![allow(dead_code)] // TODO requires optional feature compilation + use metrics::counter; use vector_lib::internal_event::InternalEvent; use vector_lib::{ diff --git a/src/internal_events/influxdb.rs b/src/internal_events/influxdb.rs index 007f5565f1..4b9d0dfa6b 100644 --- a/src/internal_events/influxdb.rs +++ b/src/internal_events/influxdb.rs @@ -1,6 +1,6 @@ use metrics::counter; use vector_lib::internal_event::InternalEvent; -use vector_lib::internal_event::{error_stage, error_type, ComponentEventsDropped, UNINTENTIONAL}; +use vector_lib::internal_event::{ComponentEventsDropped, UNINTENTIONAL, error_stage, error_type}; #[derive(Debug)] pub struct InfluxdbEncodingError { diff --git a/src/internal_events/kafka.rs b/src/internal_events/kafka.rs index ea1b198be7..d62bec3858 100644 --- a/src/internal_events/kafka.rs +++ b/src/internal_events/kafka.rs @@ -1,3 +1,5 @@ +#![allow(dead_code)] // TODO requires optional feature compilation + use metrics::{counter, gauge}; use vector_lib::internal_event::InternalEvent; use vector_lib::{ diff --git a/src/internal_events/kubernetes_logs.rs b/src/internal_events/kubernetes_logs.rs index 77dac30b19..28ed04fd77 100644 --- a/src/internal_events/kubernetes_logs.rs +++ b/src/internal_events/kubernetes_logs.rs @@ -1,9 +1,10 @@ use metrics::counter; -use vector_lib::internal_event::InternalEvent; +use vector_lib::internal_event::{INTENTIONAL, InternalEvent}; use vector_lib::{ - internal_event::{error_stage, error_type, ComponentEventsDropped, UNINTENTIONAL}, + internal_event::{ComponentEventsDropped, UNINTENTIONAL, error_stage, error_type}, json_size::JsonSize, }; +use vrl::core::Value; use crate::event::Event; @@ -205,3 +206,35 @@ impl InternalEvent for KubernetesLifecycleError { }); } } + +#[derive(Debug)] +pub struct KubernetesMergedLineTooBigError<'a> { + pub event: &'a Value, + pub configured_limit: usize, + pub encountered_size_so_far: usize, +} + +impl InternalEvent for KubernetesMergedLineTooBigError<'_> { + fn emit(self) { + error!( + message = "Found line that exceeds max_merged_line_bytes; discarding.", + event = ?self.event, + configured_limit = self.configured_limit, + encountered_size_so_far = self.encountered_size_so_far, + internal_log_rate_limit = true, + error_type = error_type::CONDITION_FAILED, + stage = error_stage::RECEIVING, + ); + counter!( + "component_errors_total", + "error_code" => "reading_line_from_kubernetes_log", + "error_type" => error_type::CONDITION_FAILED, + "stage" => error_stage::RECEIVING, + ) + .increment(1); + emit!(ComponentEventsDropped:: { + count: 1, + reason: "Found line that exceeds max_merged_line_bytes; discarding.", + }); + } +} diff --git a/src/internal_events/log_to_metric.rs b/src/internal_events/log_to_metric.rs index 0469cdacdd..0bf851330d 100644 --- a/src/internal_events/log_to_metric.rs +++ b/src/internal_events/log_to_metric.rs @@ -2,7 +2,7 @@ use std::num::ParseFloatError; use metrics::counter; use vector_lib::internal_event::InternalEvent; -use vector_lib::internal_event::{error_stage, error_type, ComponentEventsDropped, UNINTENTIONAL}; +use vector_lib::internal_event::{ComponentEventsDropped, UNINTENTIONAL, error_stage, error_type}; pub struct LogToMetricFieldNullError<'a> { pub field: &'a str, diff --git a/src/internal_events/loki.rs b/src/internal_events/loki.rs index 2c95d524c6..d251771994 100644 --- a/src/internal_events/loki.rs +++ b/src/internal_events/loki.rs @@ -1,6 +1,6 @@ use metrics::counter; +use vector_lib::internal_event::{ComponentEventsDropped, INTENTIONAL, InternalEvent}; use vector_lib::internal_event::{error_stage, error_type}; -use vector_lib::internal_event::{ComponentEventsDropped, InternalEvent, INTENTIONAL}; #[derive(Debug)] pub struct LokiEventUnlabeledError; diff --git a/src/internal_events/lua.rs b/src/internal_events/lua.rs index 2805b5b579..f39070fcb6 100644 --- a/src/internal_events/lua.rs +++ b/src/internal_events/lua.rs @@ -1,6 +1,6 @@ use metrics::{counter, gauge}; use vector_lib::internal_event::InternalEvent; -use vector_lib::internal_event::{error_stage, error_type, ComponentEventsDropped, UNINTENTIONAL}; +use vector_lib::internal_event::{ComponentEventsDropped, UNINTENTIONAL, error_stage, error_type}; use crate::transforms::lua::v2::BuildError; diff --git a/src/internal_events/metric_to_log.rs b/src/internal_events/metric_to_log.rs index 79a9d11705..a7a3e19253 100644 --- a/src/internal_events/metric_to_log.rs +++ b/src/internal_events/metric_to_log.rs @@ -1,7 +1,7 @@ use metrics::counter; use serde_json::Error; use vector_lib::internal_event::InternalEvent; -use vector_lib::internal_event::{error_stage, error_type, ComponentEventsDropped, UNINTENTIONAL}; +use vector_lib::internal_event::{ComponentEventsDropped, UNINTENTIONAL, error_stage, error_type}; #[derive(Debug)] pub struct MetricToLogSerializeError { diff --git a/src/internal_events/mod.rs b/src/internal_events/mod.rs index 85bdbad78b..b30eddda95 100644 --- a/src/internal_events/mod.rs +++ b/src/internal_events/mod.rs @@ -2,6 +2,7 @@ pub mod prelude; mod adaptive_concurrency; +#[cfg(feature = "transforms-aggregate")] mod aggregate; #[cfg(any(feature = "sources-amqp", feature = "sinks-amqp"))] mod amqp; @@ -78,6 +79,7 @@ mod kafka; mod kubernetes_logs; #[cfg(feature = "transforms-log_to_metric")] mod log_to_metric; +#[cfg(feature = "sources-heroku_logs")] mod logplex; #[cfg(feature = "sinks-loki")] mod loki; @@ -92,6 +94,11 @@ mod mqtt; #[cfg(feature = "sources-nginx_metrics")] mod nginx_metrics; mod open; +#[cfg(any( + feature = "sources-kubernetes_logs", + feature = "transforms-log_to_metric", + feature = "sinks-datadog_events", +))] mod parser; #[cfg(feature = "sources-postgresql_metrics")] mod postgresql_metrics; @@ -108,7 +115,9 @@ mod pulsar; mod redis; #[cfg(feature = "transforms-impl-reduce")] mod reduce; +#[cfg(feature = "transforms-remap")] mod remap; +#[cfg(feature = "transforms-impl-sample")] mod sample; #[cfg(feature = "sinks-sematext")] mod sematext_metrics; @@ -124,9 +133,14 @@ mod template; #[cfg(feature = "transforms-throttle")] mod throttle; mod udp; +#[cfg(unix)] mod unix; -#[cfg(feature = "sinks-websocket")] +#[cfg(any(feature = "sources-websocket", feature = "sinks-websocket"))] mod websocket; +#[cfg(feature = "sinks-websocket-server")] +mod websocket_server; +#[cfg(feature = "transforms-window")] +mod window; #[cfg(any( feature = "sources-file", @@ -134,8 +148,15 @@ mod websocket; feature = "sinks-file", ))] mod file; + +#[cfg(windows)] mod windows; +#[cfg(any(feature = "transforms-log_to_metric", feature = "sinks-loki"))] +mod expansion; +#[cfg(any(feature = "transforms-log_to_metric", feature = "sinks-loki"))] +pub use self::expansion::*; + #[cfg(feature = "sources-mongodb_metrics")] pub(crate) use mongodb_metrics::*; @@ -227,7 +248,11 @@ pub(crate) use self::metric_to_log::*; pub(crate) use self::mqtt::*; #[cfg(feature = "sources-nginx_metrics")] pub(crate) use self::nginx_metrics::*; -#[allow(unused_imports)] +#[cfg(any( + feature = "sources-kubernetes_logs", + feature = "transforms-log_to_metric", + feature = "sinks-datadog_events", +))] pub(crate) use self::parser::*; #[cfg(feature = "sources-postgresql_metrics")] pub(crate) use self::postgresql_metrics::*; @@ -259,10 +284,15 @@ pub(crate) use self::tag_cardinality_limit::*; pub(crate) use self::throttle::*; #[cfg(unix)] pub(crate) use self::unix::*; -#[cfg(feature = "sinks-websocket")] +#[cfg(any(feature = "sources-websocket", feature = "sinks-websocket"))] pub(crate) use self::websocket::*; +#[cfg(feature = "sinks-websocket-server")] +pub(crate) use self::websocket_server::*; +#[cfg(feature = "transforms-window")] +pub(crate) use self::window::*; #[cfg(windows)] pub(crate) use self::windows::*; + pub use self::{ adaptive_concurrency::*, batch::*, common::*, conditions::*, encoding_transcode::*, heartbeat::*, http::*, open::*, process::*, socket::*, tcp::*, template::*, udp::*, diff --git a/src/internal_events/open.rs b/src/internal_events/open.rs index b5a58d4011..ccd9d8cf54 100644 --- a/src/internal_events/open.rs +++ b/src/internal_events/open.rs @@ -1,8 +1,8 @@ use std::{ hint, sync::{ - atomic::{AtomicUsize, Ordering}, Arc, + atomic::{AtomicUsize, Ordering}, }, }; diff --git a/src/internal_events/parser.rs b/src/internal_events/parser.rs index 84cd0556b4..46f296a114 100644 --- a/src/internal_events/parser.rs +++ b/src/internal_events/parser.rs @@ -1,10 +1,12 @@ +#![allow(dead_code)] // TODO requires optional feature compilation + use std::borrow::Cow; use metrics::counter; use vector_lib::internal_event::InternalEvent; -use vector_lib::internal_event::{error_stage, error_type, ComponentEventsDropped, UNINTENTIONAL}; +use vector_lib::internal_event::{ComponentEventsDropped, UNINTENTIONAL, error_stage, error_type}; -fn truncate_string_at(s: &str, maxlen: usize) -> Cow { +fn truncate_string_at(s: &str, maxlen: usize) -> Cow<'_, str> { let ellipsis: &str = "[...]"; if s.len() >= maxlen { let mut len = maxlen - ellipsis.len(); diff --git a/src/internal_events/prelude.rs b/src/internal_events/prelude.rs index 8da12ce4b8..820ae81440 100644 --- a/src/internal_events/prelude.rs +++ b/src/internal_events/prelude.rs @@ -5,9 +5,14 @@ feature = "sources-utils-http", ))] pub(crate) fn http_error_code(code: u16) -> String { - format!("http_response_{}", code) + format!("http_response_{code}") } +#[cfg(any( + feature = "sources-aws_kinesis_firehose", + feature = "sources-exec", + feature = "sources-heroku_logs", +))] pub(crate) fn io_error_code(error: &std::io::Error) -> &'static str { use std::io::ErrorKind::*; diff --git a/src/internal_events/prometheus.rs b/src/internal_events/prometheus.rs index 752e8e238f..984fd9980b 100644 --- a/src/internal_events/prometheus.rs +++ b/src/internal_events/prometheus.rs @@ -1,9 +1,11 @@ +#![allow(dead_code)] // TODO requires optional feature compilation + #[cfg(feature = "sources-prometheus-scrape")] use std::borrow::Cow; use metrics::counter; use vector_lib::internal_event::InternalEvent; -use vector_lib::internal_event::{error_stage, error_type, ComponentEventsDropped, UNINTENTIONAL}; +use vector_lib::internal_event::{ComponentEventsDropped, UNINTENTIONAL, error_stage, error_type}; #[cfg(feature = "sources-prometheus-scrape")] use vector_lib::prometheus::parser::ParserError; diff --git a/src/internal_events/pulsar.rs b/src/internal_events/pulsar.rs index c03dea3870..838a2aadd9 100644 --- a/src/internal_events/pulsar.rs +++ b/src/internal_events/pulsar.rs @@ -1,8 +1,10 @@ -use metrics::counter; +#![allow(dead_code)] // TODO requires optional feature compilation + #[cfg(feature = "sources-pulsar")] use metrics::Counter; +use metrics::counter; use vector_lib::internal_event::{ - error_stage, error_type, ComponentEventsDropped, InternalEvent, UNINTENTIONAL, + ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, }; #[derive(Debug)] diff --git a/src/internal_events/reduce.rs b/src/internal_events/reduce.rs index b9887aaf54..87b68b89d4 100644 --- a/src/internal_events/reduce.rs +++ b/src/internal_events/reduce.rs @@ -1,5 +1,5 @@ use metrics::counter; -use vector_lib::internal_event::{error_stage, error_type, InternalEvent}; +use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; use vrl::path::PathParseError; use vrl::value::KeyString; diff --git a/src/internal_events/remap.rs b/src/internal_events/remap.rs index d05e6dfb9c..0275262655 100644 --- a/src/internal_events/remap.rs +++ b/src/internal_events/remap.rs @@ -1,7 +1,7 @@ use metrics::counter; use vector_lib::internal_event::InternalEvent; use vector_lib::internal_event::{ - error_stage, error_type, ComponentEventsDropped, INTENTIONAL, UNINTENTIONAL, + ComponentEventsDropped, INTENTIONAL, UNINTENTIONAL, error_stage, error_type, }; #[derive(Debug)] diff --git a/src/internal_events/sample.rs b/src/internal_events/sample.rs index c113b2df54..c502272908 100644 --- a/src/internal_events/sample.rs +++ b/src/internal_events/sample.rs @@ -1,4 +1,4 @@ -use vector_lib::internal_event::{ComponentEventsDropped, InternalEvent, INTENTIONAL}; +use vector_lib::internal_event::{ComponentEventsDropped, INTENTIONAL, InternalEvent}; #[derive(Debug)] pub struct SampleEventDiscarded; diff --git a/src/internal_events/sematext_metrics.rs b/src/internal_events/sematext_metrics.rs index b4b5be737f..41ebe60fed 100644 --- a/src/internal_events/sematext_metrics.rs +++ b/src/internal_events/sematext_metrics.rs @@ -1,6 +1,6 @@ use metrics::counter; use vector_lib::internal_event::InternalEvent; -use vector_lib::internal_event::{error_stage, error_type, ComponentEventsDropped, UNINTENTIONAL}; +use vector_lib::internal_event::{ComponentEventsDropped, UNINTENTIONAL, error_stage, error_type}; use crate::event::metric::Metric; diff --git a/src/internal_events/socket.rs b/src/internal_events/socket.rs index 97a648da25..5f2f7aac30 100644 --- a/src/internal_events/socket.rs +++ b/src/internal_events/socket.rs @@ -1,3 +1,5 @@ +use std::net::Ipv4Addr; + use metrics::{counter, histogram}; use vector_lib::internal_event::{ComponentEventsDropped, InternalEvent, UNINTENTIONAL}; use vector_lib::{ @@ -120,7 +122,7 @@ impl InternalEvent for SocketBindError { error = %self.error, error_code = "socket_bind", error_type = error_type::IO_FAILED, - stage = error_stage::RECEIVING, + stage = error_stage::INITIALIZING, %mode, internal_log_rate_limit = true, ); @@ -128,8 +130,46 @@ impl InternalEvent for SocketBindError { "component_errors_total", "error_code" => "socket_bind", "error_type" => error_type::IO_FAILED, - "stage" => error_stage::RECEIVING, + "stage" => error_stage::INITIALIZING, + "mode" => mode, + ) + .increment(1); + } +} + +#[derive(Debug)] +pub struct SocketMulticastGroupJoinError { + pub error: E, + pub group_addr: Ipv4Addr, + pub interface: Ipv4Addr, +} + +impl InternalEvent for SocketMulticastGroupJoinError { + fn emit(self) { + // Multicast groups are only used in UDP mode + let mode = SocketMode::Udp.as_str(); + let group_addr = self.group_addr.to_string(); + let interface = self.interface.to_string(); + + error!( + message = "Error joining multicast group.", + error = %self.error, + error_code = "socket_multicast_group_join", + error_type = error_type::IO_FAILED, + stage = error_stage::INITIALIZING, + %mode, + %group_addr, + %interface, + internal_log_rate_limit = true, + ); + counter!( + "component_errors_total", + "error_code" => "socket_multicast_group_join", + "error_type" => error_type::IO_FAILED, + "stage" => error_stage::INITIALIZING, "mode" => mode, + "group_addr" => group_addr, + "interface" => interface, ) .increment(1); } diff --git a/src/internal_events/splunk_hec.rs b/src/internal_events/splunk_hec.rs index 4ff5838cdc..52232f3624 100644 --- a/src/internal_events/splunk_hec.rs +++ b/src/internal_events/splunk_hec.rs @@ -11,7 +11,7 @@ mod sink { use serde_json::Error; use vector_lib::internal_event::InternalEvent; use vector_lib::internal_event::{ - error_stage, error_type, ComponentEventsDropped, UNINTENTIONAL, + ComponentEventsDropped, UNINTENTIONAL, error_stage, error_type, }; use crate::{ diff --git a/src/internal_events/statsd_sink.rs b/src/internal_events/statsd_sink.rs index 165ef885ea..c22af8f6cb 100644 --- a/src/internal_events/statsd_sink.rs +++ b/src/internal_events/statsd_sink.rs @@ -2,7 +2,7 @@ use metrics::counter; use vector_lib::internal_event::InternalEvent; use crate::event::metric::{MetricKind, MetricValue}; -use vector_lib::internal_event::{error_stage, error_type, ComponentEventsDropped, UNINTENTIONAL}; +use vector_lib::internal_event::{ComponentEventsDropped, UNINTENTIONAL, error_stage, error_type}; #[derive(Debug)] pub struct StatsdInvalidMetricError<'a> { diff --git a/src/internal_events/tag_cardinality_limit.rs b/src/internal_events/tag_cardinality_limit.rs index 4c0c315eb1..9db97e39f4 100644 --- a/src/internal_events/tag_cardinality_limit.rs +++ b/src/internal_events/tag_cardinality_limit.rs @@ -1,5 +1,5 @@ use metrics::counter; -use vector_lib::internal_event::{ComponentEventsDropped, InternalEvent, INTENTIONAL}; +use vector_lib::internal_event::{ComponentEventsDropped, INTENTIONAL, InternalEvent}; pub struct TagCardinalityLimitRejectingEvent<'a> { pub metric_name: &'a str, diff --git a/src/internal_events/tcp.rs b/src/internal_events/tcp.rs index 23ec1fc88e..5ff074a158 100644 --- a/src/internal_events/tcp.rs +++ b/src/internal_events/tcp.rs @@ -1,7 +1,7 @@ use std::net::SocketAddr; use metrics::counter; -use vector_lib::internal_event::{error_stage, error_type, InternalEvent}; +use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; use crate::{internal_events::SocketOutgoingConnectionError, tls::TlsError}; diff --git a/src/internal_events/template.rs b/src/internal_events/template.rs index 82df77cefd..5d16bc46c7 100644 --- a/src/internal_events/template.rs +++ b/src/internal_events/template.rs @@ -1,6 +1,6 @@ use metrics::counter; -use vector_lib::internal_event::{error_stage, error_type}; use vector_lib::internal_event::{ComponentEventsDropped, InternalEvent, UNINTENTIONAL}; +use vector_lib::internal_event::{error_stage, error_type}; pub struct TemplateRenderingError<'a> { pub field: Option<&'a str>, @@ -13,7 +13,7 @@ impl InternalEvent for TemplateRenderingError<'_> { let mut msg = "Failed to render template".to_owned(); if let Some(field) = self.field { use std::fmt::Write; - _ = write!(msg, " for \"{}\"", field); + _ = write!(msg, " for \"{field}\""); } msg.push('.'); diff --git a/src/internal_events/throttle.rs b/src/internal_events/throttle.rs index 3bbc67be88..5999033d4b 100644 --- a/src/internal_events/throttle.rs +++ b/src/internal_events/throttle.rs @@ -1,5 +1,5 @@ use metrics::counter; -use vector_lib::internal_event::{ComponentEventsDropped, InternalEvent, INTENTIONAL}; +use vector_lib::internal_event::{ComponentEventsDropped, INTENTIONAL, InternalEvent}; #[derive(Debug)] pub(crate) struct ThrottleEventDiscarded { diff --git a/src/internal_events/udp.rs b/src/internal_events/udp.rs index 15e9df4c2b..26f63aa879 100644 --- a/src/internal_events/udp.rs +++ b/src/internal_events/udp.rs @@ -1,6 +1,6 @@ use metrics::counter; use vector_lib::internal_event::{ - error_stage, error_type, ComponentEventsDropped, InternalEvent, UNINTENTIONAL, + ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, }; use crate::internal_events::SocketOutgoingConnectionError; diff --git a/src/internal_events/unix.rs b/src/internal_events/unix.rs index a45777ab8f..ea91cd483b 100644 --- a/src/internal_events/unix.rs +++ b/src/internal_events/unix.rs @@ -1,8 +1,10 @@ +#![allow(dead_code)] // TODO requires optional feature compilation + use std::{io::Error, path::Path}; use metrics::counter; use vector_lib::internal_event::{ - error_stage, error_type, ComponentEventsDropped, InternalEvent, UNINTENTIONAL, + ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, }; use crate::internal_events::SocketOutgoingConnectionError; diff --git a/src/internal_events/websocket.rs b/src/internal_events/websocket.rs index 59d6457494..ac037a6af0 100644 --- a/src/internal_events/websocket.rs +++ b/src/internal_events/websocket.rs @@ -1,43 +1,51 @@ +#![allow(dead_code)] // TODO requires optional feature compilation + use std::error::Error; -use std::fmt::Debug; +use std::fmt::{Debug, Display, Formatter, Result}; -use metrics::counter; +use metrics::{counter, histogram}; +use tokio_tungstenite::tungstenite::error::Error as TungsteniteError; use vector_lib::internal_event::InternalEvent; -use vector_lib::internal_event::{error_stage, error_type}; +use vector_common::{ + internal_event::{error_stage, error_type}, + json_size::JsonSize, +}; + +pub const PROTOCOL: &str = "websocket"; #[derive(Debug)] -pub struct WsConnectionEstablished; +pub struct WebSocketConnectionEstablished; -impl InternalEvent for WsConnectionEstablished { +impl InternalEvent for WebSocketConnectionEstablished { fn emit(self) { debug!(message = "Connected."); counter!("connection_established_total").increment(1); } fn name(&self) -> Option<&'static str> { - Some("WsConnectionEstablished") + Some("WebSocketConnectionEstablished") } } #[derive(Debug)] -pub struct WsConnectionFailedError { +pub struct WebSocketConnectionFailedError { pub error: Box, } -impl InternalEvent for WsConnectionFailedError { +impl InternalEvent for WebSocketConnectionFailedError { fn emit(self) { error!( message = "WebSocket connection failed.", error = %self.error, - error_code = "ws_connection_error", + error_code = "websocket_connection_error", error_type = error_type::CONNECTION_FAILED, stage = error_stage::SENDING, internal_log_rate_limit = true, ); counter!( "component_errors_total", - "error_code" => "ws_connection_failed", + "error_code" => "websocket_connection_failed", "error_type" => error_type::CONNECTION_FAILED, "stage" => error_stage::SENDING, ) @@ -45,42 +53,43 @@ impl InternalEvent for WsConnectionFailedError { } fn name(&self) -> Option<&'static str> { - Some("WsConnectionFailed") + Some("WebSocketConnectionFailedError") } } #[derive(Debug)] -pub struct WsConnectionShutdown; +pub struct WebSocketConnectionShutdown; -impl InternalEvent for WsConnectionShutdown { +impl InternalEvent for WebSocketConnectionShutdown { fn emit(self) { warn!(message = "Closed by the server."); counter!("connection_shutdown_total").increment(1); } fn name(&self) -> Option<&'static str> { - Some("WsConnectionShutdown") + Some("WebSocketConnectionShutdown") } } #[derive(Debug)] -pub struct WsConnectionError { +pub struct WebSocketConnectionError { pub error: tokio_tungstenite::tungstenite::Error, } -impl InternalEvent for WsConnectionError { +impl InternalEvent for WebSocketConnectionError { fn emit(self) { error!( message = "WebSocket connection error.", error = %self.error, - error_code = "ws_connection_error", + error_code = "websocket_connection_error", error_type = error_type::WRITER_FAILED, stage = error_stage::SENDING, internal_log_rate_limit = true, ); counter!( "component_errors_total", - "error_code" => "ws_connection_error", + "protocol" => PROTOCOL, + "error_code" => "websocket_connection_error", "error_type" => error_type::WRITER_FAILED, "stage" => error_stage::SENDING, ) @@ -88,6 +97,152 @@ impl InternalEvent for WsConnectionError { } fn name(&self) -> Option<&'static str> { - Some("WsConnectionError") + Some("WebSocketConnectionError") + } +} + +#[allow(dead_code)] +#[derive(Debug, Copy, Clone)] +pub enum WebSocketKind { + Ping, + Pong, + Text, + Binary, + Close, + Frame, +} + +impl Display for WebSocketKind { + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + write!(f, "{self:?}") + } +} + +#[derive(Debug)] +pub struct WebSocketBytesReceived<'a> { + pub byte_size: usize, + pub url: &'a str, + pub protocol: &'static str, + pub kind: WebSocketKind, +} + +impl InternalEvent for WebSocketBytesReceived<'_> { + fn emit(self) { + trace!( + message = "Bytes received.", + byte_size = %self.byte_size, + url = %self.url, + protocol = %self.protocol, + kind = %self.kind + ); + let counter = counter!( + "component_received_bytes_total", + "url" => self.url.to_string(), + "protocol" => self.protocol, + "kind" => self.kind.to_string() + ); + counter.increment(self.byte_size as u64); + } +} + +#[derive(Debug)] +pub struct WebSocketMessageReceived<'a> { + pub count: usize, + pub byte_size: JsonSize, + pub url: &'a str, + pub protocol: &'static str, + pub kind: WebSocketKind, +} + +impl InternalEvent for WebSocketMessageReceived<'_> { + fn emit(self) { + trace!( + message = "Events received.", + count = %self.count, + byte_size = %self.byte_size, + url = %self.url, + protcol = %self.protocol, + kind = %self.kind + ); + + let histogram = histogram!("component_received_events_count"); + histogram.record(self.count as f64); + let counter = counter!( + "component_received_events_total", + "uri" => self.url.to_string(), + "protocol" => PROTOCOL, + "kind" => self.kind.to_string() + ); + counter.increment(self.count as u64); + let counter = counter!( + "component_received_event_bytes_total", + "url" => self.url.to_string(), + "protocol" => PROTOCOL, + "kind" => self.kind.to_string() + ); + counter.increment(self.byte_size.get() as u64); + } + + fn name(&self) -> Option<&'static str> { + Some("WebSocketMessageReceived") + } +} + +#[derive(Debug)] +pub struct WebSocketReceiveError<'a> { + pub error: &'a TungsteniteError, +} + +impl InternalEvent for WebSocketReceiveError<'_> { + fn emit(self) { + error!( + message = "Error receiving message from websocket.", + error = %self.error, + error_code = "websocket_receive_error", + error_type = error_type::CONNECTION_FAILED, + stage = error_stage::PROCESSING, + internal_log_rate_limit = true, + ); + counter!( + "component_errors_total", + "protocol" => PROTOCOL, + "error_code" => "websocket_receive_error", + "error_type" => error_type::CONNECTION_FAILED, + "stage" => error_stage::PROCESSING, + ) + .increment(1); + } + + fn name(&self) -> Option<&'static str> { + Some("WebSocketReceiveError") + } +} + +#[derive(Debug)] +pub struct WebSocketSendError<'a> { + pub error: &'a TungsteniteError, +} + +impl InternalEvent for WebSocketSendError<'_> { + fn emit(self) { + error!( + message = "Error sending message to websocket.", + error = %self.error, + error_code = "websocket_send_error", + error_type = error_type::CONNECTION_FAILED, + stage = error_stage::PROCESSING, + internal_log_rate_limit = true, + ); + counter!( + "component_errors_total", + "error_code" => "websocket_send_error", + "error_type" => error_type::CONNECTION_FAILED, + "stage" => error_stage::PROCESSING, + ) + .increment(1); + } + + fn name(&self) -> Option<&'static str> { + Some("WebSocketSendError") } } diff --git a/src/internal_events/websocket_server.rs b/src/internal_events/websocket_server.rs new file mode 100644 index 0000000000..b928385e66 --- /dev/null +++ b/src/internal_events/websocket_server.rs @@ -0,0 +1,135 @@ +use std::error::Error; +use std::fmt::Debug; + +use metrics::{counter, gauge}; +use vector_lib::internal_event::InternalEvent; + +use vector_lib::internal_event::{error_stage, error_type}; + +#[derive(Debug)] +pub struct WebSocketListenerConnectionEstablished { + pub client_count: usize, + pub extra_tags: Vec<(String, String)>, +} + +impl InternalEvent for WebSocketListenerConnectionEstablished { + fn emit(self) { + debug!( + message = format!( + "Websocket client connected. Client count: {}", + self.client_count + ) + ); + counter!("connection_established_total", &self.extra_tags).increment(1); + gauge!("active_clients", &self.extra_tags).set(self.client_count as f64); + } + + fn name(&self) -> Option<&'static str> { + Some("WebSocketListenerConnectionEstablished") + } +} + +#[derive(Debug)] +pub struct WebSocketListenerConnectionFailedError { + pub error: Box, + pub extra_tags: Vec<(String, String)>, +} + +impl InternalEvent for WebSocketListenerConnectionFailedError { + fn emit(self) { + error!( + message = "WebSocket connection failed.", + error = %self.error, + error_code = "ws_connection_error", + error_type = error_type::CONNECTION_FAILED, + stage = error_stage::SENDING, + internal_log_rate_limit = true, + ); + let mut all_tags = self.extra_tags.clone(); + all_tags.extend([ + ("error_code".to_string(), "ws_connection_failed".to_string()), + ( + "error_type".to_string(), + error_type::CONNECTION_FAILED.to_string(), + ), + ("stage".to_string(), error_stage::SENDING.to_string()), + ]); + // Tags required by `component_errors_total` are dynamically added above. + // ## skip check-validity-events ## + counter!("component_errors_total", &all_tags).increment(1); + } + + fn name(&self) -> Option<&'static str> { + Some("WsListenerConnectionFailed") + } +} + +#[derive(Debug)] +pub struct WebSocketListenerConnectionShutdown { + pub client_count: usize, + pub extra_tags: Vec<(String, String)>, +} + +impl InternalEvent for WebSocketListenerConnectionShutdown { + fn emit(self) { + info!( + message = format!( + "Client connection closed. Client count: {}.", + self.client_count + ) + ); + counter!("connection_shutdown_total", &self.extra_tags).increment(1); + gauge!("active_clients", &self.extra_tags).set(self.client_count as f64); + } + + fn name(&self) -> Option<&'static str> { + Some("WebSocketListenerConnectionShutdown") + } +} + +#[derive(Debug)] +pub struct WebSocketListenerSendError { + pub error: Box, +} + +impl InternalEvent for WebSocketListenerSendError { + fn emit(self) { + error!( + message = "WebSocket message send error.", + error = %self.error, + error_code = "ws_server_connection_error", + error_type = error_type::WRITER_FAILED, + stage = error_stage::SENDING, + internal_log_rate_limit = true, + ); + counter!( + "component_errors_total", + "error_code" => "ws_server_connection_error", + "error_type" => error_type::WRITER_FAILED, + "stage" => error_stage::SENDING, + ) + .increment(1); + } + + fn name(&self) -> Option<&'static str> { + Some("WsListenerConnectionError") + } +} + +#[derive(Debug)] +pub struct WebSocketListenerMessageSent { + pub message_size: usize, + pub extra_tags: Vec<(String, String)>, +} + +impl InternalEvent for WebSocketListenerMessageSent { + fn emit(self) { + counter!("websocket_messages_sent_total", &self.extra_tags).increment(1); + counter!("websocket_bytes_sent_total", &self.extra_tags) + .increment(self.message_size as u64); + } + + fn name(&self) -> Option<&'static str> { + Some("WebSocketListenerMessageSent") + } +} diff --git a/src/internal_events/window.rs b/src/internal_events/window.rs new file mode 100644 index 0000000000..42eb62203e --- /dev/null +++ b/src/internal_events/window.rs @@ -0,0 +1,14 @@ +use vector_lib::internal_event::{ComponentEventsDropped, Count, INTENTIONAL, Registered}; + +vector_lib::registered_event!( + WindowEventsDropped => { + events_dropped: Registered> + = register!(ComponentEventsDropped::::from( + "The buffer was full" + )), + } + + fn emit(&self, data: Count) { + self.events_dropped.emit(data); + } +); diff --git a/src/internal_telemetry/allocations/allocator/stack.rs b/src/internal_telemetry/allocations/allocator/stack.rs index 5479654501..1772be0125 100644 --- a/src/internal_telemetry/allocations/allocator/stack.rs +++ b/src/internal_telemetry/allocations/allocator/stack.rs @@ -33,7 +33,9 @@ impl GroupStack { pub fn push(&mut self, group: AllocationGroupId) { self.current_top += 1; if self.current_top >= self.slots.len() { - panic!("tried to push new allocation group to the current stack, but hit the limit of {} entries", N); + panic!( + "tried to push new allocation group to the current stack, but hit the limit of {N} entries" + ); } self.slots[self.current_top] = group; } diff --git a/src/internal_telemetry/allocations/allocator/token.rs b/src/internal_telemetry/allocations/allocator/token.rs index 8409320e01..8af37c8003 100644 --- a/src/internal_telemetry/allocations/allocator/token.rs +++ b/src/internal_telemetry/allocations/allocator/token.rs @@ -64,10 +64,10 @@ impl AllocationGroupId { /// group. pub fn attach_to_span(self, span: &Span) { tracing::dispatcher::get_default(move |dispatch| { - if let Some(id) = span.id() { - if let Some(ctx) = dispatch.downcast_ref::() { - (ctx.with_allocation_group)(dispatch, &id, AllocationGroupToken::from(self)); - } + if let Some(id) = span.id() + && let Some(ctx) = dispatch.downcast_ref::() + { + (ctx.with_allocation_group)(dispatch, &id, AllocationGroupToken::from(self)); } }); } diff --git a/src/internal_telemetry/allocations/allocator/tracing.rs b/src/internal_telemetry/allocations/allocator/tracing.rs index ba768b8704..36f57df6d5 100644 --- a/src/internal_telemetry/allocations/allocator/tracing.rs +++ b/src/internal_telemetry/allocations/allocator/tracing.rs @@ -1,7 +1,7 @@ use std::{any::TypeId, marker::PhantomData, ptr::addr_of}; use tracing::{Dispatch, Id, Subscriber}; -use tracing_subscriber::{layer::Context, registry::LookupSpan, Layer}; +use tracing_subscriber::{Layer, layer::Context, registry::LookupSpan}; use super::AllocationGroupToken; @@ -54,18 +54,18 @@ where S: Subscriber + for<'a> LookupSpan<'a>, { fn on_enter(&self, id: &Id, ctx: Context<'_, S>) { - if let Some(span_ref) = ctx.span(id) { - if let Some(token) = span_ref.extensions().get::() { - token.enter(); - } + if let Some(span_ref) = ctx.span(id) + && let Some(token) = span_ref.extensions().get::() + { + token.enter(); } } fn on_exit(&self, id: &Id, ctx: Context<'_, S>) { - if let Some(span_ref) = ctx.span(id) { - if let Some(token) = span_ref.extensions().get::() { - token.exit(); - } + if let Some(span_ref) = ctx.span(id) + && let Some(token) = span_ref.extensions().get::() + { + token.exit(); } } diff --git a/src/internal_telemetry/allocations/allocator/tracing_allocator.rs b/src/internal_telemetry/allocations/allocator/tracing_allocator.rs index 4c48928d07..241c24e583 100644 --- a/src/internal_telemetry/allocations/allocator/tracing_allocator.rs +++ b/src/internal_telemetry/allocations/allocator/tracing_allocator.rs @@ -6,7 +6,7 @@ use std::{ use crate::internal_telemetry::allocations::TRACK_ALLOCATIONS; use super::{ - token::{try_with_suspended_allocation_group, AllocationGroupId}, + token::{AllocationGroupId, try_with_suspended_allocation_group}, tracer::Tracer, }; @@ -29,55 +29,59 @@ impl GroupedTraceableAllocator { unsafe impl GlobalAlloc for GroupedTraceableAllocator { #[inline] unsafe fn alloc(&self, object_layout: Layout) -> *mut u8 { - if !TRACK_ALLOCATIONS.load(Ordering::Relaxed) { - return self.allocator.alloc(object_layout); + unsafe { + if !TRACK_ALLOCATIONS.load(Ordering::Relaxed) { + return self.allocator.alloc(object_layout); + } + + // Allocate our wrapped layout and make sure the allocation succeeded. + let (actual_layout, offset_to_group_id) = get_wrapped_layout(object_layout); + let actual_ptr = self.allocator.alloc(actual_layout); + if actual_ptr.is_null() { + return actual_ptr; + } + + let group_id_ptr = actual_ptr.add(offset_to_group_id).cast::(); + + let object_size = object_layout.size(); + + try_with_suspended_allocation_group( + #[inline(always)] + |group_id| { + group_id_ptr.write(group_id.as_raw()); + self.tracer.trace_allocation(object_size, group_id); + }, + ); + actual_ptr } - - // Allocate our wrapped layout and make sure the allocation succeeded. - let (actual_layout, offset_to_group_id) = get_wrapped_layout(object_layout); - let actual_ptr = self.allocator.alloc(actual_layout); - if actual_ptr.is_null() { - return actual_ptr; - } - - let group_id_ptr = actual_ptr.add(offset_to_group_id).cast::(); - - let object_size = object_layout.size(); - - try_with_suspended_allocation_group( - #[inline(always)] - |group_id| { - group_id_ptr.write(group_id.as_raw()); - self.tracer.trace_allocation(object_size, group_id); - }, - ); - actual_ptr } #[inline] unsafe fn dealloc(&self, object_ptr: *mut u8, object_layout: Layout) { - if !TRACK_ALLOCATIONS.load(Ordering::Relaxed) { - self.allocator.dealloc(object_ptr, object_layout); - return; + unsafe { + if !TRACK_ALLOCATIONS.load(Ordering::Relaxed) { + self.allocator.dealloc(object_ptr, object_layout); + return; + } + // Regenerate the wrapped layout so we know where we have to look, as the pointer we've given relates to the + // requested layout, not the wrapped layout that was actually allocated. + let (wrapped_layout, offset_to_group_id) = get_wrapped_layout(object_layout); + + let raw_group_id = object_ptr.add(offset_to_group_id).cast::().read(); + + // Deallocate before tracking, just to make sure we're reclaiming memory as soon as possible. + self.allocator.dealloc(object_ptr, wrapped_layout); + + let object_size = object_layout.size(); + let source_group_id = AllocationGroupId::from_raw(raw_group_id); + + try_with_suspended_allocation_group( + #[inline(always)] + |_| { + self.tracer.trace_deallocation(object_size, source_group_id); + }, + ); } - // Regenerate the wrapped layout so we know where we have to look, as the pointer we've given relates to the - // requested layout, not the wrapped layout that was actually allocated. - let (wrapped_layout, offset_to_group_id) = get_wrapped_layout(object_layout); - - let raw_group_id = object_ptr.add(offset_to_group_id).cast::().read(); - - // Deallocate before tracking, just to make sure we're reclaiming memory as soon as possible. - self.allocator.dealloc(object_ptr, wrapped_layout); - - let object_size = object_layout.size(); - let source_group_id = AllocationGroupId::from_raw(raw_group_id); - - try_with_suspended_allocation_group( - #[inline(always)] - |_| { - self.tracer.trace_deallocation(object_size, source_group_id); - }, - ); } } diff --git a/src/internal_telemetry/allocations/mod.rs b/src/internal_telemetry/allocations/mod.rs index 0b0aecc6a9..658b73c207 100644 --- a/src/internal_telemetry/allocations/mod.rs +++ b/src/internal_telemetry/allocations/mod.rs @@ -3,8 +3,8 @@ mod allocator; use std::{ sync::{ - atomic::{AtomicBool, AtomicU64, Ordering}, Mutex, + atomic::{AtomicBool, AtomicU64, Ordering}, }, thread, time::Duration, @@ -17,10 +17,10 @@ use rand_distr::num_traits::ToPrimitive; use self::allocator::Tracer; pub(crate) use self::allocator::{ - without_allocation_tracing, AllocationGroupId, AllocationLayer, GroupedTraceableAllocator, + AllocationGroupId, AllocationLayer, GroupedTraceableAllocator, without_allocation_tracing, }; -const NUM_GROUPS: usize = 128; +const NUM_GROUPS: usize = 256; // Allocations are not tracked during startup. // We use the Relaxed ordering for both stores and loads of this atomic as no other threads exist when @@ -55,8 +55,8 @@ impl GroupMemStats { pub fn new() -> Self { let mut mutex = THREAD_LOCAL_REFS.lock().unwrap(); let stats_ref: &'static GroupMemStatsStorage = Box::leak(Box::new(GroupMemStatsStorage { - allocations: arr![AtomicU64::new(0) ; 128], - deallocations: arr![AtomicU64::new(0) ; 128], + allocations: arr![AtomicU64::new(0) ; 256], + deallocations: arr![AtomicU64::new(0) ; 256], })); let group_mem_stats = GroupMemStats { stats: stats_ref }; mutex.push(stats_ref); @@ -84,7 +84,7 @@ impl GroupInfo { } } -static GROUP_INFO: [Mutex; NUM_GROUPS] = arr![Mutex::new(GroupInfo::new()); 128]; +static GROUP_INFO: [Mutex; NUM_GROUPS] = arr![Mutex::new(GroupInfo::new()); 256]; pub type Allocator = GroupedTraceableAllocator; @@ -190,22 +190,22 @@ pub fn acquire_allocation_group_id( component_type: String, component_kind: String, ) -> AllocationGroupId { - if let Some(group_id) = AllocationGroupId::register() { - if let Some(group_lock) = GROUP_INFO.get(group_id.as_raw() as usize) { - let mut writer = group_lock.lock().unwrap(); - *writer = GroupInfo { - component_id, - component_kind, - component_type, - }; - - return group_id; - } + if let Some(group_id) = AllocationGroupId::register() + && let Some(group_lock) = GROUP_INFO.get(group_id.as_raw() as usize) + { + let mut writer = group_lock.lock().unwrap(); + *writer = GroupInfo { + component_id, + component_kind, + component_type, + }; + + return group_id; } - // TODO: Technically, `NUM_GROUPS` is lower (128) than the upper bound for the - // `AllocationGroupId::register` call itself (253), so we can hardcode `NUM_GROUPS` here knowing - // it's the lower of the two values and will trigger first.. but this may not always be true. - warn!("Maximum number of registrable allocation group IDs reached ({}). Allocations for component '{}' will be attributed to the root allocation group.", NUM_GROUPS, component_id); + warn!( + "Maximum number of registrable allocation group IDs reached ({}). Allocations for component '{}' will be attributed to the root allocation group.", + NUM_GROUPS, component_id + ); AllocationGroupId::ROOT } diff --git a/src/kafka.rs b/src/kafka.rs index 8567d19a06..902129d3ec 100644 --- a/src/kafka.rs +++ b/src/kafka.rs @@ -1,14 +1,14 @@ #![allow(missing_docs)] use std::path::{Path, PathBuf}; -use rdkafka::{consumer::ConsumerContext, ClientConfig, ClientContext, Statistics}; +use rdkafka::{ClientConfig, ClientContext, Statistics, consumer::ConsumerContext}; use snafu::Snafu; use tracing::Span; use vector_lib::configurable::configurable_component; use vector_lib::sensitive_string::SensitiveString; use crate::{ - internal_events::KafkaStatisticsReceived, tls::TlsEnableableConfig, tls::PEM_START_MARKER, + internal_events::KafkaStatisticsReceived, tls::PEM_START_MARKER, tls::TlsEnableableConfig, }; #[derive(Debug, Snafu)] diff --git a/src/kubernetes/reflector.rs b/src/kubernetes/reflector.rs index 14460c7398..998b02bd28 100644 --- a/src/kubernetes/reflector.rs +++ b/src/kubernetes/reflector.rs @@ -5,8 +5,8 @@ use std::{hash::Hash, sync::Arc, time::Duration}; use futures::StreamExt; use futures_util::Stream; use kube::{ - runtime::{reflector::store, watcher}, Resource, + runtime::{reflector::store, watcher}, }; use tokio::pin; use tokio_util::time::DelayQueue; @@ -132,12 +132,12 @@ mod tests { use futures_util::SinkExt; use k8s_openapi::{api::core::v1::ConfigMap, apimachinery::pkg::apis::meta::v1::ObjectMeta}; use kube::runtime::{ - reflector::{store, ObjectRef}, + reflector::{ObjectRef, store}, watcher, }; - use super::custom_reflector; use super::MetaCache; + use super::custom_reflector; #[tokio::test] async fn applied_should_add_object() { diff --git a/src/lib.rs b/src/lib.rs index 04048952af..5d98678655 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -21,6 +21,9 @@ //! The main library to support building Vector. +#[cfg(all(unix, feature = "sinks-socket"))] +#[macro_use] +extern crate cfg_if; #[macro_use] extern crate derivative; #[macro_use] @@ -89,7 +92,7 @@ pub mod kubernetes; pub mod line_agg; pub mod list; #[cfg(any(feature = "sources-nats", feature = "sinks-nats"))] -pub(crate) mod nats; +pub mod nats; pub mod net; #[allow(unreachable_pub)] pub(crate) mod proto; @@ -127,8 +130,8 @@ pub mod validate; pub mod vector_windows; pub use source_sender::SourceSender; +pub use vector_lib::{Error, Result, shutdown}; pub use vector_lib::{event, metrics, schema, tcp, tls}; -pub use vector_lib::{shutdown, Error, Result}; static APP_NAME_SLUG: std::sync::OnceLock = std::sync::OnceLock::new(); @@ -184,12 +187,12 @@ pub fn get_version() -> String { // or full debug symbols. See the Cargo Book profiling section for value meaning: // https://doc.rust-lang.org/cargo/reference/profiles.html#debug let build_string = match built_info::DEBUG { - "1" => format!("{} debug=line", build_string), - "2" | "true" => format!("{} debug=full", build_string), + "1" => format!("{build_string} debug=line"), + "2" | "true" => format!("{build_string} debug=full"), _ => build_string, }; - format!("{} ({})", pkg_version, build_string) + format!("{pkg_version} ({build_string})") } /// Includes information about the current build. diff --git a/src/line_agg.rs b/src/line_agg.rs index 35d37a4fb6..80a4f8c081 100644 --- a/src/line_agg.rs +++ b/src/line_agg.rs @@ -3,7 +3,7 @@ #![deny(missing_docs)] use std::{ - collections::{hash_map::Entry, HashMap}, + collections::{HashMap, hash_map::Entry}, hash::Hash, pin::Pin, task::{Context, Poll}, @@ -183,10 +183,13 @@ where // If we're in draining mode, short circuit here. if let Some(to_drain) = &mut this.draining { - if let Some(val) = to_drain.pop() { - return Poll::Ready(Some(val)); - } else { - return Poll::Ready(None); + match to_drain.pop() { + Some(val) => { + return Poll::Ready(Some(val)); + } + _ => { + return Poll::Ready(None); + } } } @@ -749,7 +752,7 @@ mod tests { "START msg 1".to_string(), // will be stashed ]; for i in 0..n { - lines.push(format!("line {}", i)); + lines.push(format!("line {i}")); } let config = Config { start_pattern: Regex::new("").unwrap(), @@ -760,7 +763,7 @@ mod tests { let mut expected = "START msg 1".to_string(); for i in 0..n { - write!(expected, "\nline {}", i).expect("write to String never fails"); + write!(expected, "\nline {i}").expect("write to String never fails"); } let (mut send, recv) = futures::channel::mpsc::unbounded(); diff --git a/src/list.rs b/src/list.rs index 566b07215d..3cedbc91a9 100644 --- a/src/list.rs +++ b/src/list.rs @@ -40,22 +40,22 @@ pub fn cmd(opts: &Opts) -> exitcode::ExitCode { Format::Text => { println!("Sources:"); for name in sources { - println!("- {}", name); + println!("- {name}"); } println!("\nTransforms:"); for name in transforms { - println!("- {}", name); + println!("- {name}"); } println!("\nSinks:"); for name in sinks { - println!("- {}", name); + println!("- {name}"); } println!("\nEnrichment tables:"); for name in enrichment_tables { - println!("- {}", name); + println!("- {name}"); } } Format::Json => { diff --git a/src/main.rs b/src/main.rs index eedc25bb43..6129f1948a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -10,7 +10,7 @@ fn main() -> ExitCode { #[cfg(feature = "allocation-tracing")] { use crate::vector::internal_telemetry::allocations::{ - init_allocation_tracing, REPORTING_INTERVAL_MS, TRACK_ALLOCATIONS, + REPORTING_INTERVAL_MS, TRACK_ALLOCATIONS, init_allocation_tracing, }; use std::sync::atomic::Ordering; let opts = vector::cli::Opts::get_matches() diff --git a/src/nats.rs b/src/nats.rs index 93a0a765ed..bd98c06443 100644 --- a/src/nats.rs +++ b/src/nats.rs @@ -1,3 +1,6 @@ +//! Shared helper functions for NATS source and sink. +#![allow(missing_docs)] + use nkeys::error::Error as NKeysError; use snafu::{ResultExt, Snafu}; use vector_lib::configurable::configurable_component; @@ -5,6 +8,7 @@ use vector_lib::sensitive_string::SensitiveString; use crate::tls::TlsEnableableConfig; +/// Errors that can occur during NATS configuration. #[derive(Debug, Snafu)] pub enum NatsConfigError { #[snafu(display("NATS Auth Config Error: {}", source))] @@ -30,7 +34,7 @@ NATS [documentation][nats_auth_docs]. For TLS client certificate authentication [nats_auth_docs]: https://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro" ))] -pub(crate) enum NatsAuthConfig { +pub enum NatsAuthConfig { /// Username/password authentication. UserPassword { #[configurable(derived)] @@ -65,7 +69,7 @@ impl std::fmt::Display for NatsAuthConfig { CredentialsFile { .. } => "credentials_file", Nkey { .. } => "nkey", }; - write!(f, "{}", word) + write!(f, "{word}") } } @@ -73,7 +77,7 @@ impl std::fmt::Display for NatsAuthConfig { #[configurable_component] #[derive(Clone, Debug)] #[serde(deny_unknown_fields)] -pub(crate) struct NatsAuthUserPassword { +pub struct NatsAuthUserPassword { /// Username. pub(crate) user: String, @@ -85,7 +89,7 @@ pub(crate) struct NatsAuthUserPassword { #[configurable_component] #[derive(Clone, Debug)] #[serde(deny_unknown_fields)] -pub(crate) struct NatsAuthToken { +pub struct NatsAuthToken { /// Token. pub(crate) value: SensitiveString, } @@ -94,7 +98,7 @@ pub(crate) struct NatsAuthToken { #[configurable_component] #[derive(Clone, Debug)] #[serde(deny_unknown_fields)] -pub(crate) struct NatsAuthCredentialsFile { +pub struct NatsAuthCredentialsFile { /// Path to credentials file. #[configurable(metadata(docs::examples = "/etc/nats/nats.creds"))] pub(crate) path: String, @@ -104,7 +108,7 @@ pub(crate) struct NatsAuthCredentialsFile { #[configurable_component] #[derive(Clone, Debug)] #[serde(deny_unknown_fields)] -pub(crate) struct NatsAuthNKey { +pub struct NatsAuthNKey { /// User. /// /// Conceptually, this is equivalent to a public key. diff --git a/src/providers/http.rs b/src/providers/http.rs index 8b19285035..5f6f6b1d28 100644 --- a/src/providers/http.rs +++ b/src/providers/http.rs @@ -8,7 +8,7 @@ use url::Url; use vector_lib::configurable::configurable_component; use crate::{ - config::{self, provider::ProviderConfig, ProxyConfig}, + config::{self, Format, ProxyConfig, provider::ProviderConfig}, http::HttpClient, signal, tls::{TlsConfig, TlsSettings}, @@ -53,6 +53,10 @@ pub struct HttpConfig { #[configurable(derived)] #[serde(default, skip_serializing_if = "crate::serde::is_default")] proxy: ProxyConfig, + + /// Which config format expected to be loaded + #[configurable(derived)] + config_format: Format, } impl Default for HttpConfig { @@ -63,6 +67,7 @@ impl Default for HttpConfig { poll_interval_secs: 30, tls_options: None, proxy: Default::default(), + config_format: Format::default(), } } } @@ -126,12 +131,13 @@ async fn http_request_to_config_builder( tls_options: Option<&TlsConfig>, headers: &IndexMap, proxy: &ProxyConfig, + config_format: &Format, ) -> BuildResult { let config_str = http_request(url, tls_options, headers, proxy) .await .map_err(|e| vec![e.to_owned()])?; - config::load(config_str.chunk(), crate::config::format::Format::Toml) + config::load(config_str.chunk(), *config_format) } /// Polls the HTTP endpoint after/every `poll_interval_secs`, returning a stream of `ConfigBuilder`. @@ -141,6 +147,7 @@ fn poll_http( tls_options: Option, headers: IndexMap, proxy: ProxyConfig, + config_format: Format, ) -> impl Stream { let duration = time::Duration::from_secs(poll_interval_secs); let mut interval = time::interval_at(time::Instant::now() + duration, duration); @@ -149,7 +156,7 @@ fn poll_http( loop { interval.tick().await; - match http_request_to_config_builder(&url, tls_options.as_ref(), &headers, &proxy).await { + match http_request_to_config_builder(&url, tls_options.as_ref(), &headers, &proxy, &config_format).await { Ok(config_builder) => yield signal::SignalTo::ReloadFromConfigBuilder(config_builder), Err(_) => {}, }; @@ -172,11 +179,17 @@ impl ProviderConfig for HttpConfig { let tls_options = self.tls_options.take(); let poll_interval_secs = self.poll_interval_secs; let request = self.request.clone(); + let config_format = self.config_format; let proxy = ProxyConfig::from_env().merge(&self.proxy); - let config_builder = - http_request_to_config_builder(&url, tls_options.as_ref(), &request.headers, &proxy) - .await?; + let config_builder = http_request_to_config_builder( + &url, + tls_options.as_ref(), + &request.headers, + &proxy, + &config_format, + ) + .await?; // Poll for changes to remote configuration. signal_handler.add(poll_http( @@ -185,6 +198,7 @@ impl ProviderConfig for HttpConfig { tls_options, request.headers.clone(), proxy.clone(), + config_format, )); Ok(config_builder) diff --git a/src/providers/mod.rs b/src/providers/mod.rs index 1a7f8a1ab9..f789643291 100644 --- a/src/providers/mod.rs +++ b/src/providers/mod.rs @@ -1,6 +1,6 @@ #![allow(missing_docs)] use enum_dispatch::enum_dispatch; -use vector_lib::configurable::{configurable_component, NamedComponent}; +use vector_lib::configurable::{NamedComponent, configurable_component}; use crate::{ config::{ConfigBuilder, ProviderConfig}, diff --git a/src/secrets/aws_secrets_manager.rs b/src/secrets/aws_secrets_manager.rs index 3d402536c2..bf2c6865e1 100644 --- a/src/secrets/aws_secrets_manager.rs +++ b/src/secrets/aws_secrets_manager.rs @@ -1,9 +1,9 @@ use std::collections::{HashMap, HashSet}; -use aws_sdk_secretsmanager::{config, Client}; +use aws_sdk_secretsmanager::{Client, config}; use vector_lib::configurable::{component::GenerateConfig, configurable_component}; -use crate::aws::{create_client, AwsAuthentication, ClientBuilder, RegionOrEndpoint}; +use crate::aws::{AwsAuthentication, ClientBuilder, RegionOrEndpoint, create_client}; use crate::config::ProxyConfig; use crate::tls::TlsConfig; use crate::{config::SecretBackend, signal}; diff --git a/src/secrets/directory.rs b/src/secrets/directory.rs index e9ebaebbdd..8170c208fd 100644 --- a/src/secrets/directory.rs +++ b/src/secrets/directory.rs @@ -43,7 +43,7 @@ impl SecretBackend for DirectoryBackend { &contents }; if secret.is_empty() { - return Err(format!("secret in file '{}' was empty", k).into()); + return Err(format!("secret in file '{k}' was empty").into()); } secrets.insert(k, secret.to_string()); } diff --git a/src/secrets/exec.rs b/src/secrets/exec.rs index bf300c0610..5956af7c06 100644 --- a/src/secrets/exec.rs +++ b/src/secrets/exec.rs @@ -7,9 +7,59 @@ use serde::{Deserialize, Serialize}; use tokio::{io::AsyncWriteExt, process::Command, time}; use tokio_util::codec; use vector_lib::configurable::{component::GenerateConfig, configurable_component}; +use vrl::value::Value; use crate::{config::SecretBackend, signal}; +/// Configuration for the command that will be `exec`ed +#[configurable_component(secrets("exec"))] +#[configurable(metadata(docs::enum_tag_description = "The protocol version."))] +#[derive(Clone, Debug)] +#[serde(rename_all = "snake_case", tag = "version")] +pub enum ExecVersion { + /// Expect the command to fetch the configuration options itself. + V1, + + /// Configuration options to the command are to be curried upon each request. + V1_1 { + /// The name of the backend. This is `type` field in the backend request. + backend_type: String, + /// The configuration to pass to the secrets executable. This is the `config` field in the + /// backend request. Refer to the documentation of your `backend_type `to see which options + /// are required to be set. + backend_config: Value, + }, +} + +impl ExecVersion { + fn new_query(&self, secrets: HashSet) -> ExecQuery { + match &self { + ExecVersion::V1 => ExecQuery { + version: "1.0".to_string(), + secrets, + r#type: None, + config: None, + }, + ExecVersion::V1_1 { + backend_type, + backend_config, + .. + } => ExecQuery { + version: "1.1".to_string(), + secrets, + r#type: Some(backend_type.clone()), + config: Some(backend_config.clone()), + }, + } + } +} + +impl GenerateConfig for ExecVersion { + fn generate_config() -> toml::Value { + toml::Value::try_from(ExecVersion::V1).unwrap() + } +} + /// Configuration for the `exec` secrets backend. #[configurable_component(secrets("exec"))] #[derive(Clone, Debug)] @@ -22,6 +72,10 @@ pub struct ExecBackend { /// The timeout, in seconds, to wait for the command to complete. #[serde(default = "default_timeout_secs")] pub timeout: u64, + + /// Settings for the protocol between Vector and the secrets executable. + #[serde(default = "default_protocol_version")] + pub protocol: ExecVersion, } impl GenerateConfig for ExecBackend { @@ -29,6 +83,7 @@ impl GenerateConfig for ExecBackend { toml::Value::try_from(ExecBackend { command: vec![String::from("/path/to/script")], timeout: 5, + protocol: ExecVersion::V1, }) .unwrap() } @@ -38,17 +93,20 @@ const fn default_timeout_secs() -> u64 { 5 } -#[derive(Clone, Debug, Deserialize, Serialize)] +const fn default_protocol_version() -> ExecVersion { + ExecVersion::V1 +} + +#[derive(Clone, Debug, Serialize)] struct ExecQuery { + // Fields in all versions starting from v1 version: String, secrets: HashSet, -} - -fn new_query(secrets: HashSet) -> ExecQuery { - ExecQuery { - version: "1.0".to_string(), - secrets, - } + // Fields added in v1.1 + #[serde(skip_serializing_if = "Option::is_none")] + r#type: Option, + #[serde(skip_serializing_if = "Option::is_none")] + config: Option, } #[derive(Clone, Debug, Deserialize, Serialize)] @@ -66,7 +124,7 @@ impl SecretBackend for ExecBackend { let mut output = executor::block_on(async { query_backend( &self.command, - new_query(secret_keys.clone()), + self.protocol.new_query(secret_keys.clone()), self.timeout, signal_rx, ) @@ -76,18 +134,18 @@ impl SecretBackend for ExecBackend { for k in secret_keys.into_iter() { if let Some(secret) = output.get_mut(&k) { if let Some(e) = &secret.error { - return Err(format!("secret for key '{}' was not retrieved: {}", k, e).into()); + return Err(format!("secret for key '{k}' was not retrieved: {e}").into()); } if let Some(v) = secret.value.take() { if v.is_empty() { - return Err(format!("secret for key '{}' was empty", k).into()); + return Err(format!("secret for key '{k}' was empty").into()); } secrets.insert(k.to_string(), v); } else { - return Err(format!("secret for key '{}' was empty", k).into()); + return Err(format!("secret for key '{k}' was empty").into()); } } else { - return Err(format!("secret for key '{}' was not retrieved", k).into()); + return Err(format!("secret for key '{k}' was not retrieved").into()); } } Ok(secrets) @@ -117,12 +175,10 @@ async fn query_backend( let mut stderr_stream = child .stderr .map(|s| codec::FramedRead::new(s, codec::LinesCodec::new())) - .take() .ok_or("unable to acquire stderr")?; let mut stdout_stream = child .stdout .map(|s| codec::FramedRead::new(s, codec::BytesCodec::new())) - .take() .ok_or("unable to acquire stdout")?; let query = serde_json::to_vec(&query)?; @@ -148,7 +204,7 @@ async fn query_backend( match stdout { None => break, Some(Ok(b)) => output.extend(b), - Some(Err(e)) => return Err(format!("Error while reading from an exec backend stdout: {}.", e).into()), + Some(Err(e)) => return Err(format!("Error while reading from an exec backend stdout: {e}.").into()), } } _ = &mut timeout => { @@ -161,3 +217,79 @@ async fn query_backend( let response = serde_json::from_slice::>(&output)?; Ok(response) } + +#[cfg(test)] +mod tests { + use crate::{ + config::SecretBackend, + secrets::exec::{ExecBackend, ExecVersion}, + }; + use rstest::rstest; + use std::{ + collections::{HashMap, HashSet}, + path::PathBuf, + }; + use tokio::sync::broadcast; + use vrl::value; + + fn make_test_backend(protocol: ExecVersion) -> ExecBackend { + let command_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/behavior/secrets/mock_secrets_exec.py"); + ExecBackend { + command: ["python", command_path.to_str().unwrap()] + .map(String::from) + .to_vec(), + timeout: 5, + protocol, + } + } + + #[tokio::test(flavor = "multi_thread")] + #[rstest( + protocol, + case(ExecVersion::V1), + case(ExecVersion::V1_1 { + backend_type: "file.json".to_string(), + backend_config: value!({"file_path": "/abc.json"}), + }) + )] + async fn test_exec_backend(protocol: ExecVersion) { + let mut backend = make_test_backend(protocol); + let (_tx, mut rx) = broadcast::channel(1); + // These fake secrets are statically contained in mock_secrets_exec.py + let fake_secret_values: HashMap = [ + ("fake_secret_1", "123456"), + ("fake_secret_2", "123457"), + ("fake_secret_3", "123458"), + ("fake_secret_4", "123459"), + ("fake_secret_5", "123460"), + ] + .into_iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); + // Calling the mock_secrets_exec.py program with the expected secret keys should provide + // the values expected above in `fake_secret_values` + let fetched_keys = backend + .retrieve(fake_secret_values.keys().cloned().collect(), &mut rx) + .await + .unwrap(); + // Assert response is as expected + assert_eq!(fetched_keys.len(), 5); + for (fake_secret_key, fake_secret_value) in fake_secret_values { + assert_eq!(fetched_keys.get(&fake_secret_key), Some(&fake_secret_value)); + } + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_exec_backend_missing_secrets() { + let mut backend = make_test_backend(ExecVersion::V1); + let (_tx, mut rx) = broadcast::channel(1); + let query_secrets: HashSet = + ["fake_secret_900"].into_iter().map(String::from).collect(); + let fetched_keys = backend.retrieve(query_secrets.clone(), &mut rx).await; + assert_eq!( + format!("{}", fetched_keys.unwrap_err()), + "secret for key 'fake_secret_900' was not retrieved: backend does not provide secret key" + ); + } +} diff --git a/src/secrets/file.rs b/src/secrets/file.rs index 221a293622..c103385493 100644 --- a/src/secrets/file.rs +++ b/src/secrets/file.rs @@ -34,11 +34,11 @@ impl SecretBackend for FileBackend { for k in secret_keys.into_iter() { if let Some(secret) = output.get(&k) { if secret.is_empty() { - return Err(format!("secret for key '{}' was empty", k).into()); + return Err(format!("secret for key '{k}' was empty").into()); } secrets.insert(k, secret.to_string()); } else { - return Err(format!("secret for key '{}' was not retrieved", k).into()); + return Err(format!("secret for key '{k}' was not retrieved").into()); } } Ok(secrets) diff --git a/src/secrets/mod.rs b/src/secrets/mod.rs index febe708f25..b8e9c9129a 100644 --- a/src/secrets/mod.rs +++ b/src/secrets/mod.rs @@ -2,8 +2,9 @@ use std::collections::{HashMap, HashSet}; use enum_dispatch::enum_dispatch; -use vector_lib::configurable::{configurable_component, NamedComponent}; +use vector_lib::configurable::configurable_component; +use crate::config::GenerateConfig; use crate::{config::SecretBackend, signal}; #[cfg(feature = "secrets-aws-secrets-manager")] @@ -13,12 +14,50 @@ mod exec; mod file; mod test; -/// Configurable secret backends in Vector. +/// Configuration options to retrieve secrets from external backend in order to avoid storing secrets in plaintext +/// in Vector config. Multiple backends can be configured. Use `SECRET[.]` to tell Vector to retrieve the secret. This placeholder is replaced by the secret +/// retrieved from the relevant backend. +/// +/// When `type` is `exec`, the provided command will be run and provided a list of +/// secrets to fetch, determined from the configuration file, on stdin as JSON in the format: +/// +/// ```json +/// {"version": "1.0", "secrets": ["secret1", "secret2"]} +/// ``` +/// +/// The executable is expected to respond with the values of these secrets on stdout, also as JSON, in the format: +/// +/// ```json +/// { +/// "secret1": {"value": "secret_value", "error": null}, +/// "secret2": {"value": null, "error": "could not fetch the secret"} +/// } +/// ``` +/// If an `error` is returned for any secrets, or if the command exits with a non-zero status code, +/// Vector will log the errors and exit. +/// +/// Otherwise, the secret must be a JSON text string with key/value pairs. For example: +/// ```json +/// { +/// "username": "test", +/// "password": "example-password" +/// } +/// ``` +/// +/// If an error occurred while reading the file or retrieving the secrets, Vector logs the error and exits. +/// +/// Secrets are loaded when Vector starts or if Vector receives a `SIGHUP` signal triggering its +/// configuration reload process. #[allow(clippy::large_enum_variant)] -#[configurable_component] +#[configurable_component(global_option("secret"))] #[derive(Clone, Debug)] #[enum_dispatch(SecretBackend)] #[serde(tag = "type", rename_all = "snake_case")] +#[configurable(metadata( + docs::enum_tag_description = "secret type", + docs::common = false, + docs::required = false, +))] pub enum SecretBackends { /// File. File(file::FileBackend), @@ -38,16 +77,11 @@ pub enum SecretBackends { Test(test::TestBackend), } -// TODO: Use `enum_dispatch` here. -impl NamedComponent for SecretBackends { - fn get_component_name(&self) -> &'static str { - match self { - Self::File(config) => config.get_component_name(), - Self::Directory(config) => config.get_component_name(), - Self::Exec(config) => config.get_component_name(), - #[cfg(feature = "secrets-aws-secrets-manager")] - Self::AwsSecretsManager(config) => config.get_component_name(), - Self::Test(config) => config.get_component_name(), - } +impl GenerateConfig for SecretBackends { + fn generate_config() -> toml::Value { + toml::Value::try_from(Self::File(file::FileBackend { + path: "path/to/file".into(), + })) + .unwrap() } } diff --git a/src/serde.rs b/src/serde.rs index 6a0fd579cd..29b28f732f 100644 --- a/src/serde.rs +++ b/src/serde.rs @@ -2,8 +2,8 @@ use indexmap::map::IndexMap; use serde::{Deserialize, Serialize}; use vector_lib::codecs::{ - decoding::{DeserializerConfig, FramingConfig}, BytesDecoderConfig, BytesDeserializerConfig, + decoding::{DeserializerConfig, FramingConfig}, }; use vector_lib::configurable::configurable_component; pub use vector_lib::serde::{bool_or_struct, is_default}; @@ -90,7 +90,7 @@ impl Fields { FieldsOrValue::Value(v) => Box::new(std::iter::once((k, v))), FieldsOrValue::Fields(f) => Box::new( f.all_fields() - .map(move |(nested_k, v)| (format!("{}.{}", k, nested_k), v)), + .map(move |(nested_k, v)| (format!("{k}.{nested_k}"), v)), ), } }) diff --git a/src/service.rs b/src/service.rs index 7059e424b7..08283dba9f 100644 --- a/src/service.rs +++ b/src/service.rs @@ -200,7 +200,9 @@ pub fn cmd(opts: &Opts) -> exitcode::ExitCode { } }, None => { - error!("You must specify a sub command. Valid sub commands are [start, stop, restart, install, uninstall]."); + error!( + "You must specify a sub command. Valid sub commands are [start, stop, restart, install, uninstall]." + ); exitcode::USAGE } } diff --git a/src/signal.rs b/src/signal.rs index 91a1e3515f..6745448cae 100644 --- a/src/signal.rs +++ b/src/signal.rs @@ -1,6 +1,7 @@ #![allow(missing_docs)] use snafu::Snafu; +use std::collections::HashSet; use tokio::{runtime::Runtime, sync::broadcast}; use tokio_stream::{Stream, StreamExt}; @@ -14,17 +15,38 @@ pub type SignalRx = broadcast::Receiver; /// Control messages used by Vector to drive topology and shutdown events. #[allow(clippy::large_enum_variant)] // discovered during Rust upgrade to 1.57; just allowing for now since we did previously pub enum SignalTo { + /// Signal to reload given components. + ReloadComponents(HashSet), /// Signal to reload config from a string. ReloadFromConfigBuilder(ConfigBuilder), /// Signal to reload config from the filesystem. ReloadFromDisk, + /// Signal to reload all enrichment tables. + ReloadEnrichmentTables, /// Signal to shutdown process. Shutdown(Option), /// Shutdown process immediately. Quit, } -#[derive(Clone, Debug, Snafu)] +impl PartialEq for SignalTo { + fn eq(&self, other: &Self) -> bool { + use SignalTo::*; + + match (self, other) { + (ReloadComponents(a), ReloadComponents(b)) => a == b, + // TODO: This will require a lot of plumbing but ultimately we can derive equality for config builders. + (ReloadFromConfigBuilder(_), ReloadFromConfigBuilder(_)) => true, + (ReloadFromDisk, ReloadFromDisk) => true, + (ReloadEnrichmentTables, ReloadEnrichmentTables) => true, + (Shutdown(a), Shutdown(b)) => a == b, + (Quit, Quit) => true, + _ => false, + } + } +} + +#[derive(Clone, Debug, Snafu, PartialEq, Eq)] pub enum ShutdownError { // For future work: It would be nice if we could keep the actual errors in here, but // `crate::Error` doesn't implement `Clone`, and adding `DynClone` for errors is tricky. @@ -50,7 +72,15 @@ impl SignalPair { /// Create a new signal handler pair, and set them up to receive OS signals. pub fn new(runtime: &Runtime) -> Self { let (handler, receiver) = SignalHandler::new(); + + #[cfg(unix)] let signals = os_signals(runtime); + + // If we passed `runtime` here, we would get the following: + // error[E0521]: borrowed data escapes outside of associated function + #[cfg(windows)] + let signals = os_signals(); + handler.forever(runtime, signals); Self { handler, receiver } } @@ -154,8 +184,8 @@ impl SignalHandler { /// Signals from OS/user. #[cfg(unix)] -fn os_signals(runtime: &Runtime) -> impl Stream { - use tokio::signal::unix::{signal, SignalKind}; +fn os_signals(runtime: &Runtime) -> impl Stream + use<> { + use tokio::signal::unix::{SignalKind, signal}; // The `signal` function must be run within the context of a Tokio runtime. runtime.block_on(async { @@ -193,7 +223,7 @@ fn os_signals(runtime: &Runtime) -> impl Stream { /// Signals from OS/user. #[cfg(windows)] -fn os_signals(_: &Runtime) -> impl Stream { +fn os_signals() -> impl Stream { use futures::future::FutureExt; async_stream::stream! { diff --git a/src/sink_ext.rs b/src/sink_ext.rs index f1d1ad3ca8..8267712d5e 100644 --- a/src/sink_ext.rs +++ b/src/sink_ext.rs @@ -28,10 +28,10 @@ use std::{ future::Future, pin::Pin, - task::{ready, Context, Poll}, + task::{Context, Poll, ready}, }; -use futures::{stream::Peekable, Sink, SinkExt, Stream, StreamExt}; +use futures::{Sink, SinkExt, Stream, StreamExt, stream::Peekable}; impl VecSinkExt for T where T: Sink {} diff --git a/src/sinks/amqp/channel.rs b/src/sinks/amqp/channel.rs new file mode 100644 index 0000000000..6bb8f6f47c --- /dev/null +++ b/src/sinks/amqp/channel.rs @@ -0,0 +1,87 @@ +use super::config::AmqpSinkConfig; +use super::service::AmqpError; +use crate::amqp::AmqpConfig; +use deadpool::managed::Pool; +use lapin::options::ConfirmSelectOptions; + +pub type AmqpSinkChannels = Pool; + +pub(super) fn new_channel_pool(config: &AmqpSinkConfig) -> crate::Result { + let max_channels = config.max_channels.try_into().map_err(|_| { + Box::new(AmqpError::PoolError { + error: "max_channels must fit into usize".into(), + }) + })?; + if max_channels == 0 { + return Err(Box::new(AmqpError::PoolError { + error: "max_channels must be positive".into(), + })); + } + let channel_manager = AmqpSinkChannelManager::new(&config.connection); + let channels = Pool::builder(channel_manager) + .max_size(max_channels) + .runtime(deadpool::Runtime::Tokio1) + .build()?; + debug!("AMQP channel pool created with max size: {}", max_channels); + Ok(channels) +} + +/// A channel pool manager for the AMQP sink. +/// This manager is responsible for creating and recycling AMQP channels. +/// It uses the `deadpool` crate to manage the channels. +pub(crate) struct AmqpSinkChannelManager { + config: AmqpConfig, +} + +impl deadpool::managed::Manager for AmqpSinkChannelManager { + type Type = lapin::Channel; + type Error = AmqpError; + + async fn create(&self) -> Result { + let channel = Self::new_channel(&self.config).await?; + info!( + message = "Created a new channel to the AMQP broker.", + id = channel.id(), + internal_log_rate_limit = true, + ); + Ok(channel) + } + + async fn recycle( + &self, + channel: &mut Self::Type, + _: &deadpool::managed::Metrics, + ) -> deadpool::managed::RecycleResult { + let state = channel.status().state(); + if state == lapin::ChannelState::Connected { + Ok(()) + } else { + Err((AmqpError::ChannelClosed { state }).into()) + } + } +} + +impl AmqpSinkChannelManager { + /// Creates a new channel pool manager for the AMQP sink. + pub fn new(config: &AmqpConfig) -> Self { + Self { + config: config.clone(), + } + } + + /// Creates a new AMQP channel using the configuration of this sink. + async fn new_channel(config: &AmqpConfig) -> Result { + let (_, channel) = config + .connect() + .await + .map_err(|e| AmqpError::ConnectFailed { error: e })?; + + // Enable confirmations on the channel. + channel + .confirm_select(ConfirmSelectOptions::default()) + .await + .map_err(|e| AmqpError::ConnectFailed { error: Box::new(e) })?; + + Ok(channel) + } +} diff --git a/src/sinks/amqp/config.rs b/src/sinks/amqp/config.rs index 1c5794b002..5b747d77b4 100644 --- a/src/sinks/amqp/config.rs +++ b/src/sinks/amqp/config.rs @@ -1,8 +1,11 @@ //! Configuration functionality for the `AMQP` sink. +use super::channel::AmqpSinkChannels; use crate::{amqp::AmqpConfig, sinks::prelude::*}; -use lapin::{types::ShortString, BasicProperties}; -use std::sync::Arc; -use vector_lib::codecs::TextSerializerConfig; +use lapin::{BasicProperties, types::ShortString}; +use vector_lib::{ + codecs::TextSerializerConfig, + internal_event::{error_stage, error_type}, +}; use super::sink::AmqpSink; @@ -17,12 +20,15 @@ pub struct AmqpPropertiesConfig { /// Content-Encoding for the AMQP messages. pub(crate) content_encoding: Option, - /// Expiration for AMQP messages (in milliseconds) + /// Expiration for AMQP messages (in milliseconds). pub(crate) expiration_ms: Option, + + /// Priority for AMQP messages. It can be templated to an integer between 0 and 255 inclusive. + pub(crate) priority: Option, } impl AmqpPropertiesConfig { - pub(super) fn build(&self) -> BasicProperties { + pub(super) fn build(&self, event: &Event) -> Option { let mut prop = BasicProperties::default(); if let Some(content_type) = &self.content_type { prop = prop.with_content_type(ShortString::from(content_type.clone())); @@ -33,7 +39,23 @@ impl AmqpPropertiesConfig { if let Some(expiration_ms) = &self.expiration_ms { prop = prop.with_expiration(ShortString::from(expiration_ms.to_string())); } - prop + if let Some(priority_template) = &self.priority { + let priority = priority_template.render(event).unwrap_or_else(|error| { + warn!( + message = "Failed to render numeric template for \"properties.priority\".", + error = %error, + error_type = error_type::TEMPLATE_FAILED, + stage = error_stage::PROCESSING, + internal_log_rate_limit = true, + ); + Default::default() + }); + + // Clamp the value to the range of 0-255, as AMQP priority is a u8. + let priority = priority.clamp(0, u8::MAX.into()) as u8; + prop = prop.with_priority(priority); + } + Some(prop) } } @@ -68,6 +90,14 @@ pub struct AmqpSinkConfig { skip_serializing_if = "crate::serde::is_default" )] pub(crate) acknowledgements: AcknowledgementsConfig, + + /// Maximum number of AMQP channels to keep active (channels are created as needed). + #[serde(default = "default_max_channels")] + pub(crate) max_channels: u32, +} + +const fn default_max_channels() -> u32 { + 4 } impl Default for AmqpSinkConfig { @@ -79,6 +109,7 @@ impl Default for AmqpSinkConfig { encoding: TextSerializerConfig::default().into(), connection: AmqpConfig::default(), acknowledgements: AcknowledgementsConfig::default(), + max_channels: default_max_channels(), } } } @@ -89,7 +120,8 @@ impl GenerateConfig for AmqpSinkConfig { r#"connection_string = "amqp://localhost:5672/%2f" routing_key = "user_id" exchange = "test" - encoding.codec = "json""#, + encoding.codec = "json" + max_channels = 4"#, ) .unwrap() } @@ -100,7 +132,7 @@ impl GenerateConfig for AmqpSinkConfig { impl SinkConfig for AmqpSinkConfig { async fn build(&self, _cx: SinkContext) -> crate::Result<(VectorSink, Healthcheck)> { let sink = AmqpSink::new(self.clone()).await?; - let hc = healthcheck(Arc::clone(&sink.channel)).boxed(); + let hc = healthcheck(sink.channels.clone()).boxed(); Ok((VectorSink::from_event_streamsink(sink), hc)) } @@ -113,9 +145,11 @@ impl SinkConfig for AmqpSinkConfig { } } -pub(super) async fn healthcheck(channel: Arc) -> crate::Result<()> { +pub(super) async fn healthcheck(channels: AmqpSinkChannels) -> crate::Result<()> { trace!("Healthcheck started."); + let channel = channels.get().await?; + if !channel.status().connected() { return Err(Box::new(std::io::Error::new( std::io::ErrorKind::BrokenPipe, @@ -127,7 +161,126 @@ pub(super) async fn healthcheck(channel: Arc) -> crate::Result<( Ok(()) } -#[test] -pub fn generate_config() { - crate::test_util::test_generate_config::(); +#[cfg(test)] +mod tests { + use super::*; + use crate::config::format::{Format, deserialize}; + + #[test] + pub fn generate_config() { + crate::test_util::test_generate_config::(); + } + + fn assert_config_priority_eq(config: AmqpSinkConfig, event: &LogEvent, priority: u8) { + assert_eq!( + config + .properties + .unwrap() + .priority + .unwrap() + .render(event) + .unwrap(), + priority as u64 + ); + } + + #[test] + pub fn parse_config_priority_static() { + for (format, config) in [ + ( + Format::Yaml, + r#" + exchange: "test" + routing_key: "user_id" + encoding: + codec: "json" + connection_string: "amqp://user:password@127.0.0.1:5672/" + properties: + priority: 1 + "#, + ), + ( + Format::Toml, + r#" + exchange = "test" + routing_key = "user_id" + encoding.codec = "json" + connection_string = "amqp://user:password@127.0.0.1:5672/" + properties = { priority = 1 } + "#, + ), + ( + Format::Json, + r#" + { + "exchange": "test", + "routing_key": "user_id", + "encoding": { + "codec": "json" + }, + "connection_string": "amqp://user:password@127.0.0.1:5672/", + "properties": { + "priority": 1 + } + } + "#, + ), + ] { + let config: AmqpSinkConfig = deserialize(config, format).unwrap(); + let event = LogEvent::from_str_legacy("message"); + assert_config_priority_eq(config, &event, 1); + } + } + + #[test] + pub fn parse_config_priority_templated() { + for (format, config) in [ + ( + Format::Yaml, + r#" + exchange: "test" + routing_key: "user_id" + encoding: + codec: "json" + connection_string: "amqp://user:password@127.0.0.1:5672/" + properties: + priority: "{{ .priority }}" + "#, + ), + ( + Format::Toml, + r#" + exchange = "test" + routing_key = "user_id" + encoding.codec = "json" + connection_string = "amqp://user:password@127.0.0.1:5672/" + properties = { priority = "{{ .priority }}" } + "#, + ), + ( + Format::Json, + r#" + { + "exchange": "test", + "routing_key": "user_id", + "encoding": { + "codec": "json" + }, + "connection_string": "amqp://user:password@127.0.0.1:5672/", + "properties": { + "priority": "{{ .priority }}" + } + } + "#, + ), + ] { + let config: AmqpSinkConfig = deserialize(config, format).unwrap(); + let event = { + let mut event = LogEvent::from_str_legacy("message"); + event.insert("priority", 2); + event + }; + assert_config_priority_eq(config, &event, 2); + } + } } diff --git a/src/sinks/amqp/encoder.rs b/src/sinks/amqp/encoder.rs index 7529fa12b6..3655ab3d93 100644 --- a/src/sinks/amqp/encoder.rs +++ b/src/sinks/amqp/encoder.rs @@ -26,7 +26,7 @@ impl encoding::Encoder for AmqpEncoder { let mut encoder = self.encoder.clone(); encoder .encode(input, &mut body) - .map_err(|_| io::Error::new(io::ErrorKind::Other, "unable to encode"))?; + .map_err(|_| io::Error::other("unable to encode"))?; let body = body.freeze(); write_all(writer, 1, body.as_ref())?; diff --git a/src/sinks/amqp/integration_tests.rs b/src/sinks/amqp/integration_tests.rs index b5f1bdd320..64acb267ac 100644 --- a/src/sinks/amqp/integration_tests.rs +++ b/src/sinks/amqp/integration_tests.rs @@ -1,18 +1,20 @@ use super::*; use crate::{ + SourceSender, amqp::await_connection, config::{SinkConfig, SinkContext}, shutdown::ShutdownSignal, - template::Template, + sinks::amqp::channel::new_channel_pool, + template::{Template, UnsignedIntTemplate}, test_util::{ - components::{run_and_assert_sink_compliance, SINK_TAGS}, + components::{SINK_TAGS, run_and_assert_sink_compliance}, random_lines_with_stream, random_string, }, - SourceSender, }; +use config::AmqpPropertiesConfig; use futures::StreamExt; -use std::{collections::HashSet, sync::Arc, time::Duration}; -use vector_lib::config::LogNamespace; +use std::{collections::HashSet, time::Duration}; +use vector_lib::{config::LogNamespace, event::LogEvent}; pub fn make_config() -> AmqpSinkConfig { let mut config = AmqpSinkConfig { @@ -21,9 +23,9 @@ pub fn make_config() -> AmqpSinkConfig { }; let user = std::env::var("AMQP_USER").unwrap_or_else(|_| "guest".to_string()); let pass = std::env::var("AMQP_PASSWORD").unwrap_or_else(|_| "guest".to_string()); + let host = std::env::var("AMQP_HOST").unwrap_or_else(|_| "rabbitmq".to_string()); let vhost = std::env::var("AMQP_VHOST").unwrap_or_else(|_| "%2f".to_string()); - config.connection.connection_string = - format!("amqp://{}:{}@rabbitmq:5672/{}", user, pass, vhost); + config.connection.connection_string = format!("amqp://{user}:{pass}@{host}:5672/{vhost}"); config } @@ -35,8 +37,8 @@ async fn healthcheck() { let mut config = make_config(); config.exchange = Template::try_from(exchange.as_str()).unwrap(); await_connection(&config.connection).await; - let (_conn, channel) = config.connection.connect().await.unwrap(); - super::config::healthcheck(Arc::new(channel)).await.unwrap(); + let channels = new_channel_pool(&config).unwrap(); + super::config::healthcheck(channels).await.unwrap(); } #[tokio::test] @@ -124,6 +126,10 @@ async fn amqp_happy_path() { { let msg = try_msg.unwrap(); let s = String::from_utf8_lossy(msg.data.as_slice()).into_owned(); + + let msg_priority = *msg.properties.priority(); + assert_eq!(msg_priority, None); + out.push(s); } else { failures += 1; @@ -218,3 +224,124 @@ async fn amqp_round_trip() { assert_eq!(output.len(), nb_events_published); } + +async fn amqp_priority_with_template( + template: &str, + event_field_priority: Option, + expected_priority: Option, +) { + let mut config = make_config(); + let exchange = format!("test-{}-exchange", random_string(10)); + config.exchange = Template::try_from(exchange.as_str()).unwrap(); + config.properties = Some(AmqpPropertiesConfig { + priority: Some(UnsignedIntTemplate::try_from(template).unwrap()), + ..Default::default() + }); + + await_connection(&config.connection).await; + let (_conn, channel) = config.connection.connect().await.unwrap(); + let exchange_opts = lapin::options::ExchangeDeclareOptions { + auto_delete: true, + ..Default::default() + }; + channel + .exchange_declare( + &exchange, + lapin::ExchangeKind::Fanout, + exchange_opts, + lapin::types::FieldTable::default(), + ) + .await + .unwrap(); + + let cx = SinkContext::default(); + let (sink, healthcheck) = config.build(cx).await.unwrap(); + healthcheck.await.expect("Health check failed"); + + // prepare consumer + let queue = format!("test-{}-queue", random_string(10)); + let queue_opts = lapin::options::QueueDeclareOptions { + auto_delete: true, + ..Default::default() + }; + let queue_args = { + let mut args = lapin::types::FieldTable::default(); + args.insert( + lapin::types::ShortString::from("x-max-priority"), + lapin::types::AMQPValue::ShortInt(10), // Maximum priority value + ); + args + }; + channel + .queue_declare(&queue, queue_opts, queue_args) + .await + .unwrap(); + + channel + .queue_bind( + &queue, + &exchange, + "", + lapin::options::QueueBindOptions::default(), + lapin::types::FieldTable::default(), + ) + .await + .unwrap(); + + let consumer = format!("test-{}-consumer", random_string(10)); + let mut consumer = channel + .basic_consume( + &queue, + &consumer, + lapin::options::BasicConsumeOptions::default(), + lapin::types::FieldTable::default(), + ) + .await + .unwrap(); + + // Send a single event with a priority defined in the event + let input = random_string(100); + let event = { + let mut event = LogEvent::from_str_legacy(&input); + if let Some(priority) = event_field_priority { + event.insert("priority", priority); + } + event + }; + + let events = futures::stream::iter(vec![event]); + run_and_assert_sink_compliance(sink, events, &SINK_TAGS).await; + + if let Ok(Some(try_msg)) = tokio::time::timeout(Duration::from_secs(10), consumer.next()).await + { + let msg = try_msg.unwrap(); + let msg_priority = *msg.properties.priority(); + let output = String::from_utf8_lossy(msg.data.as_slice()).into_owned(); + + assert_eq!(msg_priority, expected_priority); + assert_eq!(output, input); + } else { + panic!("Did not consume message in time."); + } +} + +#[tokio::test] +async fn amqp_priority_template_variable() { + crate::test_util::trace_init(); + + amqp_priority_with_template("{{ priority }}", Some(5), Some(5)).await; +} + +#[tokio::test] +async fn amqp_priority_template_constant() { + crate::test_util::trace_init(); + + amqp_priority_with_template("5", None, Some(5)).await; +} + +#[tokio::test] +async fn amqp_priority_template_out_of_bounds() { + crate::test_util::trace_init(); + + amqp_priority_with_template("100000", None, Some(u8::MAX)).await; +} diff --git a/src/sinks/amqp/mod.rs b/src/sinks/amqp/mod.rs index 749f892f1c..6b478c4adc 100644 --- a/src/sinks/amqp/mod.rs +++ b/src/sinks/amqp/mod.rs @@ -1,5 +1,6 @@ //! `AMQP` sink. //! Handles version AMQP 0.9.1 which is used by RabbitMQ. +mod channel; mod config; mod encoder; mod request_builder; @@ -15,7 +16,5 @@ use snafu::Snafu; #[derive(Debug, Snafu)] enum BuildError { #[snafu(display("creating amqp producer failed: {}", source))] - AmqpCreateFailed { - source: Box, - }, + AmqpCreateFailed { source: vector_common::Error }, } diff --git a/src/sinks/amqp/service.rs b/src/sinks/amqp/service.rs index 44c475f208..3875907d4c 100644 --- a/src/sinks/amqp/service.rs +++ b/src/sinks/amqp/service.rs @@ -3,12 +3,11 @@ use crate::sinks::prelude::*; use bytes::Bytes; use futures::future::BoxFuture; -use lapin::{options::BasicPublishOptions, BasicProperties}; +use lapin::{BasicProperties, options::BasicPublishOptions}; use snafu::Snafu; -use std::{ - sync::Arc, - task::{Context, Poll}, -}; +use std::task::{Context, Poll}; + +use super::channel::AmqpSinkChannels; /// The request contains the data to send to `AMQP` together /// with the information need to route the message. @@ -79,11 +78,11 @@ impl DriverResponse for AmqpResponse { /// The tower service that handles the actual sending of data to `AMQP`. pub(super) struct AmqpService { - pub(super) channel: Arc, + pub(super) channels: AmqpSinkChannels, } #[derive(Debug, Snafu)] -pub(super) enum AmqpError { +pub enum AmqpError { #[snafu(display("Failed retrieving Acknowledgement: {}", error))] AcknowledgementFailed { error: lapin::Error }, @@ -92,6 +91,15 @@ pub(super) enum AmqpError { #[snafu(display("Received Negative Acknowledgement from AMQP broker."))] Nack, + + #[snafu(display("Failed to open AMQP channel: {}", error))] + ConnectFailed { error: vector_common::Error }, + + #[snafu(display("Channel is not writeable: {:?}", state))] + ChannelClosed { state: lapin::ChannelState }, + + #[snafu(display("Channel pool error: {}", error))] + PoolError { error: vector_common::Error }, } impl Service for AmqpService { @@ -106,9 +114,13 @@ impl Service for AmqpService { } fn call(&mut self, req: AmqpRequest) -> Self::Future { - let channel = Arc::clone(&self.channel); + let channel = self.channels.clone(); Box::pin(async move { + let channel = channel.get().await.map_err(|error| AmqpError::PoolError { + error: Box::new(error), + })?; + let byte_size = req.body.len(); let fut = channel .basic_publish( diff --git a/src/sinks/amqp/sink.rs b/src/sinks/amqp/sink.rs index 922065a58a..17ebbcc68b 100644 --- a/src/sinks/amqp/sink.rs +++ b/src/sinks/amqp/sink.rs @@ -1,16 +1,16 @@ //! The sink for the `AMQP` sink that wires together the main stream that takes the //! event and sends it to `AMQP`. use crate::sinks::prelude::*; -use lapin::{options::ConfirmSelectOptions, BasicProperties}; +use lapin::BasicProperties; use serde::Serialize; -use std::sync::Arc; +use super::channel::AmqpSinkChannels; use super::{ + BuildError, config::{AmqpPropertiesConfig, AmqpSinkConfig}, encoder::AmqpEncoder, request_builder::AmqpRequestBuilder, service::AmqpService, - BuildError, }; /// Stores the event together with the rendered exchange and routing_key values. @@ -27,7 +27,7 @@ pub(super) struct AmqpEvent { } pub(super) struct AmqpSink { - pub(super) channel: Arc, + pub(super) channels: AmqpSinkChannels, exchange: Template, routing_key: Option