From 702874f13d6390cd8598bdf0265c194557093cdf Mon Sep 17 00:00:00 2001 From: Lukasz Drozdz Date: Wed, 5 Mar 2025 14:28:42 +0100 Subject: [PATCH 1/3] experiment CI --- .github/workflows/exaforce-publish.yaml | 34 ++-- scripts/environment/bootstrap-ubuntu-24.04.sh | 179 ++++++++++++++++++ 2 files changed, 196 insertions(+), 17 deletions(-) create mode 100755 scripts/environment/bootstrap-ubuntu-24.04.sh diff --git a/.github/workflows/exaforce-publish.yaml b/.github/workflows/exaforce-publish.yaml index 45600f338e8e4..f0f3bd88830b2 100644 --- a/.github/workflows/exaforce-publish.yaml +++ b/.github/workflows/exaforce-publish.yaml @@ -33,7 +33,7 @@ env: jobs: generate-publish-metadata: name: Generate Publish-related Metadata - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 timeout-minutes: 5 outputs: vector_version: ${{ steps.generate-publish-metadata.outputs.vector_version }} @@ -47,7 +47,7 @@ jobs: build-x86_64-unknown-linux-gnu-packages: name: Build Vector for x86_64-unknown-linux-gnu (.tar.gz, DEB, RPM) - runs-on: ubuntu-20.04-xl + runs-on: ubuntu-24.04-xl needs: generate-publish-metadata timeout-minutes: 90 env: @@ -57,13 +57,13 @@ jobs: - name: Checkout Vector uses: actions/checkout@v4 - name: Bootstrap runner environment (Ubuntu-specific) - run: sudo -E bash scripts/environment/bootstrap-ubuntu-20.04.sh - # Required for some reason for 1.83 - - name: Install Rust toolchain explicitly - run: | - rustup set profile minimal - rustup toolchain install $(grep 'channel' rust-toolchain.toml | cut -d '"' -f 2) --profile minimal - rustup default $(grep 'channel' rust-toolchain.toml | cut -d '"' -f 2) + run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh + # # Required for some reason for 1.83 + # - name: Install Rust toolchain explicitly + # run: | + # rustup set profile minimal + # rustup toolchain install $(grep 'channel' rust-toolchain.toml | cut -d '"' -f 2) --profile minimal + # rustup default $(grep 'channel' rust-toolchain.toml | cut -d '"' -f 2) - name: Bootstrap runner environment (generic) run: bash scripts/environment/prepare.sh - name: Build Vector @@ -78,7 +78,7 @@ jobs: build-aarch64-unknown-linux-gnu-packages: name: Build Vector for aarch64-unknown-linux-gnu (.tar.gz) - runs-on: ubuntu-20.04-xl + runs-on: ubuntu-24.04-xl timeout-minutes: 180 needs: generate-publish-metadata env: @@ -89,12 +89,12 @@ jobs: uses: actions/checkout@v4 - name: Bootstrap runner environment (Ubuntu-specific) run: sudo -E bash scripts/environment/bootstrap-ubuntu-20.04.sh - # Required for some reason for 1.83 - - name: Install Rust toolchain explicitly - run: | - rustup set profile minimal - rustup toolchain install $(grep 'channel' rust-toolchain.toml | cut -d '"' -f 2) --profile minimal - rustup default $(grep 'channel' rust-toolchain.toml | cut -d '"' -f 2) + # # Required for some reason for 1.83 + # - name: Install Rust toolchain explicitly + # run: | + # rustup set profile minimal + # rustup toolchain install $(grep 'channel' rust-toolchain.toml | cut -d '"' -f 2) --profile minimal + # rustup default $(grep 'channel' rust-toolchain.toml | cut -d '"' -f 2) - name: Bootstrap runner environment (generic) run: bash scripts/environment/prepare.sh - name: Build Vector @@ -155,7 +155,7 @@ jobs: publish-docker: name: Publish to Docker - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 timeout-minutes: 15 needs: - generate-publish-metadata diff --git a/scripts/environment/bootstrap-ubuntu-24.04.sh b/scripts/environment/bootstrap-ubuntu-24.04.sh new file mode 100755 index 0000000000000..ca481e6219228 --- /dev/null +++ b/scripts/environment/bootstrap-ubuntu-24.04.sh @@ -0,0 +1,179 @@ +#! /usr/bin/env bash +# Refer to https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2404-Readme.md +# for all runner information such as OS version and installed software. + +set -e -o verbose + +if [ -n "$RUSTFLAGS" ] +then + # shellcheck disable=SC2016 + echo '$RUSTFLAGS MUST NOT be set in CI configs as it overrides settings in `.cargo/config.toml`.' + exit 1 +fi + +export DEBIAN_FRONTEND=noninteractive +export ACCEPT_EULA=Y + +echo 'Acquire::Retries "5";' > /etc/apt/apt.conf.d/80-retries + +apt-get update --yes + +apt-get install --yes \ + software-properties-common \ + apt-utils \ + apt-transport-https + +# Deps +apt-get install --yes --no-install-recommends \ + build-essential \ + ca-certificates \ + cmake \ + cmark-gfm \ + curl \ + gawk \ + gnupg2 \ + gnupg-agent \ + gnuplot \ + jq \ + libclang-dev \ + libsasl2-dev \ + libssl-dev \ + llvm \ + locales \ + pkg-config \ + rename \ + rpm \ + ruby-bundler \ + shellcheck \ + sudo \ + unzip \ + wget + +# Cue +TEMP=$(mktemp -d) +curl \ + -L https://github.com/cue-lang/cue/releases/download/v0.10.0/cue_v0.10.0_linux_amd64.tar.gz \ + -o "${TEMP}/cue_v0.10.0_linux_amd64.tar.gz" +tar \ + -xvf "${TEMP}/cue_v0.10.0_linux_amd64.tar.gz" \ + -C "${TEMP}" +cp "${TEMP}/cue" /usr/bin/cue +rm -rf "$TEMP" + +# Grease +# Grease is used for the `make release-github` task. +TEMP=$(mktemp -d) +curl \ + -L https://github.com/vectordotdev/grease/releases/download/v1.0.1/grease-1.0.1-linux-amd64.tar.gz \ + -o "${TEMP}/grease-1.0.1-linux-amd64.tar.gz" +tar \ + -xvf "${TEMP}/grease-1.0.1-linux-amd64.tar.gz" \ + -C "${TEMP}" +cp "${TEMP}/grease/bin/grease" /usr/bin/grease +rm -rf "$TEMP" + +# Locales +locale-gen en_US.UTF-8 +dpkg-reconfigure locales + +if ! command -v rustup ; then + # https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2404-Readme.md#rust-tools + curl https://sh.rustup.rs -sSf | sh -s -- -y --profile minimal +fi + +# Rust/Cargo should already be installed on both GH Actions-provided Ubuntu 24.04 images _and_ +# by our own Ubuntu 24.04 images, so this is really just make sure the path is configured. +if [ -n "${CI-}" ] ; then + echo "${HOME}/.cargo/bin" >> "${GITHUB_PATH}" + # we often run into OOM issues in CI due to the low memory vs. CPU ratio on c5 instances + echo "CARGO_BUILD_JOBS=$(($(nproc) /2))" >> "${GITHUB_ENV}" +else + echo "export PATH=\"$HOME/.cargo/bin:\$PATH\"" >> "${HOME}/.bash_profile" +fi + +# Docker. +if ! [ -x "$(command -v docker)" ]; then + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - + add-apt-repository \ + "deb [arch=$(dpkg --print-architecture)] https://download.docker.com/linux/ubuntu \ + xenial \ + stable" + # Install those new things + apt-get update --yes + apt-get install --yes docker-ce docker-ce-cli containerd.io + + # ubuntu user doesn't exist in scripts/environment/Dockerfile which runs this + usermod --append --groups docker ubuntu || true +fi + +bash scripts/environment/install-protoc.sh + +# Node.js, npm and yarn. +# Note: the current LTS for the Node.js toolchain is 18.x +if ! [ -x "$(command -v node)" ]; then + curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | apt-key add - + add-apt-repository \ + "deb [arch=$(dpkg --print-architecture)] https://deb.nodesource.com/node_18.x \ + nodistro \ + main" + # Install those new things + apt-get update --yes + apt-get install --yes nodejs + + # enable corepack (enables the yarn and pnpm package managers) + # ref: https://nodejs.org/docs/latest-v18.x/api/corepack.html + corepack enable +fi + +# Apt cleanup +apt-get clean + +# Set up the default "deny all warnings" build flags +CARGO_OVERRIDE_DIR="${HOME}/.cargo" +CARGO_OVERRIDE_CONF="${CARGO_OVERRIDE_DIR}/config.toml" +cat <>"$CARGO_OVERRIDE_CONF" +[target.'cfg(linux)'] +rustflags = [ "-D", "warnings" ] +EOF + +# Install mold, because the system linker wastes a bunch of time. +# +# Notably, we don't install/configure it when we're going to do anything with `cross`, as `cross` takes the Cargo +# configuration from the host system and ships it over... which isn't good when we're overriding the `rustc-wrapper` +# and all of that. +if [ -z "${DISABLE_MOLD:-""}" ] ; then + # We explicitly put `mold-wrapper.so` right beside `mold` itself because it's hard-coded to look in the same directory + # first when trying to load the shared object, so we can dodge having to care about the "right" lib folder to put it in. + TEMP=$(mktemp -d) + MOLD_VERSION=1.2.1 + MOLD_TARGET=mold-${MOLD_VERSION}-$(uname -m)-linux + curl -fsSL "https://github.com/rui314/mold/releases/download/v${MOLD_VERSION}/${MOLD_TARGET}.tar.gz" \ + --output "$TEMP/${MOLD_TARGET}.tar.gz" + tar \ + -xvf "${TEMP}/${MOLD_TARGET}.tar.gz" \ + -C "${TEMP}" + cp "${TEMP}/${MOLD_TARGET}/bin/mold" /usr/bin/mold + cp "${TEMP}/${MOLD_TARGET}/lib/mold/mold-wrapper.so" /usr/bin/mold-wrapper.so + rm -rf "$TEMP" + + # Create our rustc wrapper script that we'll use to actually invoke `rustc` such that `mold` will wrap it and intercept + # anything linking calls to use `mold` instead of `ld`, etc. + CARGO_BIN_DIR="${CARGO_OVERRIDE_DIR}/bin" + mkdir -p "$CARGO_BIN_DIR" + + RUSTC_WRAPPER="${CARGO_BIN_DIR}/wrap-rustc" + cat <"$RUSTC_WRAPPER" +#!/bin/sh +exec mold -run "\$@" +EOF + chmod +x "$RUSTC_WRAPPER" + + # Now configure Cargo to use our rustc wrapper script. + cat <>"$CARGO_OVERRIDE_CONF" +[build] +rustc-wrapper = "$RUSTC_WRAPPER" +EOF +fi + +mkdir -p /var/lib/vector +chmod 777 /var/lib/vector From 61688c4e202ac250ecf376b7a51b0cd2e8020fbf Mon Sep 17 00:00:00 2001 From: Lukasz Drozdz Date: Wed, 5 Mar 2025 15:00:14 +0100 Subject: [PATCH 2/3] experiment CI --- .github/workflows/exaforce-publish.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/exaforce-publish.yaml b/.github/workflows/exaforce-publish.yaml index f0f3bd88830b2..c67cdf257cc8c 100644 --- a/.github/workflows/exaforce-publish.yaml +++ b/.github/workflows/exaforce-publish.yaml @@ -88,7 +88,7 @@ jobs: - name: Checkout Vector uses: actions/checkout@v4 - name: Bootstrap runner environment (Ubuntu-specific) - run: sudo -E bash scripts/environment/bootstrap-ubuntu-20.04.sh + run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh # # Required for some reason for 1.83 # - name: Install Rust toolchain explicitly # run: | From d6401efe8ecdfb8b7a62333931487b99382d4ce1 Mon Sep 17 00:00:00 2001 From: Lukasz Drozdz Date: Thu, 6 Mar 2025 11:07:27 +0100 Subject: [PATCH 3/3] cleanup --- .github/workflows/exaforce-publish.yaml | 12 -- scripts/environment/bootstrap-ubuntu-20.04.sh | 178 ------------------ 2 files changed, 190 deletions(-) delete mode 100755 scripts/environment/bootstrap-ubuntu-20.04.sh diff --git a/.github/workflows/exaforce-publish.yaml b/.github/workflows/exaforce-publish.yaml index c67cdf257cc8c..6a433e2cdd126 100644 --- a/.github/workflows/exaforce-publish.yaml +++ b/.github/workflows/exaforce-publish.yaml @@ -58,12 +58,6 @@ jobs: uses: actions/checkout@v4 - name: Bootstrap runner environment (Ubuntu-specific) run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh - # # Required for some reason for 1.83 - # - name: Install Rust toolchain explicitly - # run: | - # rustup set profile minimal - # rustup toolchain install $(grep 'channel' rust-toolchain.toml | cut -d '"' -f 2) --profile minimal - # rustup default $(grep 'channel' rust-toolchain.toml | cut -d '"' -f 2) - name: Bootstrap runner environment (generic) run: bash scripts/environment/prepare.sh - name: Build Vector @@ -89,12 +83,6 @@ jobs: uses: actions/checkout@v4 - name: Bootstrap runner environment (Ubuntu-specific) run: sudo -E bash scripts/environment/bootstrap-ubuntu-24.04.sh - # # Required for some reason for 1.83 - # - name: Install Rust toolchain explicitly - # run: | - # rustup set profile minimal - # rustup toolchain install $(grep 'channel' rust-toolchain.toml | cut -d '"' -f 2) --profile minimal - # rustup default $(grep 'channel' rust-toolchain.toml | cut -d '"' -f 2) - name: Bootstrap runner environment (generic) run: bash scripts/environment/prepare.sh - name: Build Vector diff --git a/scripts/environment/bootstrap-ubuntu-20.04.sh b/scripts/environment/bootstrap-ubuntu-20.04.sh deleted file mode 100755 index 2ccb4b56d052c..0000000000000 --- a/scripts/environment/bootstrap-ubuntu-20.04.sh +++ /dev/null @@ -1,178 +0,0 @@ -#! /usr/bin/env bash -set -e -o verbose - -if [ -n "$RUSTFLAGS" ] -then - # shellcheck disable=SC2016 - echo '$RUSTFLAGS MUST NOT be set in CI configs as it overrides settings in `.cargo/config.toml`.' - exit 1 -fi - -export DEBIAN_FRONTEND=noninteractive -export ACCEPT_EULA=Y - -echo 'Acquire::Retries "5";' > /etc/apt/apt.conf.d/80-retries - -apt-get update --yes - -apt-get install --yes \ - software-properties-common \ - apt-utils \ - apt-transport-https - -# Deps -apt-get install --yes --no-install-recommends \ - awscli \ - build-essential \ - ca-certificates \ - cmake \ - cmark-gfm \ - curl \ - gawk \ - gnupg2 \ - gnupg-agent \ - gnuplot \ - jq \ - libclang-dev \ - libsasl2-dev \ - libssl-dev \ - llvm \ - locales \ - pkg-config \ - rename \ - rpm \ - ruby-bundler \ - shellcheck \ - sudo \ - unzip \ - wget - -# Cue -TEMP=$(mktemp -d) -curl \ - -L https://github.com/cue-lang/cue/releases/download/v0.10.0/cue_v0.10.0_linux_amd64.tar.gz \ - -o "${TEMP}/cue_v0.10.0_linux_amd64.tar.gz" -tar \ - -xvf "${TEMP}/cue_v0.10.0_linux_amd64.tar.gz" \ - -C "${TEMP}" -cp "${TEMP}/cue" /usr/bin/cue -rm -rf "$TEMP" - -# Grease -# Grease is used for the `make release-github` task. -TEMP=$(mktemp -d) -curl \ - -L https://github.com/vectordotdev/grease/releases/download/v1.0.1/grease-1.0.1-linux-amd64.tar.gz \ - -o "${TEMP}/grease-1.0.1-linux-amd64.tar.gz" -tar \ - -xvf "${TEMP}/grease-1.0.1-linux-amd64.tar.gz" \ - -C "${TEMP}" -cp "${TEMP}/grease/bin/grease" /usr/bin/grease -rm -rf "$TEMP" - -# Locales -locale-gen en_US.UTF-8 -dpkg-reconfigure locales - -if ! command -v rustup ; then - # Rust/Cargo should already be installed on both GH Actions-provided Ubuntu 20.04 images _and_ - # by our own Ubuntu 20.04 images - curl https://sh.rustup.rs -sSf | sh -s -- -y --profile minimal -fi - -# Rust/Cargo should already be installed on both GH Actions-provided Ubuntu 20.04 images _and_ -# by our own Ubuntu 20.04 images, so this is really just make sure the path is configured. -if [ -n "${CI-}" ] ; then - echo "${HOME}/.cargo/bin" >> "${GITHUB_PATH}" - # we often run into OOM issues in CI due to the low memory vs. CPU ratio on c5 instances - echo "CARGO_BUILD_JOBS=$(($(nproc) /2))" >> "${GITHUB_ENV}" -else - echo "export PATH=\"$HOME/.cargo/bin:\$PATH\"" >> "${HOME}/.bash_profile" -fi - -# Docker. -if ! [ -x "$(command -v docker)" ]; then - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - add-apt-repository \ - "deb [arch=$(dpkg --print-architecture)] https://download.docker.com/linux/ubuntu \ - xenial \ - stable" - # Install those new things - apt-get update --yes - apt-get install --yes docker-ce docker-ce-cli containerd.io - - # ubuntu user doesn't exist in scripts/environment/Dockerfile which runs this - usermod --append --groups docker ubuntu || true -fi - -bash scripts/environment/install-protoc.sh - -# Node.js, npm and yarn. -# Note: the current LTS for the Node.js toolchain is 18.x -if ! [ -x "$(command -v node)" ]; then - curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | apt-key add - - add-apt-repository \ - "deb [arch=$(dpkg --print-architecture)] https://deb.nodesource.com/node_18.x \ - nodistro \ - main" - # Install those new things - apt-get update --yes - apt-get install --yes nodejs - - # enable corepack (enables the yarn and pnpm package managers) - # ref: https://nodejs.org/docs/latest-v18.x/api/corepack.html - corepack enable -fi - -# Apt cleanup -apt-get clean - -# Set up the default "deny all warnings" build flags -CARGO_OVERRIDE_DIR="${HOME}/.cargo" -CARGO_OVERRIDE_CONF="${CARGO_OVERRIDE_DIR}/config.toml" -cat <>"$CARGO_OVERRIDE_CONF" -[target.'cfg(linux)'] -rustflags = [ "-D", "warnings" ] -EOF - -# Install mold, because the system linker wastes a bunch of time. -# -# Notably, we don't install/configure it when we're going to do anything with `cross`, as `cross` takes the Cargo -# configuration from the host system and ships it over... which isn't good when we're overriding the `rustc-wrapper` -# and all of that. -if [ -z "${DISABLE_MOLD:-""}" ] ; then - # We explicitly put `mold-wrapper.so` right beside `mold` itself because it's hard-coded to look in the same directory - # first when trying to load the shared object, so we can dodge having to care about the "right" lib folder to put it in. - TEMP=$(mktemp -d) - MOLD_VERSION=1.2.1 - MOLD_TARGET=mold-${MOLD_VERSION}-$(uname -m)-linux - curl -fsSL "https://github.com/rui314/mold/releases/download/v${MOLD_VERSION}/${MOLD_TARGET}.tar.gz" \ - --output "$TEMP/${MOLD_TARGET}.tar.gz" - tar \ - -xvf "${TEMP}/${MOLD_TARGET}.tar.gz" \ - -C "${TEMP}" - cp "${TEMP}/${MOLD_TARGET}/bin/mold" /usr/bin/mold - cp "${TEMP}/${MOLD_TARGET}/lib/mold/mold-wrapper.so" /usr/bin/mold-wrapper.so - rm -rf "$TEMP" - - # Create our rustc wrapper script that we'll use to actually invoke `rustc` such that `mold` will wrap it and intercept - # anything linking calls to use `mold` instead of `ld`, etc. - CARGO_BIN_DIR="${CARGO_OVERRIDE_DIR}/bin" - mkdir -p "$CARGO_BIN_DIR" - - RUSTC_WRAPPER="${CARGO_BIN_DIR}/wrap-rustc" - cat <"$RUSTC_WRAPPER" -#!/bin/sh -exec mold -run "\$@" -EOF - chmod +x "$RUSTC_WRAPPER" - - # Now configure Cargo to use our rustc wrapper script. - cat <>"$CARGO_OVERRIDE_CONF" -[build] -rustc-wrapper = "$RUSTC_WRAPPER" -EOF -fi - -mkdir -p /var/lib/vector -chmod 777 /var/lib/vector