diff --git a/.github/actions/build-fixtures/action.yaml b/.github/actions/build-fixtures/action.yaml index 36fff598e02..98148e97246 100644 --- a/.github/actions/build-fixtures/action.yaml +++ b/.github/actions/build-fixtures/action.yaml @@ -25,6 +25,9 @@ inputs: evm_ref: description: "Override the t8n tool branch / tag / commit" default: "" + upload: + description: "Upload the filled fixtures. Set to false to rehearse a fill and discard its output." + default: "true" runs: using: "composite" steps: @@ -77,11 +80,12 @@ runs: exit "$EXIT_CODE" fi - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + if: inputs.upload == 'true' with: name: fixtures_${{ inputs.release_name }} path: fixtures_${{ inputs.release_name }}.tar.gz - name: Upload fixture directory (split) - if: inputs.split_label != '' + if: inputs.upload == 'true' && inputs.split_label != '' uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: fixtures__${{ inputs.split_label }} diff --git a/.github/configs/feature.yaml b/.github/configs/feature.yaml index 7fca18fd9ef..ba528347f43 100644 --- a/.github/configs/feature.yaml +++ b/.github/configs/feature.yaml @@ -1,8 +1,3 @@ -# Unless filling for special features, all features should fill for previous forks (starting from Frontier) too -mainnet: - evm-type: eels - fill-params: --until=BPO4 --generate-all-formats - monad: evm-type: eels fill-params: -m blockchain_test --from=MONAD_EIGHT --until=MONAD_TEN --chain-id=143 -k "not invalid_header" diff --git a/.github/scripts/check_release_matrix.py b/.github/scripts/check_release_matrix.py new file mode 100755 index 00000000000..556d5b92bf7 --- /dev/null +++ b/.github/scripts/check_release_matrix.py @@ -0,0 +1,124 @@ +#!/usr/bin/env -S uv run --script +# +# /// script +# requires-python = ">=3.12" +# dependencies = [ +# "pyyaml", +# ] +# /// +""" +Build the job matrix for a fixture release rehearsal. + +Usage: `check_release_matrix.py [features] [branch]`, where `features` +is an optional comma- or space-separated subset of the feature names in +`.github/configs/feature.yaml`. + +With no `features`, an EIP branch rehearses the feature it releases for +its own EIP and nothing else, and every other branch rehearses every +feature. Either way a branch that adds a feature is covered without +touching the workflow. + +Reuse `generate_build_matrix.py` so a rehearsal fills exactly what the +release fills, then flatten the per-feature matrices into the single +`fill_matrix` a `strategy.matrix` consumes. +""" + +import json +import re +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent)) + +from generate_build_matrix import ( # noqa: E402 + FEATURE_CONFIG, + FORK_RANGES_CONFIG, + build_matrix, + fail, + load_config, +) + +# EIP branches follow `eips//eip-`, with `+` joining the EIP +# numbers a combined branch carries, e.g. `eips/amsterdam/eip-2345+3456`. +EIP_BRANCH_RE = re.compile(r"^eips/[^/]+/eip-([0-9]+(?:\+[0-9]+)*)$") + + +def eip_features(defined: list[str], branch: str) -> list[str]: + """ + Return the features *branch* releases for its own EIPs. + + An EIP branch names its feature after the EIPs it carries, so + `eips/monad_next/eip-7997` releases `monad_eip7997` and + `eips/amsterdam/eip-2345+3456` releases `monad_eip2345+3456`. The + numbers must match in full: a combined branch does not claim the + feature of either EIP on its own, and neither claims the combined + one. Return an empty list for any other branch, and for an EIP + branch that has not declared a feature of its own yet. + """ + match = EIP_BRANCH_RE.match(branch) + if not match: + return [] + numbers = re.compile(rf"eip{re.escape(match.group(1))}(?![0-9+])") + return [name for name in defined if numbers.search(name)] + + +def defined_features(config: dict) -> list[str]: + """Return every feature name in `feature.yaml`, in config order.""" + return [ + name for name, feature in config.items() if isinstance(feature, dict) + ] + + +def select_features(config: dict, requested: str, branch: str) -> list[str]: + """ + Narrow the rehearsal to the requested features. + + An empty request falls back to the features *branch* releases for + its own EIPs, then to every feature: this fork releases all of + them. An unknown name fails the run rather than silently + rehearsing less than was asked for. + """ + defined = defined_features(config) + names = [name for name in requested.replace(",", " ").split() if name] + if names: + unknown = [name for name in names if name not in defined] + if unknown: + fail( + f"unknown feature(s) {', '.join(unknown)}; " + f"{FEATURE_CONFIG} defines {', '.join(defined)}" + ) + return names + from_branch = eip_features(defined, branch) + if from_branch: + print( + f"Branch '{branch}' releases {', '.join(from_branch)}; " + "rehearsing only that.", + file=sys.stderr, + ) + return from_branch + return defined + + +def main() -> None: + """Print the rehearsal's feature list and fill matrix to stdout.""" + requested = sys.argv[1] if len(sys.argv) > 1 else "" + branch = sys.argv[2] if len(sys.argv) > 2 else "" + + config = load_config(FEATURE_CONFIG) + fork_ranges = load_config(FORK_RANGES_CONFIG) or [] + + features = select_features(config, requested, branch) + if not features: + fail(f"{FEATURE_CONFIG} defines no feature") + + matrix: list[dict] = [] + for name in features: + entries, _ = build_matrix(config[name], name, fork_ranges) + matrix.extend(entries) + + print(f"features={json.dumps(features)}") + print(f"fill_matrix={json.dumps(matrix)}") + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/check_release.yaml b/.github/workflows/check_release.yaml new file mode 100644 index 00000000000..8b1f7447b6f --- /dev/null +++ b/.github/workflows/check_release.yaml @@ -0,0 +1,174 @@ +name: Check Fixture Release + +run-name: ${{ github.event_name == 'workflow_dispatch' && format('Check Fixture Release ({0}) {1}', inputs.depth, inputs.features) || 'Check Fixture Release' }} + +# Rehearse the fixture releases this fork ships, so a fill that a change +# breaks surfaces on the branch that broke it rather than on the next +# release attempt. +# +# Two tiers, both running on every pull request and on pushes to the +# branches that release in parallel with the fork branch, +# `from-upstream` and `eips/**`. +# +# `collect` is cheap: it reads the features the branch releases out of +# `.github/configs/feature.yaml`, builds the release's job matrix +# from it and collects its tests, catching malformed feature entries, +# import errors and parametrization errors in a few minutes. It gates +# `fill`, which is comprehensive: `fill` runs the very fill the release +# runs, via the release's own action, and is the only tier that catches +# a test that fills wrong. A manual dispatch can ask for `collect` +# alone, and can narrow the run to a subset of the features. + +on: + push: + branches: + - from-upstream + - "eips/**" + paths-ignore: + - "**.md" + - "LICENSE*" + - ".gitignore" + - ".vscode/**" + - "whitelist.txt" + - "docs/**" + - "mkdocs.yml" + pull_request: + paths-ignore: + - "**.md" + - "LICENSE*" + - ".gitignore" + - ".vscode/**" + - "whitelist.txt" + - "docs/**" + - "mkdocs.yml" + workflow_dispatch: + inputs: + depth: + description: "full = fill every feature (hours); collect = validate the config and collect the tests (minutes)" + required: true + type: choice + options: [full, collect] + default: full + features: + description: "Features to rehearse, e.g. monad_runloop. Empty = an EIP branch's own feature, or every feature elsewhere." + required: false + type: string + +concurrency: + group: ${{ github.workflow }}-${{ github.ref || github.run_id }} + cancel-in-progress: ${{ github.ref_name != github.event.repository.default_branch }} + +permissions: + contents: read + +jobs: + setup: + runs-on: ubuntu-latest + outputs: + features: ${{ steps.matrix.outputs.features }} + fill_matrix: ${{ steps.matrix.outputs.fill_matrix }} + fill: ${{ steps.depth.outputs.fill }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + submodules: false + - uses: ./.github/actions/setup-uv + + - name: Select the features to rehearse + id: matrix + shell: bash + env: + INPUT_FEATURES: ${{ inputs.features }} + # The head branch, so a pull request from an EIP branch + # rehearses that branch's feature; `ref_name` is the merge ref + # on a pull request and the branch everywhere else. + BRANCH: ${{ github.head_ref || github.ref_name }} + run: | + # The feature selection and the per-feature release matrix live + # in (and are shared with the release workflow via) + # check_release_matrix.py. + uv run -q .github/scripts/check_release_matrix.py \ + "$INPUT_FEATURES" "$BRANCH" | tee -a "$GITHUB_OUTPUT" + + - name: Decide whether to fill + id: depth + shell: bash + env: + EVENT: ${{ github.event_name }} + DEPTH: ${{ inputs.depth }} + run: | + # Every automatic trigger fills; only a dispatch can ask for + # the cheap tier alone. + if [ "$EVENT" = "workflow_dispatch" ] && [ "$DEPTH" != "full" ] + then + fill=false + else + fill=true + fi + echo "Fill: $fill" + echo "fill=$fill" >> "$GITHUB_OUTPUT" + + collect: + name: collect (${{ matrix.feature }}) + needs: setup + runs-on: ubuntu-24.04 + timeout-minutes: 30 + strategy: + # A rehearsal wants every feature's verdict, not the first failure. + fail-fast: false + matrix: + feature: ${{ fromJson(needs.setup.outputs.features) }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + submodules: true + - uses: ./.github/actions/setup-uv + - name: Install EEST + run: uv sync --no-progress + + - name: Extract fixture release properties from config + id: properties + run: | + uv run -q .github/scripts/get_release_props.py ${{ matrix.feature }} >> "$GITHUB_OUTPUT" + + - name: Collect the release's tests + shell: bash + run: | + # `fill-params` is interpolated rather than passed through a + # variable so its quoting survives, as in `build-fixtures`. + # Collecting nothing means an empty release, so pytest's exit + # code 5 fails here where a release's fork-range split + # tolerates it. + status=0 + uv run fill --collect-only -q \ + ${{ steps.properties.outputs.fill-params }} > collected.txt \ + || status=$? + tail -n 20 collected.txt + exit "$status" + + fill: + name: fill (${{ matrix.label || matrix.feature }}) + needs: [setup, collect] + if: needs.setup.outputs.fill == 'true' + runs-on: ubuntu-24.04 + # Hosted runners cap a job at six hours regardless; the explicit + # timeout keeps a wedged fill from burning the whole budget. + timeout-minutes: 360 + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(needs.setup.outputs.fill_matrix) }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + submodules: true + + - uses: ./.github/actions/build-fixtures + with: + release_name: ${{ matrix.feature }} + from_fork: ${{ matrix.from_fork }} + until_fork: ${{ matrix.until_fork }} + split_label: ${{ matrix.label }} + # A rehearsal only reports whether the fill succeeds; keeping + # its fixtures would cost storage for output nothing consumes. + upload: "false"