diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index d8de8b5ecd..3d612254a9 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -273,6 +273,7 @@ jobs: BEHAVIOUR_SCM: github BEHAVIOUR_CI: githubactions BEHAVIOUR_INSTALL_MODE: per-repo + BEHAVIOUR_FULLSEND_REF: ${{ github.event.pull_request.head.sha || github.sha }} BEHAVIOUR_ARTIFACT_DIR: ${{ runner.temp }}/behaviour-artifacts E2E_GCP_PROJECT_ID: ${{ secrets.E2E_GCP_PROJECT_ID }} E2E_GCP_WIF_PROVIDER: ${{ secrets.E2E_GCP_WIF_PROVIDER }} diff --git a/docs/ADRs/0066-behaviour-tests-with-gherkin-and-drivers.md b/docs/ADRs/0066-behaviour-tests-with-gherkin-and-drivers.md index 4d33f84380..3f55aa30de 100644 --- a/docs/ADRs/0066-behaviour-tests-with-gherkin-and-drivers.md +++ b/docs/ADRs/0066-behaviour-tests-with-gherkin-and-drivers.md @@ -29,17 +29,17 @@ Runtime selection is shared with production via `defaults.runtime` in org `confi - Add **behaviour tests** under `e2e/behaviour/` using **godog** and portable Gherkin feature files. - Exercise **real SCM + real CI** through **driver interfaces** (`scm.Driver`, `ci.Driver`, `install.Driver`); v1 implementations target GitHub and GitHub Actions. - Substitute inference with a **dummy runtime** (`runtime: dummy` in per-repo config, or `defaults.runtime: dummy` for per-org) that executes scripted operations in the real OpenShell sandbox and emits `behaviour-results.json`. -- Select backends via **runner env** (`BEHAVIOUR_SCM`, `BEHAVIOUR_CI`, `BEHAVIOUR_INSTALL_MODE`); feature files stay install-mode agnostic. v1 runs **per-repo** against the halfsend org pool; the suite provisions fullsend via `fullsend github setup` rather than requiring pre-installed orgs. +- Select backends via **runner env** (`BEHAVIOUR_SCM`, `BEHAVIOUR_CI`, `BEHAVIOUR_INSTALL_MODE`); feature files stay install-mode agnostic. ~~v1 runs **per-repo** against the halfsend org pool; the suite provisions fullsend via `fullsend github setup` rather than requiring pre-installed orgs.~~ **Note (2026-09, #6815):** Behaviour tests now use the dedicated `fullsend-ai-test` org with ephemeral `bt-{uuid}-{slot}` repos and `repos install --fullsend-ref`. - Use **compatibility tags** (`@skip:*`, `@requires:*`) to filter scenarios for future backends; tags do not select configuration. ## Consequences - Behaviour tests can pass while prompt quality regresses; LLM evals remain necessary for instruction coverage. - Behaviour orgs are provisioned at suite start with `--runtime dummy`; production orgs must not use dummy unintentionally. -- **Note (2026-07, #5439 / PR #5489):** Numbered behaviour pool repos (`test-repo-NN`) are lazily created and installed on first scenario use via the ensurer internal to `install.Driver`; suite-start provisioning still applies to the shared admin/`test-repo` install path where used. +- ~~**Note (2026-07, #5439 / PR #5489):** Numbered behaviour pool repos (`test-repo-NN`) are lazily created and installed on first scenario use via the ensurer internal to `install.Driver`; suite-start provisioning still applies to the shared admin/`test-repo` install path where used.~~ **Note (2026-09, #6815):** Repos are now ephemeral `bt-{uuid}-{slot}` in `fullsend-ai-test`, created per scenario and deleted on deallocation. - Adding GitLab or Tekton requires new drivers and runner env values, not feature file rewrites. - Dummy runtime op vocabulary stays minimal; new ops require runtime + docs updates when scenarios need them. -- Behaviour tests depend on live external infrastructure: GitHub API, GitHub Actions runners, GCP WIF/mint, and the shared halfsend org pool. Transient outages, API rate limits, or pool org state corruption can fail the suite; CI distinguishes infrastructure failures from regressions via workflow logs and artifact inspection, but there is no offline fallback. -- Behaviour tests share the halfsend org pool and lock mechanism with admin e2e tests (`e2e.yml` runs both jobs). Lock hold time scales with scenario count; pool size was doubled to absorb the additional load and can be increased again if contention appears. +- Behaviour tests depend on live external infrastructure: GitHub API, GitHub Actions runners, GCP WIF/mint, and ~~the shared halfsend org pool~~ the dedicated `fullsend-ai-test` org. Transient outages, API rate limits, or ~~pool org state corruption~~ ephemeral repo lifecycle failures can fail the suite; CI distinguishes infrastructure failures from regressions via workflow logs and artifact inspection, but there is no offline fallback. **Note (2026-09, #6815):** Behaviour tests migrated from the shared halfsend pool to `fullsend-ai-test` with ephemeral repos; pool-specific failure modes (lock contention, state corruption) no longer apply. +- ~~Behaviour tests share the halfsend org pool and lock mechanism with admin e2e tests (`e2e.yml` runs both jobs). Lock hold time scales with scenario count; pool size was doubled to absorb the additional load and can be increased again if contention appears.~~ **Note (2026-09, #6815):** Behaviour tests no longer share the halfsend org pool or lock mechanism. They use `fullsend-ai-test` with unique `bt-{uuid}-{slot}` repos per CI run, eliminating pool contention. -> **Note (2026-07):** Shared live-test infrastructure (org pool, CLI runner, cleanup) lives in `pkg/e2etest/`; the Gherkin framework lives in `pkg/behaviourtest/`. In-repo runners remain under `e2e/behaviour/` and `e2e/admin/`. +> **Note (2026-07):** Shared live-test infrastructure (~~org pool, CLI runner, cleanup~~ `TokenForBehaviourOrg`, `BehaviourTestOrg`) lives in `pkg/e2etest/`; the Gherkin framework lives in `pkg/behaviourtest/`. In-repo runners remain under `e2e/behaviour/` and `e2e/admin/`. **(2026-09, #6815):** Behaviour tests use `fullsend-ai-test` with ephemeral repos; org pool and lock helpers are used only by admin e2e. diff --git a/docs/guides/dev/behaviour-drivers.md b/docs/guides/dev/behaviour-drivers.md index 3928c83893..a4bff73702 100644 --- a/docs/guides/dev/behaviour-drivers.md +++ b/docs/guides/dev/behaviour-drivers.md @@ -28,23 +28,25 @@ Set when starting the suite (not in feature files): BEHAVIOUR_SCM=github # also: gitlab; future: forgejo BEHAVIOUR_CI=githubactions # also: gitlabci; future: tekton BEHAVIOUR_INSTALL_MODE=per-repo # v1 default and only supported value +BEHAVIOUR_FULLSEND_REF= # head SHA for repos install --fullsend-ref (CI: pr head sha) +E2E_KEEP_REPOS=false # set true to preserve ephemeral repos for debugging ENVIRONMENT=dev # mint/infra target: dev (default) or stage ``` -The suite in `e2e/behaviour/suite_test.go` (or an external runner) acquires a pool org via `pkg/e2etest`, runs pre-install cleanup, calls an `install.Factory` (e.g. `install.NewRepoPoolCFMintPreviews(...)`) to get a unified `install.Driver` that owns mint deploy, pool allocation, repo ensure, and teardown. The suite constructs SCM and CI drivers, then runs godog with `pkg/behaviourtest/suite.InitScenario`. `InitScenario` clones a template `*world.World` per scenario. When a scenario calls "Given the enrolled test repository", `Driver.AllocateRepo` leases a unique repo name and ensures it is created and installed. `Driver.DeallocateRepo` returns the name in the After hook. `Driver.Finalize` tears down suite-scoped resources (e.g. preview mint) and reclaims outstanding leases. Unsupported `BEHAVIOUR_INSTALL_MODE` or `ENVIRONMENT` values fail at suite startup. `ENVIRONMENT` is `dev` or `stage` (empty defaults to `dev`). +The suite in `e2e/behaviour/suite_test.go` (or an external runner) uses the dedicated `fullsend-ai-test` org, calls an `install.Factory` (e.g. `install.NewRepoPoolCFMintPreviews(...)`) to get a unified `install.Driver` that owns mint deploy, ephemeral repo allocation, repo ensure, and teardown. The suite constructs SCM and CI drivers, then runs godog with `pkg/behaviourtest/suite.InitScenario`. `InitScenario` clones a template `*world.World` per scenario. When a scenario calls "Given the enrolled test repository", `Driver.AllocateRepo` leases a unique ephemeral repo name (`bt-{uuid}-{slot}`) and ensures it is created and installed. `Driver.DeallocateRepo` deletes the ephemeral repo and returns the slot in the After hook. `Driver.Finalize` tears down suite-scoped resources (e.g. preview mint) and reclaims outstanding leases. Unsupported `BEHAVIOUR_INSTALL_MODE` or `ENVIRONMENT` values fail at suite startup. `ENVIRONMENT` is `dev` or `stage` (empty defaults to `dev`). ### Install driver (unified) The suite uses a single unified `install.Driver` constructed via `install.Factory` (e.g. `install.NewRepoPoolCFMintPreviews` or `install.NewRepoPoolExternalMint`). Each concrete driver owns the full lifecycle: 1. Deploys the mint (RepoPoolCFMintPreviews: CF Worker preview; RepoPoolExternalMint: pre-configured URL). -2. Manages an internal channel-based pool of repo names (`test-repo-01` … `test-repo-12`). -3. Lazily creates and installs numbered pool repos on demand via an internal ensurer (concurrent-safe via singleflight). +2. Manages an internal channel-based pool of ephemeral repo slots (`bt-{uuid}-{slot}`). +3. Lazily creates and installs ephemeral repos on demand via an internal ensurer (concurrent-safe via singleflight). 4. Exposes `AllocateRepo` / `DeallocateRepo` / `Finalize` / `Capacity`. The Factory takes the allocated org name plus runtime dependencies (forge client, token, CLI binary, GCP project, logger). Driver-specific inputs (PEMs, allowlists, pool size, mint URL) come from env or are computed inside the driver. The suite does not construct or thread pool, ensurer, or mint driver types directly — all internal lifecycle is encapsulated inside the concrete driver returned by the factory. Default concurrency is `driver.Capacity()`; `GODOG_CONCURRENCY` overrides it (warn, do not fail, if concurrency > Capacity). -Pool orgs must already have shared GitHub Apps, org-level mint enrollment, and per-repo mint enrollment for each numbered repo (one-time GCP admin step on the hosted mint project). The driver does not run `fullsend admin install` or `fullsend mint enroll`. See [e2e-testing.md](e2e-testing.md#behaviour-tests-and-per-repo-mint-enrollment). +The `fullsend-ai-test` org must have shared GitHub Apps and org-level mint enrollment. Per-repo mint enrollment for ephemeral repos is pre-provisioned by a GCP admin on the hosted mint project. The driver does not run `fullsend admin install` or `fullsend mint enroll`. See [e2e-testing.md](e2e-testing.md#behaviour-tests-and-per-repo-mint-enrollment). `Finalize` (RepoPoolCFMintPreviews) abandons the preview alias via `fullsend mint delete --platform=cloudflare` and reclaims any outstanding leases with an error. The RepoPoolExternalMint driver's teardown is a no-op. @@ -71,7 +73,7 @@ Steps use `w.Org` and `w.RepoName` (the allocated repo name) plus per-repo const ## Testing drivers -Prefer unit tests with `httptest` for REST helpers. Optional smoke scenarios against live backends mirror admin e2e credentials (`GITHUB_TOKEN`, halfsend org pool). +Prefer unit tests with `httptest` for REST helpers. Optional smoke scenarios against live backends mirror admin e2e credentials (`GITHUB_TOKEN`, `fullsend-ai-test` org). ## Future backends checklist diff --git a/docs/guides/dev/behaviour-testing.md b/docs/guides/dev/behaviour-testing.md index 6f62213cd2..27221dcc35 100644 --- a/docs/guides/dev/behaviour-testing.md +++ b/docs/guides/dev/behaviour-testing.md @@ -77,7 +77,7 @@ And the agent will output issues.out with: Every scenario runs the stage under the dummy runtime selected at install time (`github setup … --runtime dummy`). The runtime layer gets two kinds of coverage without leasing extra repos or adding wall time: - **Core (every run):** `Then the run selected the "dummy" runtime` reads the `runtime` field the runner writes into `metrics.json`, proving the repo's `.fullsend/config.yaml` `runtime:` reached backend selection. Use it in one representative scenario per stage; the artifact is already downloaded for the other assertions. -- **Runtime-specific (gated):** `Given the repository runtime is ""` commits `runtime: ` to the leased repo's config for this scenario only (CleanupScenario restores `dummy` — slots are reused, so never set it any other way; the step refuses if the slot is not on `dummy` to begin with). The custom-harness step commits only a placeholder for a relative `agent:` path, which a real runtime cannot act on, so follow it with `And a pi agent "" defined as:` and a docstring holding the full agent file (frontmatter + body) — `{{fixture:fixtures//.json}}` inlines a result fixture so the model has a concrete, deterministic file to write (the custom harness carries no post-script, so nothing validates it; the assertions are on the transcript and metrics). Then the scenario dispatches the harness and asserts on artifacts: `the run selected the "pi" runtime`, `the pi session transcript records at least one tool call` (the agent used a tool through pi; with security enabled the run refuses to start without the intact hook adapter, so the call was mediated by it — the step does not inspect hook output), `the run metrics report tokens`. Such scenarios cost a real model run on the pool repo's repo-scoped Vertex WIF and must be tagged `@requires:capability:runtime-` so they only run where the runner declares the capability; `make behaviour-test` declares `runtime-pi` by default (a `Makefile` variable, so a PR adding a gated scenario exercises it on its own `pull_request_target` run — the workflow file itself comes from `main`); `BEHAVIOUR_CAPABILITIES= make behaviour-test` skips them. See `features/runtime/pi.feature`. `features/runtime/pi-openai.feature` is the same shape on `openai/gpt-5.6-luna` with the `openai` provider instead of Vertex host files; it is gated on `runtime-pi-openai`, which is **not** declared by default because it needs an OpenAI organization mapped to the pool repositories plus their `FULLSEND_OPENAI_*` variables ([OpenAI Workload Identity](../infrastructure/openai-workload-identity.md)). +- **Runtime-specific (gated):** `Given the repository runtime is ""` commits `runtime: ` to the leased repo's config for this scenario only (CleanupScenario restores `dummy` — slots are reused, so never set it any other way; the step refuses if the slot is not on `dummy` to begin with). The custom-harness step commits only a placeholder for a relative `agent:` path, which a real runtime cannot act on, so follow it with `And a pi agent "" defined as:` and a docstring holding the full agent file (frontmatter + body) — `{{fixture:fixtures//.json}}` inlines a result fixture so the model has a concrete, deterministic file to write (the custom harness carries no post-script, so nothing validates it; the assertions are on the transcript and metrics). Then the scenario dispatches the harness and asserts on artifacts: `the run selected the "pi" runtime`, `the pi session transcript records at least one tool call` (the agent used a tool through pi; with security enabled the run refuses to start without the intact hook adapter, so the call was mediated by it — the step does not inspect hook output), `the run metrics report tokens`. Such scenarios cost a real model run on the leased repo's repo-scoped Vertex WIF and must be tagged `@requires:capability:runtime-` so they only run where the runner declares the capability; `make behaviour-test` declares `runtime-pi` by default (a `Makefile` variable, so a PR adding a gated scenario exercises it on its own `pull_request_target` run — the workflow file itself comes from `main`); `BEHAVIOUR_CAPABILITIES= make behaviour-test` skips them. See `features/runtime/pi.feature`. `features/runtime/pi-openai.feature` is the same shape on `openai/gpt-5.6-luna` with the `openai` provider instead of Vertex host files; it is gated on `runtime-pi-openai`, which is **not** declared by default because it needs an OpenAI organization mapped to the test org repositories plus their `FULLSEND_OPENAI_*` variables ([OpenAI Workload Identity](../infrastructure/openai-workload-identity.md)). - **Per-agent (every run):** `Given the repository agents are configured with:` with a YAML docstring (`triage:\n runtime: dummy`) sets runtime/model/effort on the leased repo's `agents:` entries (a name-only entry for a built-in, the sourced entry for a custom agent; only the settings given change) — validated the way `fullsend run` validates them — and CleanupScenario restores the pre-scenario `agents:` list. Pair it with `the repository runtime is ""` and pin every agent the scenario can dispatch (triage hands off to `code` via `ready-to-code`) back to `dummy`, then assert `the run selected the "dummy" runtime from "agents.triage"`, which also checks `runtime_source` in `metrics.json` ends with that entry — proof the per-agent entry decided, at dummy cost. The gated second scenario in the same file leaves the repo on `dummy` and puts one custom agent on pi with `model: haiku` from its entry (the harness says `opus`); `the run requested model "haiku" from "agents." and the provider reported a "haiku" model` checks `requested_model`, `override_source`, the reported `model` and `num_turns` in `metrics.json`. See `features/runtime/agent-settings.feature`. Do not add runtime coverage to `e2e/admin` (org-mode install, deprecated per ADR 0044) or behind new `fullsend admin` flags. @@ -188,17 +188,18 @@ The first row proves execution via an auxiliary file; the second row emits the s ## Running locally ```bash -# Local: gh auth login or export GH_TOKEN/GITHUB_TOKEN with access to halfsend org pool +# Local: gh auth login or export GH_TOKEN/GITHUB_TOKEN with access to fullsend-ai-test org make behaviour-test ``` ### Parallel execution The suite runs scenarios in parallel by default (`GODOG_CONCURRENCY=12`, -matching the repo pool size). Each scenario gets its own `World` clone and -leases a unique `test-repo-NN` from the pool, so no cross-scenario state -is shared. The `behaviour-test` Make target includes `-race` to catch -data races under concurrent execution. +matching the driver pool size). Each scenario gets its own `World` clone +and leases a unique ephemeral `bt-{uuid}-{slot}` repo in the single +`fullsend-ai-test` org, so no cross-scenario state is shared and +concurrent CI runs never collide. The `behaviour-test` Make target +includes `-race` to catch data races under concurrent execution. To adjust concurrency: @@ -216,22 +217,37 @@ GODOG_CONCURRENCY=1 make behaviour-test Serial mode (`GODOG_CONCURRENCY=1`) is useful when debugging a single scenario or when `-v` output from multiple scenarios would interleave. -In CI, the test runner mints cross-org `e2e` installation tokens via OIDC (same as admin e2e) for GitHub API operations. Triage workflows on the pool org's `test-repo` mint same-org `triage` tokens from vendored reusable workflows; those require per-repo mint enrollment (`PER_REPO_WIF_REPOS`) on the hosted mint project. Pool `test-repo` repos are enrolled once by a GCP admin — not during CI install. The install driver provisions repo-scoped inference WIF via `fullsend inference provision` before `github setup`. See [e2e-testing.md](e2e-testing.md#behaviour-tests-and-per-repo-mint-enrollment). +Set `E2E_KEEP_REPOS=true` to preserve ephemeral repos after deallocation +for post-mortem debugging. Without this, repos are deleted when each +scenario's After hook calls `DeallocateRepo`. + +When `BEHAVIOUR_FULLSEND_REF` is set (falls back to `GITHUB_HEAD_REF` then +`GITHUB_REF_NAME`), the install driver uses `repos install --fullsend-ref` +instead of `github setup --vendor`. This avoids vendoring a ~50 MB binary +per ephemeral repo — `action.yml` resolves the SHA and builds from source. + +In CI, the test runner mints a `e2e` installation token for the +`fullsend-ai-test` org via OIDC. Triage workflows on ephemeral repos mint +same-org `triage` tokens from reusable workflows; those require per-repo +mint enrollment (`PER_REPO_WIF_REPOS`) on the hosted mint project. The +install driver provisions repo-scoped inference WIF via +`fullsend inference provision` before install. See +[e2e-testing.md](e2e-testing.md#behaviour-tests-and-per-repo-mint-enrollment). ### Repo allocation via unified Driver -The `Given the enrolled test repository` step allocates a repo via `Driver.AllocateRepo(ctx)`. The unified `install.Driver` (constructed by a `Factory` during suite setup) owns pool leasing and lazy create+install internally: +The `Given the enrolled test repository` step allocates a repo via `Driver.AllocateRepo(ctx)`. The unified `install.Driver` (constructed by a `Factory` during suite setup) owns slot leasing and lazy create+install internally: -1. Leases a slot from the internal pool (blocks until one is free or ctx is cancelled). +1. Leases a slot from the internal pool (blocks until one is free or ctx is cancelled). Slot names are unique per run: `bt-{uuid}-{slot}`. 2. Creates the repo if it does not exist (the forge's `auto_init` provides the initial commit). -3. Validates post-install files; if validation fails, runs `fullsend github setup` (and inference provision when configured). +3. Installs fullsend via `repos install --fullsend-ref` (when configured) or `github setup --vendor`. 4. Caches results by `org/repo` key so subsequent scenarios reuse the same State. -The After hook calls `Driver.DeallocateRepo` to return the slot. `Driver.Finalize` tears down suite-scoped resources (e.g. preview mint) and reclaims outstanding leases with an error. +The After hook calls `Driver.DeallocateRepo` to return the slot and delete the ephemeral repo (unless `E2E_KEEP_REPOS=true`). `Driver.Finalize` tears down suite-scoped resources (e.g. preview mint) and reclaims outstanding leases with an error. -Concurrent callers for the same repo are serialized via `singleflight.Group` — only one goroutine runs the create+install flow while others wait. This removes the requirement for numbered `test-repo-NN` repos to be pre-provisioned in the pool org. +Concurrent callers for the same repo are serialized via `singleflight.Group` — only one goroutine runs the create+install flow while others wait. -**Suite duration:** Because each leased `test-repo-NN` pays create + inference provision + `github setup` on first use in a run, serial godog suites take longer than the old shared-`test-repo` model. CI budgets **45 minutes** for the behaviour job (`timeout-minutes` and `go test -timeout`) to match. +**Suite duration:** Because each leased repo pays create + inference provision + install on first use in a run, serial godog suites take longer than a pre-provisioned model. CI budgets **45 minutes** for the behaviour job (`timeout-minutes` and `go test -timeout`) to match. Runner env (defaults shown): @@ -239,9 +255,11 @@ Runner env (defaults shown): BEHAVIOUR_SCM=github # also: gitlab; future: forgejo BEHAVIOUR_CI=githubactions # also: gitlabci; future: tekton BEHAVIOUR_INSTALL_MODE=per-repo +BEHAVIOUR_FULLSEND_REF= # head SHA for repos install --fullsend-ref (CI: pr head sha) +E2E_KEEP_REPOS=false # set true to preserve ephemeral repos for debugging ENVIRONMENT=dev # mint/infra target: dev (default, local and PRs) or stage (push to main) -E2E_GCP_PROJECT_ID=... # inference project; install runs inference provision per pool repo -E2E_GCP_WIF_PROVIDER=... # CI job GCP auth (not written to pool test-repo secrets) +E2E_GCP_PROJECT_ID=... # inference project; install runs inference provision per ephemeral repo +E2E_GCP_WIF_PROVIDER=... # CI job GCP auth (not written to ephemeral test-org repo secrets) TEST_ACTOR_WRITE_PAT=... # write-level human-like actor PAT (CI: same-named repo secret) TEST_ACTOR_TRIAGE_PAT=... # triage-level human-like actor PAT TEST_ACTOR_OUTSIDER_PAT=... # outsider human-like actor PAT (no org write on base) @@ -260,15 +278,15 @@ The three test actor accounts (`fstest-write`, `fstest-triage`, `fstest-outsider | fullsend-ai org member | No | No | No | | Permission on `fullsend-ai/fullsend` | Read | Read | Read | | Permission on `fullsend-ai/agents` | Read | Read | Read | -| Write access | Pool-org `test-repo-NN` repos only | Pool-org `test-repo-NN` repos only | None (outsider) | +| Write access | Ephemeral `bt-{uuid}-{slot}` repos in `fullsend-ai-test` only | Ephemeral `bt-{uuid}-{slot}` repos in `fullsend-ai-test` only | None (outsider) | -**Blast-radius containment:** All three accounts hold classic PATs. Because the accounts are not members of the `fullsend-ai` org and have only read permission on production repositories (`fullsend-ai/fullsend`, `fullsend-ai/agents`), a compromised PAT cannot push commits, merge PRs, or modify settings on any production repo. Write capability is scoped exclusively to disposable `test-repo-NN` infrastructure in the pool org, which is ephemeral and rebuilt each CI run. +**Blast-radius containment:** All three accounts hold classic PATs. Because the accounts are not members of the `fullsend-ai` org and have only read permission on production repositories (`fullsend-ai/fullsend`, `fullsend-ai/agents`), a compromised PAT cannot push commits, merge PRs, or modify settings on any production repo. Write capability is scoped exclusively to ephemeral `bt-{uuid}-{slot}` repos in `fullsend-ai-test`, which are created per scenario and deleted on deallocation. **Re-verification guidance:** Re-verify account permissions whenever: - A new test actor account is added - An existing account is granted additional repository or org access -- The pool-org infrastructure changes (new orgs, new repo naming) +- The test-org infrastructure changes (org settings, repo naming) To verify, check org membership and repository permissions via the GitHub API: @@ -292,13 +310,13 @@ Fork dispatch scenarios test `pull_request_target` harness triggering from cross ### Logical fork name → leased base -Gherkin keeps a stable logical name (for example `"test-repo-fork"`). At runtime, `Given a fork` remaps that name to **`{World.RepoName}-fork`** when the scenario has leased a numbered base (for example leased `test-repo-07` → actual fork repo `test-repo-07-fork`). Feature files should keep using `"test-repo-fork"`; do not hard-code `test-repo-NN-fork` in Gherkin. +Gherkin keeps a stable logical name (for example `"test-repo-fork"`). At runtime, `Given a fork` remaps that name to **`{World.RepoName}-fork`** when the scenario has leased an ephemeral base (for example leased `bt-a1b2c3d4-01` → actual fork repo `bt-a1b2c3d4-01-fork`). Feature files should keep using `"test-repo-fork"`; do not hard-code repo names in Gherkin. -### Pool-org prerequisites +### Test-org prerequisites -Fork scenarios require the pool org to have: +Fork scenarios require the `fullsend-ai-test` org to have: -- **Permission to create forks** of the leased enrolled base (`test-repo-NN`) under the same org. The `Given a fork` step creates `{leased}-fork` idempotently when missing. +- **Permission to create forks** of the leased enrolled base (`bt-{uuid}-{slot}`) under the same org. The `Given a fork` step creates `{leased}-fork` idempotently when missing. - **The same installation token** must have write access to both the base repo and the fork repo within the org, since the e2e bot commits to the fork and opens cross-fork PRs. ### Fork lifecycle @@ -361,7 +379,7 @@ Reference: [`ensureRepoExists`](../../../pkg/behaviourtest/drivers/install/ensur ### Fork name derivation depends on `World.RepoName` -The `Given a fork` step resolves the fork repo name by replacing the `test-repo` prefix with `World.RepoName`. For example, the logical Gherkin name `"test-repo-fork"` with a leased base `test-repo-07` resolves to `test-repo-07-fork`. +The `Given a fork` step resolves the fork repo name by replacing the `test-repo` prefix with `World.RepoName`. For example, the logical Gherkin name `"test-repo-fork"` with a leased base `bt-a1b2c3d4-01` resolves to `bt-a1b2c3d4-01-fork`. When modifying repo naming, leasing, or provisioning logic, verify that fork steps still resolve correctly. If `World.RepoName` changes (e.g., because leasing logic changes), fork resolution breaks — scenarios that use `Given a fork` will create or look for the wrong repo. @@ -389,9 +407,9 @@ URL dispatch scenarios test `FetchAgentHarness` URL resolution for agents whose ### Harness-hosting repository -The `Given a harness-hosting repository ""` step creates a public repository in the pool org to host harness YAML files. The repo is: +The `Given a harness-hosting repository ""` step creates a public repository in the `fullsend-ai-test` org to host harness YAML files. The repo is: -- **Ephemeral / per-scenario** — created per-scenario and deleted by `CleanupScenario` (same lifecycle as fork repos). When a leased repo is in use, the logical name is remapped via `resolveHostRepoName` (e.g. `"url-harness-host"` + leased `"test-repo-07"` → `"test-repo-07-url-harness-host"`) so parallel scenarios each get their own isolated hosting repo. +- **Ephemeral / per-scenario** — created per-scenario and deleted by `CleanupScenario` (same lifecycle as fork repos). When a leased repo is in use, the logical name is remapped via `resolveHostRepoName` (e.g. `"url-harness-host"` + leased `"bt-a1b2c3d4-01"` → `"bt-a1b2c3d4-01-url-harness-host"`) so parallel scenarios each get their own isolated hosting repo. - **Public** — required for unauthenticated `raw.githubusercontent.com` access. The step calls `EnsureRepoPublic` to detect and fix org policies that force repos private. ### URL-sourced custom harness @@ -422,7 +440,7 @@ Background: URL-dispatch scenarios require a vendored CLI binary that includes `FetchPolicy`-aware harness dispatch. Production dispatch uses `fetch.DefaultPolicy` (allows `github.com` and `raw.githubusercontent.com`) when `Options.FetchPolicy` is nil — this is what enables URL-sourced agents to resolve `raw.githubusercontent.com` URLs. -The install driver's internal ensurer always re-vendors the CLI binary (`github setup --vendor`) even when a prior install's post-install validation passes. This guarantees leased pool repos run the binary built from the current checkout rather than a stale binary from a previous CI run. Without re-vendoring, pool repos that passed validation would keep a pre-fix binary and silently fail to dispatch URL-sourced agents. +When `BEHAVIOUR_FULLSEND_REF` is set, the install driver uses `repos install --fullsend-ref` to install from the current checkout's ref — `action.yml` resolves the SHA and builds from source, avoiding the ~50 MB vendored binary per repo. When `BEHAVIOUR_FULLSEND_REF` is not set, the driver falls back to `github setup --vendor`, re-vendoring the CLI binary even when a prior install's post-install validation passes. This guarantees leased ephemeral repos run the binary built from the current checkout rather than a stale binary from a previous CI run. Without re-install, ephemeral repos that passed validation would keep a pre-fix binary and silently fail to dispatch URL-sourced agents. The settle step (polling for GitHub Actions workflow readiness) is skipped on re-vendors since the workflow file already existed — only fresh installs incur the settle wait. @@ -435,7 +453,7 @@ require github.com/fullsend-ai/fullsend v0.x.y // released tag, not @main ``` - Import `github.com/fullsend-ai/fullsend/pkg/behaviourtest/...` for world, steps, drivers, and `suite.InitScenario`. -- Import `github.com/fullsend-ai/fullsend/pkg/e2etest` for org pool acquisition, env config, CLI build/run, and cleanup. +- Import `github.com/fullsend-ai/fullsend/pkg/e2etest` for org/token resolution, env config, CLI build/run, and cleanup. - Set `world.FixturesRoot` to the module-relative fixtures directory (e.g. `"behaviour"` in the agents repo). - Build the fullsend CLI with `e2etest.BuildModuleBinary(t, "github.com/fullsend-ai/fullsend")` — not `BuildCLIBinary`, which resolves the **current** module root. - Run with `-tags behaviour` and the same env vars as CI (see above). diff --git a/docs/guides/dev/e2e-testing.md b/docs/guides/dev/e2e-testing.md index b15bf6e391..ed516e9e70 100644 --- a/docs/guides/dev/e2e-testing.md +++ b/docs/guides/dev/e2e-testing.md @@ -38,7 +38,7 @@ Optional environment variables: | `E2E_LOCK_TIMEOUT` | Max wait for a free pool org (default 10m) | | `E2E_GCP_PROJECT_ID` | GCP project for inference setup (`github setup --inference-project`) | -Behaviour tests use the same pool orgs but install via `fullsend github setup` (per-repo) instead of `fullsend admin install`. See [behaviour-testing.md](behaviour-testing.md) and [behaviour-drivers.md](behaviour-drivers.md). +Behaviour tests use a dedicated `fullsend-ai-test` org with ephemeral `bt-{uuid}-{slot}` repos and install via `repos install --fullsend-ref`. See [behaviour-testing.md](behaviour-testing.md) and [behaviour-drivers.md](behaviour-drivers.md). Tests acquire an exclusive lock on one org from the pool (`halfsend-01` … `halfsend-12`) — see [ADR 0040](../../ADRs/0040-org-pool-for-parallel-e2e-tests.md). @@ -86,11 +86,11 @@ Prefer **`wrangler versions upload --name=mint-test --preview-alias=…`** so ru ### Behaviour tests and per-repo mint enrollment -Behaviour tests install fullsend in **per-repo** mode (`fullsend github setup`). Triage workflows mint same-org `triage` tokens from vendored reusable workflows; that requires per-repo mint enrollment (`PER_REPO_WIF_REPOS`). The install driver does **not** run `mint enroll` — pool org behaviour repos must be enrolled once by a GCP admin on the hosted mint project. +Behaviour tests install fullsend in **per-repo** mode using `repos install --fullsend-ref` (or `github setup --vendor` as fallback). They run in the dedicated `fullsend-ai-test` org with ephemeral `bt-{uuid}-{slot}` repos that are created on demand and deleted after each scenario. Triage workflows on ephemeral repos mint same-org `triage` tokens from reusable workflows; that requires per-repo mint enrollment (`PER_REPO_WIF_REPOS`). The install driver does **not** run `mint enroll` — per-repo enrollment for ephemeral repo names must be pre-provisioned by a GCP admin on the hosted mint project. -Admin e2e uses the singular `halfsend-NN/test-repo` name. Behaviour tests allocate numbered `halfsend-NN/test-repo-01` … `test-repo-12` names via the unified `install.Driver`; these repos are **lazily created and installed** on demand (see [behaviour-testing.md](behaviour-testing.md#repo-allocation-via-unified-driver)). Pre-provisioning numbered repos in the pool org is no longer required — mint enrollment for those names is still pre-provisioned so it is not on the critical path. Enroll base names only — do **not** enroll `*-fork` names (forks are ephemeral PR sources and mint against the enrolled base repo). GitHub repositories need not exist yet — enroll is a mint allowlist / WIF-provider update only. +Admin e2e uses the singular `halfsend-NN/test-repo` name. Behaviour tests use the `fullsend-ai-test` org with unique `bt-{uuid}-{slot}` repo names via the unified `install.Driver`; these repos are **lazily created and installed** on demand (see [behaviour-testing.md](behaviour-testing.md#repo-allocation-via-unified-driver)). Concurrent CI runs never collide because each run generates a unique UUID prefix. Enroll base names only — do **not** enroll `*-fork` names (forks are ephemeral PR sources and mint against the enrolled base repo). GitHub repositories need not exist yet — enroll is a mint allowlist / WIF-provider update only. -Inference (`E2E_GCP_PROJECT_ID`) and mint (`it-gcp-konflux-dev-fullsend` for the hosted mint) may be different GCP projects. The behaviour install driver runs `fullsend inference provision /test-repo` using CI credentials on the inference project (same access model as admin e2e), then passes the repo-scoped WIF provider to `github setup`. `E2E_GCP_WIF_PROVIDER` authenticates the CI job itself; it is not written to pool org repos. +Inference (`E2E_GCP_PROJECT_ID`) and mint (`it-gcp-konflux-dev-fullsend` for the hosted mint) may be different GCP projects. The behaviour install driver runs `fullsend inference provision /` using CI credentials on the inference project (same access model as admin e2e), then passes the repo-scoped WIF provider to install. `E2E_GCP_WIF_PROVIDER` authenticates the CI job itself; it is not written to test-org repo secrets. The CI service account needs inference-provision IAM on `E2E_GCP_PROJECT_ID`: @@ -99,17 +99,13 @@ The CI service account needs inference-provision IAM on `E2E_GCP_PROJECT_ID`: | `roles/iam.workloadIdentityPoolAdmin` | Create/update repo-scoped inference WIF providers | | `roles/resourcemanager.projectIamAdmin` | Grant `roles/aiplatform.user` to repo WIF principals | -One-time enrollment for all pool orgs (idempotent). Enroll the singular admin `test-repo` (used by the driver today) and the behaviour pool `test-repo-01` … `test-repo-12` (pre-provisioned for planned parallelization): +One-time enrollment for admin e2e pool orgs (idempotent): ```bash export GCP_PROJECT=it-gcp-konflux-dev-fullsend for i in $(seq -w 1 12); do go run ./cmd/fullsend mint enroll "halfsend-${i}/test-repo" \ --project="$GCP_PROJECT" --region=us-central1 - for j in $(seq -w 1 12); do - go run ./cmd/fullsend mint enroll "halfsend-${i}/test-repo-${j}" \ - --project="$GCP_PROJECT" --region=us-central1 - done done ``` diff --git a/docs/guides/infrastructure/mint-administration.md b/docs/guides/infrastructure/mint-administration.md index 8c32848fb3..78a34fede1 100644 --- a/docs/guides/infrastructure/mint-administration.md +++ b/docs/guides/infrastructure/mint-administration.md @@ -76,7 +76,9 @@ The CLI defaults to this URL. You can also set the `FULLSEND_MINT_URL` repositor `roles/owner` covers all of the above for users with broad access. - **Behaviour / e2e pool orgs:** Enroll `halfsend-NN/test-repo` (admin e2e) and `halfsend-NN/test-repo-01` … `test-repo-12` (lazily created and installed on demand by the unified `install.Driver` — see [behaviour-testing.md](../dev/behaviour-testing.md#repo-allocation-via-unified-driver)) on the hosted mint (`PER_REPO_WIF_REPOS`). Run `fullsend mint enroll owner/repo` once per name — not from CI; do not enroll `*-fork` names. Repos need not exist at enrollment time — enroll is a mint allowlist / WIF-provider update only; the unified driver creates the repos when a behaviour scenario first leases them. See [e2e-testing.md](../dev/e2e-testing.md#behaviour-tests-and-per-repo-mint-enrollment). + **Admin e2e pool orgs:** Enroll `halfsend-NN/test-repo` on the hosted mint (`PER_REPO_WIF_REPOS`). Run `fullsend mint enroll owner/repo` once per name — not from CI; do not enroll `*-fork` names. Repos need not exist at enrollment time — enroll is a mint allowlist / WIF-provider update only. See [e2e-testing.md](../dev/e2e-testing.md#behaviour-tests-and-per-repo-mint-enrollment). + + **Behaviour tests:** Behaviour tests use the `fullsend-ai-test` org with ephemeral `bt-{uuid}-{slot}` repos. Per-repo mint enrollment for these dynamic names is handled via wildcard allowlisting during the CF preview mint deploy — no static enrollment of numbered repo names is needed. See [behaviour-testing.md](../dev/behaviour-testing.md#repo-allocation-via-unified-driver). An administrator can grant all required roles with a single script: diff --git a/docs/runtimes/pi.md b/docs/runtimes/pi.md index 34de380462..3d3206bc76 100644 --- a/docs/runtimes/pi.md +++ b/docs/runtimes/pi.md @@ -48,7 +48,7 @@ through fullsend's table, and a bare id gets the provider from `FULLSEND_PI_PROV > calls through the hook adapter, run-scoped provider deleted at the end, expired in place under > `--keep-sandbox`), plus the placeholder-generation experiments recorded in the ADR. The WIF path > has no live run yet; `features/runtime/pi-openai.feature` stays gated on `runtime-pi-openai` -> until an OpenAI organization is mapped to the pool repositories. +> until an OpenAI organization is mapped to the test org repositories. Harness `model:` and `agents:` entry `model:` values accept the `provider/id` form directly (`xai-vertex/xai/grok-4.6`); a harness can also select a provider with a bare `model:` plus diff --git a/e2e/behaviour/suite_test.go b/e2e/behaviour/suite_test.go index 037b4ddcd6..1dd7ceb790 100644 --- a/e2e/behaviour/suite_test.go +++ b/e2e/behaviour/suite_test.go @@ -9,7 +9,6 @@ import ( "testing" "github.com/cucumber/godog" - "github.com/google/uuid" "github.com/fullsend-ai/fullsend/pkg/behaviourtest/drivers/ci" gaci "github.com/fullsend-ai/fullsend/pkg/behaviourtest/drivers/ci/githubactions" @@ -35,22 +34,16 @@ func TestBehaviourSuite(t *testing.T) { } e2eCfg := e2etest.LoadEnvConfig(t) - ctx := context.Background() - runID := uuid.New().String() - org, token, err := e2etest.AcquireOrg(ctx, e2eCfg, runID, e2etest.OrgPool(), e2eCfg.LockTimeout, t.Logf) + org := e2etest.BehaviourTestOrg + token, err := e2etest.TokenForBehaviourOrg(e2eCfg) if err != nil { - t.Fatalf("acquiring org: %v", err) + t.Fatalf("getting token for %s: %v", org, err) } client := e2etest.NewLiveClient(token) - t.Cleanup(func() { - e2etest.ReleaseLock(context.Background(), client, org, runID, t) - }) binary := e2etest.BuildCLIBinary(t) - e2etest.CleanupStaleResources(ctx, client, token, org, t) - // Call the Factory to get the unified driver. The factory deploys // the preview mint and constructs all internal pieces (pool, ensurer). // Driver-specific config (PEMs, suite name, pool size) is read from diff --git a/pkg/behaviourtest/drivers/install/common/setup.go b/pkg/behaviourtest/drivers/install/common/setup.go index 2b77a7358a..6d8272a52e 100644 --- a/pkg/behaviourtest/drivers/install/common/setup.go +++ b/pkg/behaviourtest/drivers/install/common/setup.go @@ -3,6 +3,8 @@ package common import ( "encoding/json" "fmt" + "os" + "path/filepath" "strings" ) @@ -39,6 +41,59 @@ func RunGitHubSetup( return nil } +// RunReposInstall runs fullsend repos install with --fullsend-ref for +// ref-pinned installs. This replaces github setup --vendor for ephemeral +// repos that resolve the CLI binary at runtime via action.yml. +func RunReposInstall( + binary, token, target, fullsendRef, mintURL, gcpProjectID string, + runCLI CLIRunnerFunc, + logf func(string, ...any), +) error { + if fullsendRef == "" { + return fmt.Errorf("repos install %s: --fullsend-ref is required", target) + } + // Each install uses an isolated temp manifest so concurrent + // allocations don't race on a shared repos.yaml. + tmpManifest, err := os.CreateTemp("", "bt-manifest-*.yaml") + if err != nil { + return fmt.Errorf("repos install %s: creating temp manifest: %w", target, err) + } + if _, err := tmpManifest.WriteString("version: 1\n"); err != nil { + tmpManifest.Close() + return fmt.Errorf("repos install %s: writing temp manifest: %w", target, err) + } + manifestPath := tmpManifest.Name() + tmpManifest.Close() + defer os.Remove(manifestPath) + + args := []string{ + "repos", "install", target, + "--fullsend-ref", fullsendRef, + "--runtime", "dummy", + "--direct", + "--forge", "github", + "-f", filepath.Clean(manifestPath), + } + if project := strings.TrimSpace(gcpProjectID); project != "" { + // Provision the WIF provider before repos install runs — repos install + // auto-derives the WIF provider from --inference-project internally, + // so only --inference-project is passed (not --inference-wif-provider, + // which repos install does not accept). + if _, err := ProvisionInference(binary, token, target, project, runCLI, logf); err != nil { + return err + } + args = append(args, "--inference-project", project) + } + if mintURL != "" { + args = append(args, "--mint-url", mintURL) + } + logf("[install] running fullsend %s", strings.Join(args, " ")) + if _, err := runCLI(binary, token, args...); err != nil { + return fmt.Errorf("repos install %s: %w", target, err) + } + return nil +} + // ProvisionInference runs inference provision and returns the WIF provider // resource name. Mirrors the per-repo driver's provisionPerRepoInference. func ProvisionInference( diff --git a/pkg/behaviourtest/drivers/install/composed.go b/pkg/behaviourtest/drivers/install/composed.go index 9e8b58d290..2c36af20c4 100644 --- a/pkg/behaviourtest/drivers/install/composed.go +++ b/pkg/behaviourtest/drivers/install/composed.go @@ -4,8 +4,11 @@ import ( "context" "errors" "fmt" + "os" "sync" + "github.com/google/uuid" + "github.com/fullsend-ai/fullsend/internal/forge" ) @@ -18,8 +21,13 @@ type composedDriver struct { org string mint mintDriver ensurer ensurer + client forge.Client logf func(string, ...any) + // keepRepos, when true (E2E_KEEP_REPOS=true), preserves ephemeral + // repos after deallocation for post-mortem debugging. + keepRepos bool + // rate, when set, samples the shared installation token's primary // rate-limit budget on every allocation and release, so a suite // that later goes blind on 403s shows in its own log how the @@ -34,27 +42,35 @@ type composedDriver struct { } // newComposedDriver constructs a unified Driver from its constituent -// parts. It pre-fills the internal pool with repo names in the form -// "test-repo-01" … "test-repo-NN". The caller (Factory) is responsible -// for deploying the mint and creating the ensurer before calling this. +// parts. It pre-fills the internal pool with unique repo names in the +// form "bt-{uuid4}-{slot}" so concurrent CI runs in the same org never +// collide. The caller (Factory) is responsible for deploying the mint +// and creating the ensurer before calling this. func newComposedDriver( org string, mint mintDriver, ensurer ensurer, + client forge.Client, + prefix string, capacity int, logf func(string, ...any), ) (Driver, error) { if capacity <= 0 { return nil, fmt.Errorf("composed driver: capacity must be positive, got %d", capacity) } + if prefix == "" { + prefix = uuid.New().String()[:4] + } names := make(chan string, capacity) for i := 1; i <= capacity; i++ { - names <- fmt.Sprintf("test-repo-%02d", i) + names <- fmt.Sprintf("bt-%s-%02d", prefix, i) } return &composedDriver{ org: org, mint: mint, ensurer: ensurer, + client: client, + keepRepos: os.Getenv("E2E_KEEP_REPOS") == "true", logf: logf, names: names, capacity: capacity, @@ -94,18 +110,45 @@ func (d *composedDriver) AllocateRepo(ctx context.Context) (string, error) { } // DeallocateRepo returns a previously allocated repo to the pool. -// Errors on unknown name or double-release. -func (d *composedDriver) DeallocateRepo(_ context.Context, repoName string) error { +// Unless E2E_KEEP_REPOS=true, the repo is deleted from the forge so +// concurrent CI runs never collide on stale repos. Errors on unknown +// name or double-release. +func (d *composedDriver) DeallocateRepo(ctx context.Context, repoName string) error { d.mu.Lock() - defer d.mu.Unlock() if _, ok := d.outstanding[repoName]; !ok { + d.mu.Unlock() return fmt.Errorf("DeallocateRepo: %q is not an outstanding lease (possible double-release)", repoName) } delete(d.outstanding, repoName) - // Send inside the lock: the channel buffer equals capacity and this - // name was removed during AllocateRepo, so the send is guaranteed - // non-blocking. + d.mu.Unlock() + + // Delete the ephemeral repo BEFORE returning the slot to the pool. + // The channel send is what makes the slot available to other + // goroutines — delaying it until after deletion completes prevents + // a race where a new allocator begins EnsureRepo on a repo that is + // still being deleted. + if d.keepRepos { + d.logf("[driver] keeping %s/%s (E2E_KEEP_REPOS=true)", d.org, repoName) + } else if d.client != nil { + forkName := repoName + "-fork" + d.logf("[driver] deleting ephemeral fork %s/%s (if exists)", d.org, forkName) + if err := d.client.DeleteRepo(ctx, d.org, forkName); err != nil && !forge.IsNotFound(err) { + d.logf("[driver] warning: failed to delete fork %s/%s: %v", d.org, forkName, err) + } + d.logf("[driver] deleting ephemeral repo %s/%s", d.org, repoName) + if err := d.client.DeleteRepo(ctx, d.org, repoName); err != nil { + if !forge.IsNotFound(err) { + d.logf("[driver] warning: failed to delete %s/%s: %v", d.org, repoName, err) + } + } + } + // Invalidate the ensurer's cache so re-allocation of this slot + // triggers a fresh create+install cycle, even when keepRepos is + // true (the repo exists but needs a clean install). + d.ensurer.InvalidateCache(d.org, repoName) + + // Return the slot to the pool only after deletion is complete. d.names <- repoName d.logf("[driver] deallocated %s/%s", d.org, repoName) d.logRateLimit("after deallocating " + d.org + "/" + repoName) diff --git a/pkg/behaviourtest/drivers/install/composed_test.go b/pkg/behaviourtest/drivers/install/composed_test.go index b156b9a058..6c7a3dac38 100644 --- a/pkg/behaviourtest/drivers/install/composed_test.go +++ b/pkg/behaviourtest/drivers/install/composed_test.go @@ -32,14 +32,14 @@ func TestNewComposedDriver_OK(t *testing.T) { e := newFakeEnsurer() mint := &fakeMintDriver{} - d, err := newComposedDriver("org", mint, e, 3, t.Logf) + d, err := newComposedDriver("org", mint, e, nil, "", 3, t.Logf) require.NoError(t, err) require.NotNil(t, d) assert.Equal(t, 3, d.Capacity()) } func TestNewComposedDriver_InvalidCapacity(t *testing.T) { - _, err := newComposedDriver("org", nil, nil, 0, t.Logf) + _, err := newComposedDriver("org", nil, nil, nil, "", 0, t.Logf) require.Error(t, err) assert.Contains(t, err.Error(), "capacity must be positive") } @@ -48,7 +48,7 @@ func TestComposedDriver_AllocateAndDeallocate(t *testing.T) { e := newFakeEnsurer() mint := &fakeMintDriver{} - d, err := newComposedDriver("org", mint, e, 3, t.Logf) + d, err := newComposedDriver("org", mint, e, nil, "", 3, t.Logf) require.NoError(t, err) ctx := context.Background() @@ -56,7 +56,7 @@ func TestComposedDriver_AllocateAndDeallocate(t *testing.T) { // Allocate a repo. name, err := d.AllocateRepo(ctx) require.NoError(t, err) - assert.Contains(t, name, "test-repo-") + assert.Contains(t, name, "bt-") // Deallocate the repo. err = d.DeallocateRepo(ctx, name) @@ -72,7 +72,7 @@ func TestComposedDriver_DeallocateUnknownName(t *testing.T) { e := newFakeEnsurer() mint := &fakeMintDriver{} - d, err := newComposedDriver("org", mint, e, 2, t.Logf) + d, err := newComposedDriver("org", mint, e, nil, "", 2, t.Logf) require.NoError(t, err) err = d.DeallocateRepo(context.Background(), "unknown-repo") @@ -84,7 +84,7 @@ func TestComposedDriver_DoubleDeallocate(t *testing.T) { e := newFakeEnsurer() mint := &fakeMintDriver{} - d, err := newComposedDriver("org", mint, e, 2, t.Logf) + d, err := newComposedDriver("org", mint, e, nil, "", 2, t.Logf) require.NoError(t, err) ctx := context.Background() @@ -105,7 +105,7 @@ func TestComposedDriver_AllocateBlocksUntilDeallocate(t *testing.T) { e := newFakeEnsurer() mint := &fakeMintDriver{} - d, err := newComposedDriver("org", mint, e, 1, t.Logf) + d, err := newComposedDriver("org", mint, e, nil, "", 1, t.Logf) require.NoError(t, err) ctx := context.Background() @@ -138,7 +138,7 @@ func TestComposedDriver_AllocateEnsureError_ReturnsNameToPool(t *testing.T) { failEnsurer := &failingEnsurer{err: fmt.Errorf("ensure failed")} mint := &fakeMintDriver{} - d, err := newComposedDriver("org", mint, failEnsurer, 1, t.Logf) + d, err := newComposedDriver("org", mint, failEnsurer, nil, "", 1, t.Logf) require.NoError(t, err) ctx := context.Background() @@ -166,7 +166,7 @@ func TestComposedDriver_FinalizeNoOutstanding(t *testing.T) { mint := &fakeMintDriver{} e := newFakeEnsurer() - d, err := newComposedDriver("org", mint, e, 2, t.Logf) + d, err := newComposedDriver("org", mint, e, nil, "", 2, t.Logf) require.NoError(t, err) err = d.Finalize(context.Background()) @@ -178,7 +178,7 @@ func TestComposedDriver_FinalizeWithOutstanding(t *testing.T) { mint := &fakeMintDriver{} e := newFakeEnsurer() - d, err := newComposedDriver("org", mint, e, 2, t.Logf) + d, err := newComposedDriver("org", mint, e, nil, "", 2, t.Logf) require.NoError(t, err) ctx := context.Background() @@ -196,7 +196,7 @@ func TestComposedDriver_FinalizeJoinsErrors(t *testing.T) { mint := &fakeMintDriver{teardownErr: fmt.Errorf("teardown boom")} e := newFakeEnsurer() - d, err := newComposedDriver("org", mint, e, 2, t.Logf) + d, err := newComposedDriver("org", mint, e, nil, "", 2, t.Logf) require.NoError(t, err) ctx := context.Background() @@ -215,7 +215,7 @@ func TestComposedDriver_ConcurrentAllocateDeallocate(t *testing.T) { mint := &fakeMintDriver{} const poolSize = 4 - d, err := newComposedDriver("org", mint, e, poolSize, t.Logf) + d, err := newComposedDriver("org", mint, e, nil, "", poolSize, t.Logf) require.NoError(t, err) const goroutines = 8 @@ -257,6 +257,8 @@ func (f *failingEnsurer) EnsureRepo(_ context.Context, _, _ string) error { return f.err } +func (f *failingEnsurer) InvalidateCache(_, _ string) {} + // rateReportingClient is a forge.Client that also reports a fixed // rate-limit observation. type rateReportingClient struct { @@ -271,7 +273,7 @@ func TestComposedDriver_SamplesRateLimitOnAllocateAndDeallocate(t *testing.T) { var lines []string logf := func(format string, args ...any) { lines = append(lines, fmt.Sprintf(format, args...)) } e := newFakeEnsurer() - d, err := newComposedDriver("org", &fakeMintDriver{}, e, 2, logf) + d, err := newComposedDriver("org", &fakeMintDriver{}, e, nil, "", 2, logf) require.NoError(t, err) // A client without a reporter, or one that has observed nothing yet, samples nothing. @@ -291,3 +293,100 @@ func TestComposedDriver_SamplesRateLimitOnAllocateAndDeallocate(t *testing.T) { assert.Contains(t, lines, "[driver] rate limit after allocating org/"+name+": remaining=42/5000 reset=2026-01-01T00:00:00Z resource=core") assert.Contains(t, lines, "[driver] rate limit after deallocating org/"+name+": remaining=42/5000 reset=2026-01-01T00:00:00Z resource=core") } + +// fakeForgeClient is a test double for forge.Client that records +// DeleteRepo calls, exercising the DeallocateRepo deletion path. +type fakeForgeClient struct { + forge.Client // embed to satisfy interface + deleteCalls []string + deleteErr error +} + +func (f *fakeForgeClient) DeleteRepo(_ context.Context, org, repo string) error { + f.deleteCalls = append(f.deleteCalls, org+"/"+repo) + return f.deleteErr +} + +func TestComposedDriver_DeallocateDeletesRepo(t *testing.T) { + e := newFakeEnsurer() + mint := &fakeMintDriver{} + fc := &fakeForgeClient{} + + d, err := newComposedDriver("org", mint, e, fc, "", 2, t.Logf) + require.NoError(t, err) + + ctx := context.Background() + name, err := d.AllocateRepo(ctx) + require.NoError(t, err) + + err = d.DeallocateRepo(ctx, name) + require.NoError(t, err) + require.Len(t, fc.deleteCalls, 2, "should call DeleteRepo for fork and main repo") + assert.Equal(t, "org/"+name+"-fork", fc.deleteCalls[0], "fork deleted first") + assert.Equal(t, "org/"+name, fc.deleteCalls[1], "main repo deleted second") +} + +func TestComposedDriver_DeallocateKeepRepos(t *testing.T) { + t.Setenv("E2E_KEEP_REPOS", "true") + + e := newFakeEnsurer() + mint := &fakeMintDriver{} + fc := &fakeForgeClient{} + + d, err := newComposedDriver("org", mint, e, fc, "", 2, t.Logf) + require.NoError(t, err) + + ctx := context.Background() + name, err := d.AllocateRepo(ctx) + require.NoError(t, err) + + err = d.DeallocateRepo(ctx, name) + require.NoError(t, err) + assert.Empty(t, fc.deleteCalls, "should NOT call DeleteRepo when E2E_KEEP_REPOS=true") +} + +func TestComposedDriver_DeallocateDeleteNotFoundIgnored(t *testing.T) { + e := newFakeEnsurer() + mint := &fakeMintDriver{} + fc := &fakeForgeClient{deleteErr: forge.ErrNotFound} + + d, err := newComposedDriver("org", mint, e, fc, "", 2, t.Logf) + require.NoError(t, err) + + ctx := context.Background() + name, err := d.AllocateRepo(ctx) + require.NoError(t, err) + + // Should not return error for ErrNotFound. + err = d.DeallocateRepo(ctx, name) + require.NoError(t, err) +} + +func TestComposedDriver_DeallocateInvalidatesCache(t *testing.T) { + e := newFakeEnsurer() + mint := &fakeMintDriver{} + fc := &fakeForgeClient{} + + d, err := newComposedDriver("org", mint, e, fc, "", 2, t.Logf) + require.NoError(t, err) + + ctx := context.Background() + name, err := d.AllocateRepo(ctx) + require.NoError(t, err) + + // Verify that the ensurer cached the key. + key := "org/" + name + e.mu.Lock() + _, cached := e.cache[key] + e.mu.Unlock() + assert.True(t, cached, "ensurer should cache key after AllocateRepo") + + // Deallocate should invalidate the cache via the forge client path. + err = d.DeallocateRepo(ctx, name) + require.NoError(t, err) + + e.mu.Lock() + _, stillCached := e.cache[key] + e.mu.Unlock() + assert.False(t, stillCached, "ensurer cache should be invalidated after DeallocateRepo") +} diff --git a/pkg/behaviourtest/drivers/install/driver.go b/pkg/behaviourtest/drivers/install/driver.go index f62bf79ee9..454fcdd37d 100644 --- a/pkg/behaviourtest/drivers/install/driver.go +++ b/pkg/behaviourtest/drivers/install/driver.go @@ -84,7 +84,7 @@ const ( // PerRepoAgentArtifact is the upload-artifact name for triage output. PerRepoAgentArtifact = "fullsend-triage" - // DefaultPoolSize is the number of test-repo-NN repos in a pool org. - // Drivers use this as the default capacity when no override is set. + // DefaultPoolSize is the default number of concurrent ephemeral repo + // slots. Drivers use this as the default capacity when no override is set. DefaultPoolSize = 12 ) diff --git a/pkg/behaviourtest/drivers/install/ensure.go b/pkg/behaviourtest/drivers/install/ensure.go index 78d1b936db..a45d903b15 100644 --- a/pkg/behaviourtest/drivers/install/ensure.go +++ b/pkg/behaviourtest/drivers/install/ensure.go @@ -51,6 +51,11 @@ type ensurer interface { // post-install validation) it runs the per-repo install flow // (inference provision + github setup). EnsureRepo(ctx context.Context, org, repoName string) error + + // InvalidateCache removes the cached "ensured" entry for org/repoName + // so the next EnsureRepo call re-runs the create+install flow. + // Called by DeallocateRepo after deleting an ephemeral repo. + InvalidateCache(org, repoName string) } // SettleFunc is called after a repo is freshly created or installed to @@ -59,13 +64,14 @@ type ensurer interface { type SettleFunc func(ctx context.Context, client forge.Client, org, repo, workflowFile string, logf func(string, ...any)) error type repoEnsurer struct { - e2eCfg e2etest.EnvConfig - client forge.Client - token string - binary string - logf func(string, ...any) - runCLI CLIRunnerFunc // injectable; defaults to e2etest.TryRunCLI - settle SettleFunc // injectable; defaults to awaitWorkflowReady + e2eCfg e2etest.EnvConfig + client forge.Client + token string + binary string + fullsendRef string // when set, use repos install --fullsend-ref instead of github setup --vendor + logf func(string, ...any) + runCLI CLIRunnerFunc // injectable; defaults to e2etest.TryRunCLI + settle SettleFunc // injectable; defaults to awaitWorkflowReady mu sync.Mutex ensured map[string]struct{} // keyed by org/repo; only successful results cached @@ -93,6 +99,37 @@ func newRepoEnsurer( } } +// newRepoEnsurerWithRef returns an ensurer that uses repos install +// --fullsend-ref instead of github setup --vendor. The ref-pinned path +// avoids vendoring a 50 MB binary per ephemeral repo. +func newRepoEnsurerWithRef( + e2eCfg e2etest.EnvConfig, + client forge.Client, + token, binary, fullsendRef string, + logf func(string, ...any), +) ensurer { + return &repoEnsurer{ + e2eCfg: e2eCfg, + client: client, + token: token, + binary: binary, + fullsendRef: fullsendRef, + logf: logf, + runCLI: e2etest.TryRunCLI, + settle: awaitWorkflowReady, + ensured: make(map[string]struct{}), + } +} + +// InvalidateCache removes the cached "ensured" entry for org/repoName +// so the next EnsureRepo call re-runs the full create+install flow. +func (e *repoEnsurer) InvalidateCache(org, repoName string) { + key := org + "/" + repoName + e.mu.Lock() + delete(e.ensured, key) + e.mu.Unlock() +} + func (e *repoEnsurer) EnsureRepo(ctx context.Context, org, repoName string) error { key := org + "/" + repoName @@ -156,13 +193,19 @@ func (e *repoEnsurer) doEnsure(ctx context.Context, org, repoName string) error // install flow and settle for Actions readiness. e.logf("[ensure] %s needs install (fresh repo)", target) - // Step 4: run github setup --vendor to install fullsend and push - // the current binary. + // Step 4: install fullsend — repos install --fullsend-ref (ref-pinned) + // or github setup --vendor (vendored binary). if err := e.installFullsend(ctx, org, repoName, target); err != nil { return err } - if err := ValidatePerRepoPostInstall(ctx, e.client, org, repoName); err != nil { - return fmt.Errorf("post-install validation for %s: %w", target, err) + if e.fullsendRef != "" { + if err := ValidatePerRepoPostInstallRefPinned(ctx, e.client, org, repoName); err != nil { + return fmt.Errorf("post-install validation (ref-pinned) for %s: %w", target, err) + } + } else { + if err := ValidatePerRepoPostInstall(ctx, e.client, org, repoName); err != nil { + return fmt.Errorf("post-install validation for %s: %w", target, err) + } } // Step 5: wait for Actions to recognise the workflow file. @@ -324,8 +367,12 @@ func (e *repoEnsurer) awaitCreation(ctx context.Context, org, repoName, target s } // installFullsend runs inference provision (when a GCP project is -// configured) and fullsend github setup for the target repo. +// configured) and either fullsend repos install --fullsend-ref (when +// fullsendRef is set) or fullsend github setup --vendor for the target repo. func (e *repoEnsurer) installFullsend(_ context.Context, _, _, target string) error { + if e.fullsendRef != "" { + return common.RunReposInstall(e.binary, e.token, target, e.fullsendRef, e.e2eCfg.MintURL, e.e2eCfg.GCPProjectID, e.runCLI, e.logf) + } return common.RunGitHubSetup(e.binary, e.token, target, e.e2eCfg.MintURL, e.e2eCfg.GCPProjectID, e.runCLI, e.logf) } diff --git a/pkg/behaviourtest/drivers/install/ensure_test.go b/pkg/behaviourtest/drivers/install/ensure_test.go index ee86459894..49f066796a 100644 --- a/pkg/behaviourtest/drivers/install/ensure_test.go +++ b/pkg/behaviourtest/drivers/install/ensure_test.go @@ -49,6 +49,13 @@ func (f *fakeEnsurer) EnsureRepo(_ context.Context, org, repoName string) error return nil } +func (f *fakeEnsurer) InvalidateCache(org, repoName string) { + key := org + "/" + repoName + f.mu.Lock() + delete(f.cache, key) + f.mu.Unlock() +} + var _ ensurer = (*fakeEnsurer)(nil) func TestFakeEnsurer_Succeeds(t *testing.T) { @@ -399,6 +406,47 @@ func TestEnsurer_DoEnsure_WithGCPProject(t *testing.T) { assert.Contains(t, cliCalls[2], "--inference-wif-provider") } +func TestNewRepoEnsurerWithRef_ReturnsNonNil(t *testing.T) { + sc := &stubClient{} + e := newRepoEnsurerWithRef(e2etest.EnvConfig{}, sc, "tok", "/bin/true", "my-branch", t.Logf) + require.NotNil(t, e) + var _ ensurer = e +} + +func TestEnsurer_DoEnsure_RefPinned(t *testing.T) { + speedUpValidateRetries(t) + sc := &stubClient{ + getRepoErr: forge.ErrNotFound, + installed: false, + } + var cliCalls [][]string + e := &repoEnsurer{ + e2eCfg: e2etest.EnvConfig{MintURL: "https://mint.test"}, + client: sc, + binary: "/usr/bin/fullsend", + token: "tok", + fullsendRef: "feature-branch", + runCLI: func(binary, token string, args ...string) (string, error) { + cliCalls = append(cliCalls, args) + if len(args) >= 2 && args[0] == "repos" && args[1] == "install" { + sc.installed = true + } + return "", nil + }, + settle: noopSettle, + logf: t.Logf, + ensured: make(map[string]struct{}), + } + + err := e.EnsureRepo(context.Background(), "org", "test-repo-ref") + require.NoError(t, err) + require.Len(t, cliCalls, 1) + assert.Equal(t, "repos", cliCalls[0][0]) + assert.Equal(t, "install", cliCalls[0][1]) + assert.Contains(t, cliCalls[0], "--fullsend-ref") + assert.Contains(t, cliCalls[0], "feature-branch") +} + func TestEnsurer_InstallCLIError_Propagated(t *testing.T) { speedUpValidateRetries(t) sc := &stubClient{installed: false} diff --git a/pkg/behaviourtest/drivers/install/repopool_cfmint_previews.go b/pkg/behaviourtest/drivers/install/repopool_cfmint_previews.go index 35df6c9181..e4b6df7f98 100644 --- a/pkg/behaviourtest/drivers/install/repopool_cfmint_previews.go +++ b/pkg/behaviourtest/drivers/install/repopool_cfmint_previews.go @@ -11,6 +11,7 @@ package install import ( "context" "crypto/rand" + "encoding/json" "fmt" "os" "path/filepath" @@ -63,9 +64,9 @@ func NewRepoPoolCFMintPreviews( cfg := cfmintConfig{ pemDir: pemDir, suiteName: envSuiteName(), - allowedOrgs: "", // per-repo mode — no org-level allowlist - perRepoWIFRepos: buildRepoList(org, poolSize), - workflowHostRepos: buildRepoList(org, poolSize), + allowedOrgs: "", // per-repo mode — no org-level allowlist + perRepoWIFRepos: "*", // accept any repo; names are generated dynamically + workflowHostRepos: "*", appSet: envAppSet(), } @@ -102,10 +103,18 @@ func buildCFMintDriver( MintURL: mintURL, GCPProjectID: gcpProjectID, } - ens := newRepoEnsurer(e2eCfg, client, token, binary, logf) + + fullsendRef := envFullsendRef() + var ens ensurer + if fullsendRef != "" { + logf("[cfmint] using repos install --fullsend-ref %s", fullsendRef) + ens = newRepoEnsurerWithRef(e2eCfg, client, token, binary, fullsendRef, logf) + } else { + ens = newRepoEnsurer(e2eCfg, client, token, binary, logf) + } // Construct and return the composed driver. - d, err := newComposedDriver(org, md, ens, poolSize, logf) + d, err := newComposedDriver(org, md, ens, client, "", poolSize, logf) if err != nil { return nil, err } @@ -336,13 +345,52 @@ func envPoolSize(logf func(string, ...any)) int { return DefaultPoolSize } -// buildRepoList constructs a comma-separated list of org/test-repo-NN. -func buildRepoList(org string, poolSize int) string { - repos := make([]string, poolSize) - for i := range poolSize { - repos[i] = fmt.Sprintf("%s/test-repo-%02d", org, i+1) +// envFullsendRef returns the fullsend ref for repos install --fullsend-ref. +// Falls back through BEHAVIOUR_FULLSEND_REF → PR head SHA from event +// payload → GITHUB_HEAD_REF → GITHUB_REF_NAME. The event-payload +// fallback is needed because pull_request_target runs the base-branch +// workflow file, which may not set BEHAVIOUR_FULLSEND_REF. Branch +// names with slashes (e.g. "agent/xxx") are rejected by IsValidRef, +// so a SHA is preferred. +func envFullsendRef() string { + if v := os.Getenv("BEHAVIOUR_FULLSEND_REF"); v != "" { + return v + } + if sha := prHeadSHAFromEvent(); sha != "" { + return sha + } + if v := os.Getenv("GITHUB_HEAD_REF"); v != "" { + return v + } + if v := os.Getenv("GITHUB_REF_NAME"); v != "" { + return v + } + return "" +} + +// prHeadSHAFromEvent reads the PR head commit SHA from the GitHub +// Actions event payload (GITHUB_EVENT_PATH). Returns "" outside CI +// or for non-PR events. +func prHeadSHAFromEvent() string { + path := os.Getenv("GITHUB_EVENT_PATH") + if path == "" { + return "" + } + data, err := os.ReadFile(path) + if err != nil { + return "" + } + var event struct { + PullRequest struct { + Head struct { + SHA string `json:"sha"` + } `json:"head"` + } `json:"pull_request"` + } + if json.Unmarshal(data, &event) != nil { + return "" } - return strings.Join(repos, ",") + return event.PullRequest.Head.SHA } // --- PEM materialization (cfmint-specific) --- diff --git a/pkg/behaviourtest/drivers/install/repopool_cfmint_previews_test.go b/pkg/behaviourtest/drivers/install/repopool_cfmint_previews_test.go index 71e310a8b9..fa6ebfa34f 100644 --- a/pkg/behaviourtest/drivers/install/repopool_cfmint_previews_test.go +++ b/pkg/behaviourtest/drivers/install/repopool_cfmint_previews_test.go @@ -375,11 +375,6 @@ func TestCFMintTeardown_CLIFailure_ReturnsError(t *testing.T) { assert.Contains(t, err.Error(), "teardown boom") } -func TestBuildRepoList(t *testing.T) { - list := buildRepoList("my-org", 3) - assert.Equal(t, "my-org/test-repo-01,my-org/test-repo-02,my-org/test-repo-03", list) -} - func TestSetupCFMintPEMDir_NoPEMVars(t *testing.T) { // When no TEST_*_PEM env vars are set, returns ("", nil). dir, err := setupCFMintPEMDir() @@ -452,6 +447,47 @@ func TestEnvSuiteName_Override(t *testing.T) { assert.Equal(t, "custom-suite", envSuiteName()) } +func TestEnvFullsendRef_PrefersBehaviourEnv(t *testing.T) { + t.Setenv("BEHAVIOUR_FULLSEND_REF", "abc123") + t.Setenv("GITHUB_HEAD_REF", "feature/branch") + t.Setenv("GITHUB_REF_NAME", "main") + assert.Equal(t, "abc123", envFullsendRef()) +} + +func TestEnvFullsendRef_FallsBackToEventPayload(t *testing.T) { + eventFile := filepath.Join(t.TempDir(), "event.json") + os.WriteFile(eventFile, []byte(`{"pull_request":{"head":{"sha":"deadbeef123"}}}`), 0o644) + t.Setenv("BEHAVIOUR_FULLSEND_REF", "") + t.Setenv("GITHUB_EVENT_PATH", eventFile) + t.Setenv("GITHUB_HEAD_REF", "agent/slash-branch") + t.Setenv("GITHUB_REF_NAME", "main") + assert.Equal(t, "deadbeef123", envFullsendRef()) +} + +func TestEnvFullsendRef_FallsBackToHeadRef(t *testing.T) { + t.Setenv("BEHAVIOUR_FULLSEND_REF", "") + t.Setenv("GITHUB_EVENT_PATH", "") + t.Setenv("GITHUB_HEAD_REF", "feature/branch") + t.Setenv("GITHUB_REF_NAME", "main") + assert.Equal(t, "feature/branch", envFullsendRef()) +} + +func TestEnvFullsendRef_FallsBackToRefName(t *testing.T) { + t.Setenv("BEHAVIOUR_FULLSEND_REF", "") + t.Setenv("GITHUB_EVENT_PATH", "") + t.Setenv("GITHUB_HEAD_REF", "") + t.Setenv("GITHUB_REF_NAME", "main") + assert.Equal(t, "main", envFullsendRef()) +} + +func TestEnvFullsendRef_EmptyWhenNoneSet(t *testing.T) { + t.Setenv("BEHAVIOUR_FULLSEND_REF", "") + t.Setenv("GITHUB_EVENT_PATH", "") + t.Setenv("GITHUB_HEAD_REF", "") + t.Setenv("GITHUB_REF_NAME", "") + assert.Equal(t, "", envFullsendRef()) +} + func TestEnvAppSet_Default(t *testing.T) { t.Setenv("BEHAVIOUR_APP_SET", "") assert.Equal(t, "fullsend-test", envAppSet()) diff --git a/pkg/behaviourtest/drivers/install/repopool_external_mint.go b/pkg/behaviourtest/drivers/install/repopool_external_mint.go index bb0d925e28..f4cce060da 100644 --- a/pkg/behaviourtest/drivers/install/repopool_external_mint.go +++ b/pkg/behaviourtest/drivers/install/repopool_external_mint.go @@ -43,8 +43,17 @@ func NewRepoPoolExternalMint( MintURL: mintURL, GCPProjectID: gcpProjectID, } - ens := newRepoEnsurer(ensCfg, client, token, binary, logf) - d, err := newComposedDriver(org, md, ens, poolSize, logf) + + fullsendRef := envFullsendRef() + var ens ensurer + if fullsendRef != "" { + logf("[external-mint] using repos install --fullsend-ref %s", fullsendRef) + ens = newRepoEnsurerWithRef(ensCfg, client, token, binary, fullsendRef, logf) + } else { + ens = newRepoEnsurer(ensCfg, client, token, binary, logf) + } + + d, err := newComposedDriver(org, md, ens, client, "", poolSize, logf) if err != nil { return nil, err } diff --git a/pkg/behaviourtest/drivers/install/validate.go b/pkg/behaviourtest/drivers/install/validate.go index 1826ed78f4..7b46dc6607 100644 --- a/pkg/behaviourtest/drivers/install/validate.go +++ b/pkg/behaviourtest/drivers/install/validate.go @@ -3,7 +3,7 @@ package install import ( "context" "fmt" - "path/filepath" + "path" "time" "github.com/fullsend-ai/fullsend/internal/config" @@ -47,6 +47,39 @@ func getFileWithRetry(ctx context.Context, client forge.Client, org, repo, path return nil, lastErr } +// ValidatePerRepoPostInstallRefPinned checks that a ref-pinned install +// (repos install --fullsend-ref) left the expected files and configuration +// in the target repo. Unlike the vendored validation, it does not check +// for the vendored binary or marker — ref-pinned installs resolve the +// binary at runtime via action.yml. +func ValidatePerRepoPostInstallRefPinned(ctx context.Context, client forge.Client, org, repo string) error { + shimPath := ".github/workflows/fullsend.yaml" + if _, err := getFileWithRetry(ctx, client, org, repo, shimPath); err != nil { + return fmt.Errorf("post-install: missing %s on %s/%s: %w", shimPath, org, repo, err) + } + + cfgPath := path.Join(".fullsend", "config.yaml") + cfgData, err := getFileWithRetry(ctx, client, org, repo, cfgPath) + if err != nil { + return fmt.Errorf("post-install: reading %s: %w", cfgPath, err) + } + cfgW, err := config.ParsePerRepoConfigWriter(cfgData) + if err != nil { + return fmt.Errorf("post-install: parsing %s: %w", cfgPath, err) + } + if err := cfgW.Validate(); err != nil { + return fmt.Errorf("post-install: invalid %s: %w", cfgPath, err) + } + cfg, ok := cfgW.(config.PerRepoConfigReader) + if !ok { + return fmt.Errorf("post-install: %s config does not implement PerRepoConfigReader", cfgPath) + } + if cfg.ConfigRuntime() != "dummy" { + return fmt.Errorf("post-install: %s runtime is %q, want dummy", cfgPath, cfg.ConfigRuntime()) + } + return nil +} + // ValidatePerRepoPostInstall checks that a per-repo install left the // expected files and configuration in the target repo. func ValidatePerRepoPostInstall(ctx context.Context, client forge.Client, org, repo string) error { @@ -55,7 +88,7 @@ func ValidatePerRepoPostInstall(ctx context.Context, client forge.Client, org, r return fmt.Errorf("post-install: missing %s on %s/%s: %w", shimPath, org, repo, err) } - cfgPath := filepath.Join(".fullsend", "config.yaml") + cfgPath := path.Join(".fullsend", "config.yaml") cfgData, err := getFileWithRetry(ctx, client, org, repo, cfgPath) if err != nil { return fmt.Errorf("post-install: reading %s: %w", cfgPath, err) diff --git a/pkg/behaviourtest/drivers/install/validate_test.go b/pkg/behaviourtest/drivers/install/validate_test.go index 490771f108..4e4935b3c8 100644 --- a/pkg/behaviourtest/drivers/install/validate_test.go +++ b/pkg/behaviourtest/drivers/install/validate_test.go @@ -83,6 +83,47 @@ func TestValidatePerRepoPostInstall_WrongRuntime(t *testing.T) { assert.Contains(t, err.Error(), "want dummy") } +func TestValidatePerRepoPostInstallRefPinned_OK(t *testing.T) { + client := forge.NewFakeClient() + org, repo := "acme", "test-repo" + perRepoCfg := config.NewPerRepoConfig(config.PerRepoDefaultRoles(), org+"/"+repo) + perRepoCfg.SetRuntime("dummy") + cfg, err := perRepoCfg.Marshal() + require.NoError(t, err) + + client.FileContents = map[string][]byte{ + org + "/" + repo + "/.github/workflows/fullsend.yaml": []byte("name: fullsend"), + org + "/" + repo + "/.fullsend/config.yaml": cfg, + } + + err = ValidatePerRepoPostInstallRefPinned(context.Background(), client, org, repo) + require.NoError(t, err) +} + +func TestValidatePerRepoPostInstallRefPinned_MissingShim(t *testing.T) { + speedUpValidateRetries(t) + client := forge.NewFakeClient() + err := ValidatePerRepoPostInstallRefPinned(context.Background(), client, "acme", "test-repo") + require.Error(t, err) + assert.Contains(t, err.Error(), "fullsend.yaml") +} + +func TestValidatePerRepoPostInstallRefPinned_WrongRuntime(t *testing.T) { + client := forge.NewFakeClient() + org, repo := "acme", "test-repo" + cfg, err := config.NewPerRepoConfig(nil, org+"/"+repo).Marshal() + require.NoError(t, err) + + client.FileContents = map[string][]byte{ + org + "/" + repo + "/.github/workflows/fullsend.yaml": []byte("name: fullsend"), + org + "/" + repo + "/.fullsend/config.yaml": cfg, + } + + err = ValidatePerRepoPostInstallRefPinned(context.Background(), client, org, repo) + require.Error(t, err) + assert.Contains(t, err.Error(), "want dummy") +} + func TestParseInferenceStatusWIFProvider_OK(t *testing.T) { out := `{ "status": "healthy", diff --git a/pkg/e2etest/auth.go b/pkg/e2etest/auth.go index 5184158fa5..353fb123aa 100644 --- a/pkg/e2etest/auth.go +++ b/pkg/e2etest/auth.go @@ -115,6 +115,13 @@ func resolveE2EToken(ctx context.Context, mintURL, targetOrg string) (string, er return result.Token, nil } +// TokenForBehaviourOrg returns an API token for the dedicated behaviour +// test org (fullsend-ai-test). It uses the same token resolution logic +// as pool org acquisition. +func TokenForBehaviourOrg(cfg EnvConfig) (string, error) { + return tokenForOrg(context.Background(), cfg.internal(), BehaviourTestOrg) +} + // tokenForOrg returns an API token for operating on a pool org. func tokenForOrg(ctx context.Context, cfg envConfig, org string) (string, error) { if cfg.useMint { diff --git a/pkg/e2etest/auth_test.go b/pkg/e2etest/auth_test.go index 92974bb534..8543c736fc 100644 --- a/pkg/e2etest/auth_test.go +++ b/pkg/e2etest/auth_test.go @@ -150,3 +150,13 @@ func TestTokenForOrg_WithoutMint(t *testing.T) { require.NoError(t, err) assert.Equal(t, "local-token", token) } + +func TestTokenForBehaviourOrg_UsesLocalToken(t *testing.T) { + t.Setenv("GH_TOKEN", "behaviour-token") + t.Setenv("GITHUB_TOKEN", "") + + cfg := EnvConfig{UseMint: false} + token, err := TokenForBehaviourOrg(cfg) + require.NoError(t, err) + assert.Equal(t, "behaviour-token", token) +} diff --git a/pkg/e2etest/testutil.go b/pkg/e2etest/testutil.go index d03f3c9429..2941ffaf75 100644 --- a/pkg/e2etest/testutil.go +++ b/pkg/e2etest/testutil.go @@ -535,6 +535,10 @@ func AcquireOrg(ctx context.Context, cfg EnvConfig, runID string, pool []string, return acquireOrg(ctx, cfg.internal(), runID, pool, timeout, logf) } +// BehaviourTestOrg is the single org used for behaviour tests with +// ephemeral per-scenario repos. +const BehaviourTestOrg = "fullsend-ai-test" + // OrgPool returns the halfsend org names used for parallel e2e runs. func OrgPool() []string { return orgPool