From 920adee68da2a5d6325b56609ed499c79346db68 Mon Sep 17 00:00:00 2001 From: tianyao Date: Thu, 27 Aug 2026 06:42:49 +0000 Subject: [PATCH 1/4] test(e2e): run Gitea safely across local and CI --- .github/workflows/ci.yml | 145 ++++++++++++++++-------------- docs/testing/real-provider-e2e.md | 90 +++++++++---------- scripts/e2e-harness.sh | 36 +++----- scripts/run-e2e.sh | 14 ++- tests/ci-workflow.test.ts | 35 ++++++++ 5 files changed, 184 insertions(+), 136 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bd6a6e8..e24590f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,31 +57,70 @@ jobs: - 'package-lock.json' - '.github/workflows/ci.yml' - # Real-provider E2E: one matrix job covering GitHub, GitLab, and Gitea (see - # docs/testing/real-provider-e2e.md). + # Gitea is secretless and disposable, so it runs on a fresh GitHub-hosted + # VM. This is the only E2E job allowed to execute fork PR code; untrusted + # code must never reach the privileged self-hosted fleet below. + gitea-e2e: + name: E2E / gitea + needs: changes + runs-on: ubuntu-latest + permissions: + contents: read + timeout-minutes: 20 + if: >- + (needs.changes.outputs.e2e-relevant == 'true' || + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.ref == 'refs/heads/main') && + (github.event_name != 'workflow_dispatch' || + github.event.inputs.provider == 'all' || + github.event.inputs.provider == 'gitea') + concurrency: + group: e2e-${{ github.head_ref || github.ref_name }}-gitea + cancel-in-progress: true + env: + E2E_KEEP_BRANCH: ${{ github.event.inputs.keep_branch }} + E2E_PR_NUMBER: ${{ github.event.pull_request.number }} + E2E_SOURCE_BRANCH: ${{ github.head_ref || github.ref_name }} + steps: + - name: Compute run-scoped workdir + run: echo "E2E_WORKDIR=$RUNNER_TEMP/git-files-sync-e2e/${{ github.run_id }}/${{ github.run_attempt }}/gitea" >> "$GITHUB_ENV" + + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + + - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 + with: + node-version: '22' + cache: npm + + - run: npm ci --ignore-scripts + + - name: Run disposable Gitea E2E + run: scripts/run-e2e.sh --provider gitea + + # Credentialed GitHub/GitLab sandboxes remain on the self-hosted fleet, but + # only trusted pushes, schedules, dispatches, and same-repository PRs may + # reach it. GitHub recommends against running fork code on self-hosted hosts. provider-e2e: name: E2E / ${{ matrix.provider }} needs: changes runs-on: [self-hosted, linux, x64, 32gb-ram] - # Runs when sync/provider-relevant paths changed, or unconditionally on - # workflow_dispatch/schedule/a push to main (main always gets the full - # tier regardless of path, per the issue's CI wiring). The per-provider - # part of the gating (internal PRs/main/dispatch/schedule get every - # provider; a fork PR only gets Gitea) can't live here: job-level `if:` - # has no access to the `matrix` context (GitHub Actions error - # "Unrecognized named-value: 'matrix'" if you try) -- only step-level - # `if:` can see it. That part is done by the "Determine whether this - # provider leg should run" step below instead, gating every later step. if: >- - needs.changes.outputs.e2e-relevant == 'true' || + (needs.changes.outputs.e2e-relevant == 'true' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || - github.ref == 'refs/heads/main' + github.ref == 'refs/heads/main') && + (github.event_name != 'pull_request' || + github.event.pull_request.head.repo.full_name == github.repository) && + (github.event_name != 'workflow_dispatch' || + github.event.inputs.provider == 'all' || + github.event.inputs.provider == 'github' || + github.event.inputs.provider == 'gitlab') strategy: fail-fast: false - max-parallel: 3 + max-parallel: 2 matrix: - provider: [github, gitlab, gitea] + provider: [github, gitlab] # One group per source branch/provider -- keyed by branch name alone # (github.head_ref || github.ref_name, same expression E2E_SOURCE_BRANCH # below uses), deliberately NOT split by event type. A `push` to a branch @@ -135,42 +174,12 @@ jobs: - name: Compute run-scoped workdir run: echo "E2E_WORKDIR=$RUNNER_TEMP/git-files-sync-e2e/${{ github.run_id }}/${{ github.run_attempt }}/${{ matrix.provider }}" >> "$GITHUB_ENV" - # Per-provider gate (needs `matrix`, so it runs as a step, not the job-level - # `if:` above -- see the comment on that `if:` for why). A fork PR (head repo - # != base repo) only gets Gitea, which needs no repository secrets and can - # safely run against an untrusted fork's code; GitHub/GitLab need real sandbox - # credentials that must never be exposed to a fork PR's workflow run. All - # other events/providers run. + # Matrix context is unavailable in the job-level `if`, so a targeted + # manual dispatch is selected here. Every non-dispatch event runs both. - name: Determine whether this provider leg should run id: gate run: | run=true - # TODO(e2e): gitea temporarily disabled in CI -- container - # provisioning against this runner fleet's Docker topology needs - # more investigation (bridge-IP reachability, health-check timing) - # than is safe to iterate on inside the shared matrix. Suite/harness - # code is untouched and passes locally (`npm run test:e2e -- - # --provider gitea`); re-enable by deleting this block once the CI - # runner behavior is confirmed. NOTE: gitea is also what normally - # covers fork PRs (no secrets needed) -- while this is disabled, - # fork PRs get no E2E coverage at all. - if [ "${{ matrix.provider }}" = "gitea" ]; then - run=false - # Make the disabled state explicit in the run log + summary so a - # green "E2E / gitea" job is never mistaken for "Gitea E2E passed". - echo "::notice::Gitea E2E is disabled in CI (runner Docker networking — see TODO below). Suite/harness code passes locally; re-enable by removing this block." - { - echo "### Gitea E2E: disabled" - echo "Reason: runner Docker networking — container provisioning against this runner fleet needs investigation (bridge-IP reachability, health-check timing)." - echo "Suite/harness code is untouched and passes locally (\`npm run test:e2e -- --provider gitea\`). The Gitea infrastructure fix is tracked separately; do not infer three-provider coverage from a green gitea leg." - echo "Re-enable by removing the gitea block in the \"Determine whether this provider leg should run\" step." - } >> "$GITHUB_STEP_SUMMARY" - fi - if [ "${{ github.event_name }}" = "pull_request" ] \ - && [ "${{ matrix.provider }}" != "gitea" ] \ - && [ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" ]; then - run=false - fi if [ "${{ github.event_name }}" = "workflow_dispatch" ] \ && [ "${{ github.event.inputs.provider }}" != "all" ] \ && [ "${{ github.event.inputs.provider }}" != "${{ matrix.provider }}" ]; then @@ -236,40 +245,40 @@ jobs: E2E_PROVIDER: ${{ matrix.provider }} run: scripts/e2e-harness.sh cleanup - # Aggregates the matrix into a single required status so branch protection - # only has to reference one check name (see docs/testing/real-provider-e2e.md - # for the "Gitea required, GitHub/GitLab not required at branch-protection - # level" split -- required-vs-optional per *provider* still comes from the - # "Determine whether this provider leg should run" step above; this gate - # only asks "did whatever ran, pass?"). A gated-off leg's steps are all - # skipped without failing the job, so it still reports "success" here. - # `if: always()` so a real provider-e2e failure is caught here and blocks - # CI/release. A cancelled matrix means a newer run in the same branch/provider - # concurrency group replaced this duplicate run; report that as neutral and - # do not start another copy of downstream CI. + # Aggregate both trust domains before downstream CI/release. A path- or + # dispatch-filtered job may be skipped; a superseded job may be cancelled. e2e-gate: name: E2E gate - needs: provider-e2e + needs: [gitea-e2e, provider-e2e] if: always() runs-on: ubuntu-latest outputs: run-ci: ${{ steps.check.outputs.run-ci }} steps: - - name: Check provider-e2e result + - name: Check E2E results id: check run: | - result="${{ needs.provider-e2e.result }}" - echo "provider-e2e result: $result" + gitea_result="${{ needs.gitea-e2e.result }}" + provider_result="${{ needs.provider-e2e.result }}" + echo "gitea-e2e result: $gitea_result" + echo "provider-e2e result: $provider_result" echo "run-ci=true" >> "$GITHUB_OUTPUT" - if [ "$result" = "cancelled" ]; then + replaced=false + for result in "$gitea_result" "$provider_result"; do + if [ "$result" = "cancelled" ]; then + replaced=true + continue + fi + if [ "$result" != "success" ] && [ "$result" != "skipped" ]; then + echo "::error::E2E dependency failed ($result) -- blocking CI/release." + exit 1 + fi + done + if [ "$replaced" = "true" ]; then echo "run-ci=false" >> "$GITHUB_OUTPUT" - echo "::notice::provider-e2e was replaced by a newer run in the same concurrency group." + echo "::notice::E2E was replaced by a newer run in the same concurrency group." exit 0 fi - if [ "$result" != "success" ] && [ "$result" != "skipped" ]; then - echo "::error::provider-e2e failed ($result) -- blocking CI/release." - exit 1 - fi CI: needs: e2e-gate diff --git a/docs/testing/real-provider-e2e.md b/docs/testing/real-provider-e2e.md index 4e7ff78..ce50bdf 100644 --- a/docs/testing/real-provider-e2e.md +++ b/docs/testing/real-provider-e2e.md @@ -56,8 +56,9 @@ uses those APIs at all: `e2e-branch-cleanup.yml`, never by the normal per-run job. - `scripts/e2e-janitor.sh` — layer 3: TTL-based sweep of any leftover `e2e/**` branch, run by `.github/workflows/e2e-janitor.yml` on a schedule. -- `scripts/run-e2e.sh` — thin local-dev wrapper: provision → seed → vitest → cleanup (CI drives - the same four steps directly as separate job steps instead). +- `scripts/run-e2e.sh` — the shared local/CI entry point: provision → seed → vitest → cleanup. + It allocates a unique temporary workdir when the caller does not supply one, so concurrent local + runs cannot overwrite each other's repository, runtime adapters, or credentials. - `e2e/config/env.ts` — reads the env vars `provision` resolved and constructs the real, already-configured `GitServiceInterface` per provider (`githubContext`/`gitlabContext`/ `giteaContext`). @@ -145,8 +146,9 @@ flowchart TD E --> J["Layer 3: e2e-janitor.yml (scheduled)\ndelete any e2e/** branch older than TTL"] ``` -1. **Layer 1 — current-run cleanup** (`scripts/e2e-harness.sh cleanup`, `if: always()` in - `ci.yml`): deletes only the one branch this run itself created, with generic +1. **Layer 1 — current-run cleanup** (`scripts/e2e-harness.sh cleanup`; the shared wrapper uses an + `EXIT` trap and the credentialed CI job also has an `if: always()` fallback): deletes only the + one branch this run itself created, with generic `git push origin --delete`. `E2E_KEEP_BRANCH=1` deliberately skips this for debugging. Never a wildcard, never touches another run's branch. 2. **Layer 2 — PR/branch lifecycle cleanup** (`.github/workflows/e2e-pr-cleanup.yml`, @@ -170,7 +172,7 @@ flowchart TD The design goal is **not** "the sandbox repos are always perfectly clean" — it's that old garbage, however it got there, can never contaminate a current run's state. -### Self-hosted runner workspace isolation +### Workspace isolation `provider-e2e` runs on a persistent self-hosted fleet, so `E2E_WORKDIR` is pinned per run/attempt/provider rather than relying on a fresh filesystem or a shared `/tmp` path: @@ -179,11 +181,11 @@ run/attempt/provider rather than relying on a fresh filesystem or a shared `/tmp $RUNNER_TEMP/git-files-sync-e2e//// ``` -set once at job level in `ci.yml` (`env.E2E_WORKDIR`) so every step in the job shares it, and a +set once near the start of each E2E job so every later process shares it, and a previous killed job's leftover files under a different run-id/attempt can never leak into the -current one. Locally, `scripts/e2e-harness.sh` falls back to a provider-namespaced (not random) -tmp dir so sequential `npm run test:e2e` invocations in the same shell session still share state -across its own provision/seed/vitest/cleanup steps. +current one. Locally, `scripts/run-e2e.sh` uses `mktemp` to allocate a unique workdir per invocation +and removes it after cleanup. `E2E_KEEP_BRANCH=1` deliberately preserves both the container/branch +and workdir for debugging. ## Running locally @@ -204,9 +206,11 @@ npm run test:e2e -- --provider gitlab # needs E2E_GITLAB_* below | `E2E_GITLAB_BASE_URL` | gitlab (optional) | defaults to `https://gitlab.com` | | `E2E_GITEA_IMAGE` | gitea (optional) | defaults to `gitea/gitea:1.22` | | `E2E_KEEP_BRANCH` | any (optional) | `1`/`true` skips teardown (branch for GitHub/GitLab, container for Gitea) so you can inspect a failing run | -| `E2E_WORKDIR` | any (optional) | shared scratch dir across provision/seed/vitest/cleanup; defaults to a provider-namespaced tmp dir | +| `E2E_WORKDIR` | any (optional) | shared scratch dir across provision/seed/vitest/cleanup; the wrapper defaults to a unique temporary directory | -Gitea needs Docker locally and nothing else. +Gitea needs Docker locally and nothing else. Its disposable container publishes port 3000 on a +Docker-assigned `127.0.0.1` port, so it never depends on the host's bridge subnet and parallel runs +do not contend for a fixed port. ### Git authentication @@ -221,15 +225,20 @@ source of truth after its container is created, so it's the one credential persi ## CI -`.github/workflows/ci.yml` runs a `provider-e2e` matrix job (`github`, `gitlab`, `gitea`) as five -steps per leg — provision, seed, the real vitest run, independent verify, cleanup (`if: always()` -so cleanup runs even if an earlier step failed) — gated on relevant paths (`src/services/**`, -`src/logic/sync-manager.ts`, `e2e/**`, `scripts/e2e-harness.sh`, `scripts/e2e-namespace.sh`, etc. — -computed by the `changes` job, since GitHub Actions' own `on.*.paths` would gate the *entire* -workflow file, including the always-must-run `CI`/release job). It always runs in full on -`workflow_dispatch`, `schedule` (weekly, Monday 06:00 UTC, for API-drift detection), and pushes to -`main`. The job carries a per-source/provider `concurrency` group (see "Isolation model" above) and -sets `E2E_WORKDIR`/`E2E_PR_NUMBER`/`E2E_SOURCE_BRANCH` once at job level, shared by every step. +`.github/workflows/ci.yml` separates E2E by trust boundary: + +- `gitea-e2e` runs the disposable, secretless Gitea sandbox on a fresh `ubuntu-latest` VM. It is + safe for fork PRs because it receives only `contents: read`, no repository secrets, and no access + to the persistent self-hosted fleet. The job invokes the same `scripts/run-e2e.sh --provider + gitea` command used locally. +- `provider-e2e` is the credentialed `github`/`gitlab` matrix on the self-hosted fleet. Its + job-level condition rejects fork PRs before a runner is allocated; a step-level gate handles + provider-specific manual dispatch because the matrix context is unavailable in a job-level + condition. + +Both paths are gated on relevant files computed by `changes`; both run for `main`, the weekly API +drift schedule, and applicable manual dispatches. Each path has a per-source/provider concurrency +group, and each uses a run/attempt/provider-scoped `E2E_WORKDIR`. Two more workflows round out the isolation model's other cleanup layers — see "Isolation model" above for what each does and why: @@ -248,28 +257,24 @@ above for what each does and why: | `E2E_GITLAB_PROJECT_ID` | secret (not a variable — it's treated as sensitive here) | | `E2E_GITLAB_TOKEN` | secret | -**Fork PRs** only run the Gitea cell (checked in the `Determine whether this provider leg should -run` step — GitHub Actions job-level `if:` can't reference the `matrix` context, so this can't -live on the job itself; it gates every later step instead) — GitHub/GitLab need real credentials -that must never be exposed to an untrusted fork's workflow run. Gitea needs no repo secrets at -all, so it's safe to run unconditionally. +**Fork PRs** only run `gitea-e2e` on `ubuntu-latest`. The credentialed GitHub/GitLab job is rejected +at job level, so untrusted code is never scheduled on the privileged self-hosted runner fleet. -**Missing credentials are always a hard failure**, never a silent skip, for any cell that -actually runs (`scripts/e2e-harness.sh`'s `normalize_env`/`: "${VAR:?...}"` checks required env -vars up front) — the job-level `if:` above is what decides whether a cell *should* run for a -given event; once it runs, it's expected to have what it needs. +**Missing credentials are always a hard failure**, never a silent skip, for a GitHub/GitLab cell +that runs (`scripts/e2e-harness.sh` checks required environment variables up front). Gitea creates +its own per-run credentials and receives no repository secrets. ## Release gating ``` -changes -> provider-e2e [github | gitlab | gitea, parallel] -> e2e-gate -> CI (shared workflow, includes semantic-release) +changes -> gitea-e2e [GitHub-hosted] ---------\ + -> e2e-gate -> CI (shared workflow, includes semantic-release) +changes -> provider-e2e [github | gitlab] -----/ ``` -`e2e-gate` runs with `if: always()` and treats `provider-e2e`'s aggregate result as pass-through -on `success` or `skipped` (the latter covers path-filtered-out runs), a neutral replacement on -`cancelled` (with downstream CI suppressed for that duplicate run), and a hard failure on any -other result. A real provider regression therefore still blocks the release instead of shipping -and being caught after the fact. +`e2e-gate` runs with `if: always()` and evaluates both dependencies. `success` and `skipped` pass; +`cancelled` means a newer run replaced this one and suppresses duplicate downstream CI; any other +result blocks CI/release. **Branch protection** (not something this repo checkout can change — a GitHub repo-settings change, left for whoever has admin access): add `E2E / gitea` as a required status check. @@ -290,22 +295,17 @@ structurally cannot produce. - **Inspecting a failing run**: set `E2E_KEEP_BRANCH=1` before running so teardown is skipped, then look at the branch/container directly. Remember to clean it up yourself afterward (see above) — or just let the janitor catch it within its TTL. -- **Gitea container port/name clashes**: each run's container is named `gfs-e2e-gitea-$$` (PID) - and binds to a Docker-assigned host port, so concurrent local runs don't collide; a leftover - container from an interrupted run can be removed manually (`docker rm -f `). +- **Gitea container port/name clashes**: each name includes run ID, attempt, and PID, while Docker + assigns its loopback host port. Concurrent runs also use separate `mktemp` workdirs. A container + left by a hard-killed local process can be removed manually (`docker rm -f `). - **`E2E_PROVIDER is not set` error**: `vitest.e2e.config.ts` refuses to run directly under `npx vitest` — always go through `npm run test:e2e -- --provider ` (or the CI steps), which set it. ## Known gaps -- SyncManager E2E against GitHub/GitLab uses the same harness as Gitea (no provider-specific - code) but has only been exercised end-to-end locally against Gitea (Docker, no external - credentials available in this environment) — not yet actually executed against live - GitHub/GitLab sandboxes from this checkout. -- The `provider-e2e` matrix job targets `runs-on: [self-hosted, linux, x64, 32gb-ram]`; its - actual execution on that fleet, and the `e2e-gate` -> `CI` dependency chain end-to-end in a - real workflow run, are unverified from this checkout (no self-hosted runner access here). +- The new GitHub-hosted `gitea-e2e` job and two-input `e2e-gate` must be confirmed by a real + workflow run before issue #139 is complete; local runs cannot prove GitHub runner behavior. - Branch-protection required-check configuration (`E2E / gitea`) is a manual follow-up for whoever has admin access to the repo. - The official Obsidian community-plugin scanner rescan (as opposed to this repo's own diff --git a/scripts/e2e-harness.sh b/scripts/e2e-harness.sh index c19ca5c..1f6cad3 100755 --- a/scripts/e2e-harness.sh +++ b/scripts/e2e-harness.sh @@ -394,41 +394,33 @@ EOF provision_gitea_container() { local image="${E2E_GITEA_IMAGE:-gitea/gitea:1.22}" - local name="gfs-e2e-gitea-$$" + local run_id="${GITHUB_RUN_ID:-local-$(date +%s)}" + local run_attempt="${GITHUB_RUN_ATTEMPT:-1}" + local name="gfs-e2e-gitea-${run_id}-${run_attempt}-$$" log "Starting gitea container ($image)" - # No -p host-port mapping: on a self-hosted runner that is *itself* a - # sibling container of the Docker daemon (confirmed to be this fleet's - # topology -- a published host port + `127.0.0.1` is only reachable from - # the Docker host's own network namespace, not from a sibling container's), - # a host-port + 127.0.0.1 URL is unreachable. The container's own bridge - # IP is reachable from any container on the same (default) Docker - # network, including the runner itself, whether the runner is bare-metal - # or a sibling container -- so use that instead. + # Gitea runs beside this script on a developer machine or fresh + # GitHub-hosted VM. Publishing to loopback avoids Docker bridge-IP routing + # assumptions and asks Docker for a collision-free host port. docker run -d --name "$name" \ + -p 127.0.0.1::3000 \ -e GITEA__security__INSTALL_LOCK=true \ "$image" >/dev/null echo "$name" >"$workdir/gitea-container-name" - # Retry: docker run -d returns before the network attachment always has - # an IP assigned yet on every runner/docker version observed. - local container_ip="" + local host_port="" for _ in 1 2 3 4 5 6 7 8 9 10; do - container_ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$name") - [ -n "$container_ip" ] && break + host_port=$(docker port "$name" 3000/tcp | sed -n 's/^127\.0\.0\.1://p' | head -n 1) + [ -n "$host_port" ] && break sleep 1 done - if [ -z "$container_ip" ]; then - echo "gitea container never got a network IP (docker inspect empty)" >&2 + if [ -z "$host_port" ]; then + echo "gitea container never published a localhost port" >&2 docker logs "$name" >&2 || true exit 1 fi # NOSONAR-justified plain HTTP (shell:S5332, x5 below): base_url never - # leaves the Docker bridge network this run created -- container_ip is a - # per-run internal address, admin_pass/token are freshly random and - # discarded when the container is torn down at cleanup, and there is no - # TLS-terminating endpoint to speak to on an ephemeral local sandbox - # container. Not a real clear-text-credential exposure. - local base_url="http://${container_ip}:3000" # NOSONAR + # leaves loopback; credentials are per-run and discarded at cleanup. + local base_url="http://127.0.0.1:${host_port}" # NOSONAR local ready_ms="${E2E_CONTAINER_READY_MS:-60000}" local poll_ms="${E2E_POLL_INTERVAL_MS:-500}" diff --git a/scripts/run-e2e.sh b/scripts/run-e2e.sh index 1da3be1..5023fa8 100755 --- a/scripts/run-e2e.sh +++ b/scripts/run-e2e.sh @@ -22,10 +22,22 @@ if [ -z "$provider" ]; then fi export E2E_PROVIDER="$provider" -export E2E_WORKDIR="${E2E_WORKDIR:-${TMPDIR:-/tmp}/gfs-e2e-${provider}}" +created_workdir=0 +if [ -z "${E2E_WORKDIR:-}" ]; then + E2E_WORKDIR=$(mktemp -d "${TMPDIR:-/tmp}/gfs-e2e-${provider}.XXXXXX") + created_workdir=1 +fi +export E2E_WORKDIR cleanup() { scripts/e2e-harness.sh cleanup || true + if [ "$created_workdir" -eq 1 ] \ + && [[ ! "${E2E_KEEP_BRANCH:-}" =~ ^(1|true)$ ]]; then + # Only remove the exact mktemp directory this invocation created. + case "$E2E_WORKDIR" in + "${TMPDIR:-/tmp}/gfs-e2e-${provider}."*) rm -rf -- "$E2E_WORKDIR" ;; + esac + fi } trap cleanup EXIT diff --git a/tests/ci-workflow.test.ts b/tests/ci-workflow.test.ts index ee42c31..3a56655 100644 --- a/tests/ci-workflow.test.ts +++ b/tests/ci-workflow.test.ts @@ -4,15 +4,50 @@ import { readFileSync } from 'node:fs'; import { describe, expect, it } from 'vitest'; const workflow = readFileSync('.github/workflows/ci.yml', 'utf8'); +const harness = readFileSync('scripts/e2e-harness.sh', 'utf8'); +const runner = readFileSync('scripts/run-e2e.sh', 'utf8'); describe('CI workflow contracts', () => { it('retries transient provider failures three times', () => { expect(workflow).toContain('max_attempts: 3'); }); + it('runs secretless Gitea E2E on an ephemeral GitHub-hosted runner', () => { + expect(workflow).toMatch(/gitea-e2e:[\s\S]*?runs-on: ubuntu-latest/); + expect(workflow).toMatch(/gitea-e2e:[\s\S]*?permissions:\n\s+contents: read/); + expect(workflow).toContain('scripts/run-e2e.sh --provider gitea'); + expect(workflow).not.toContain('Gitea E2E is disabled in CI'); + }); + + it('keeps credentialed providers on self-hosted runners away from fork PRs', () => { + expect(workflow).toContain('provider: [github, gitlab]'); + expect(workflow).toContain("github.event_name != 'pull_request'"); + expect(workflow).toContain('github.event.pull_request.head.repo.full_name == github.repository'); + }); + + it('requires both Gitea and credentialed-provider results before downstream CI', () => { + expect(workflow).toContain('needs: [gitea-e2e, provider-e2e]'); + expect(workflow).toContain('gitea_result="${{ needs.gitea-e2e.result }}"'); + expect(workflow).toContain('provider_result="${{ needs.provider-e2e.result }}"'); + }); + it('does not fail or continue downstream CI when a provider run is replaced', () => { expect(workflow).toContain('if [ "$result" = "cancelled" ]; then'); expect(workflow).toContain('echo "run-ci=false" >> "$GITHUB_OUTPUT"'); expect(workflow).toContain("if: needs.e2e-gate.outputs.run-ci == 'true'"); }); }); + +describe('local Gitea harness contracts', () => { + it('publishes Gitea on a Docker-assigned localhost port', () => { + expect(harness).toContain('-p 127.0.0.1::3000'); + expect(harness).toContain('docker port "$name" 3000/tcp'); + expect(harness).toContain('local base_url="http://127.0.0.1:${host_port}"'); + expect(harness).not.toContain("docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}'"); + }); + + it('allocates an isolated default workdir for concurrent local runs', () => { + expect(runner).toContain('mktemp -d "${TMPDIR:-/tmp}/gfs-e2e-${provider}.XXXXXX"'); + expect(runner).toContain('created_workdir=1'); + }); +}); From 18de6e0bf848597ebba2ef23aec75ecca3c7094b Mon Sep 17 00:00:00 2001 From: tianyao Date: Thu, 27 Aug 2026 07:03:04 +0000 Subject: [PATCH 2/4] fix(ci): continue after intentionally skipped E2E jobs --- .github/workflows/ci.yml | 5 ++++- docs/testing/real-provider-e2e.md | 4 +++- tests/ci-workflow.test.ts | 4 +++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e24590f..7d204b0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -282,7 +282,10 @@ jobs: CI: needs: e2e-gate - if: needs.e2e-gate.outputs.run-ci == 'true' + # `always()` prevents a skipped provider matrix (for example a targeted + # Gitea-only dispatch or fork PR) from propagating past the successful + # aggregate gate and silently skipping downstream CI. + if: always() && needs.e2e-gate.result == 'success' && needs.e2e-gate.outputs.run-ci == 'true' uses: firstsun-dev/.github/.github/workflows/obsidian-plugin-ci.yml@v1 with: plugin-id: "git-file-sync" diff --git a/docs/testing/real-provider-e2e.md b/docs/testing/real-provider-e2e.md index ce50bdf..de10a98 100644 --- a/docs/testing/real-provider-e2e.md +++ b/docs/testing/real-provider-e2e.md @@ -274,7 +274,9 @@ changes -> provider-e2e [github | gitlab] -----/ `e2e-gate` runs with `if: always()` and evaluates both dependencies. `success` and `skipped` pass; `cancelled` means a newer run replaced this one and suppresses duplicate downstream CI; any other -result blocks CI/release. +result blocks CI/release. The downstream reusable `CI` job also uses `always()` plus explicit gate +result/output checks; without it, GitHub propagates a deliberately skipped provider job through the +successful gate and silently skips CI on Gitea-only dispatches and fork PRs. **Branch protection** (not something this repo checkout can change — a GitHub repo-settings change, left for whoever has admin access): add `E2E / gitea` as a required status check. diff --git a/tests/ci-workflow.test.ts b/tests/ci-workflow.test.ts index 3a56655..70bc188 100644 --- a/tests/ci-workflow.test.ts +++ b/tests/ci-workflow.test.ts @@ -34,7 +34,9 @@ describe('CI workflow contracts', () => { it('does not fail or continue downstream CI when a provider run is replaced', () => { expect(workflow).toContain('if [ "$result" = "cancelled" ]; then'); expect(workflow).toContain('echo "run-ci=false" >> "$GITHUB_OUTPUT"'); - expect(workflow).toContain("if: needs.e2e-gate.outputs.run-ci == 'true'"); + expect(workflow).toContain( + "if: always() && needs.e2e-gate.result == 'success' && needs.e2e-gate.outputs.run-ci == 'true'", + ); }); }); From b5884fc50d71a74d30c7f0fe7d04c54f6f1ab998 Mon Sep 17 00:00:00 2001 From: tianyao Date: Thu, 27 Aug 2026 07:08:05 +0000 Subject: [PATCH 3/4] fix(ci): isolate manual E2E concurrency --- .github/workflows/ci.yml | 4 ++-- docs/testing/real-provider-e2e.md | 17 ++++++----------- tests/ci-workflow.test.ts | 5 +++++ 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7d204b0..5a126ef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -76,7 +76,7 @@ jobs: github.event.inputs.provider == 'all' || github.event.inputs.provider == 'gitea') concurrency: - group: e2e-${{ github.head_ref || github.ref_name }}-gitea + group: e2e-${{ (github.event_name == 'push' || github.event_name == 'pull_request') && (github.head_ref || github.ref_name) || format('{0}-{1}', github.event_name, github.run_id) }}-gitea cancel-in-progress: true env: E2E_KEEP_BRANCH: ${{ github.event.inputs.keep_branch }} @@ -139,7 +139,7 @@ jobs: # support the `matrix` context (unlike job-level `if:`, see the comment # below), so each matrix leg still gets its own group. concurrency: - group: e2e-${{ github.head_ref || github.ref_name }}-${{ matrix.provider }} + group: e2e-${{ (github.event_name == 'push' || github.event_name == 'pull_request') && (github.head_ref || github.ref_name) || format('{0}-{1}', github.event_name, github.run_id) }}-${{ matrix.provider }} cancel-in-progress: true env: E2E_GITHUB_OWNER: ${{ vars.E2E_GITHUB_OWNER }} diff --git a/docs/testing/real-provider-e2e.md b/docs/testing/real-provider-e2e.md index de10a98..8be3262 100644 --- a/docs/testing/real-provider-e2e.md +++ b/docs/testing/real-provider-e2e.md @@ -111,17 +111,12 @@ killed run's branch is simply never touched by the next one. ### Concurrency and cancellation -`.github/workflows/ci.yml`'s `provider-e2e` job carries a per-source-branch/per-provider -concurrency group (`e2e--`, using `github.head_ref || github.ref_name` — the -same expression as `E2E_SOURCE_BRANCH`) with `cancel-in-progress: true`, so a superseding -push/rerun cancels its own predecessor instead of the two competing for runner/provider capacity. -The group is keyed by branch name alone, deliberately *not* split by trigger event: a `push` to a -branch with an open PR fires both a `push` and a `pull_request` run for the same commit, and an -earlier version of this group keyed PR runs by number instead of branch name, putting those two -runs in different groups — so they ran fully concurrently against the same shared provider -sandbox and starved each other (observed as real GitLab API timeouts under that double load). -Keying by branch name alone means the later of the two cancels the earlier instead. The two -cleanup workflows below share this same group naming for the same branch, with +`.github/workflows/ci.yml`'s E2E jobs carry per-source/per-provider concurrency groups with +`cancel-in-progress: true`. Push and pull-request runs use the same branch identity, so a push to a +branch with an open PR cancels its duplicate instead of both competing for runner/provider +capacity. Manual dispatches and schedules use `-` instead: they must not cancel a +normal PR's required Gitea check or a push's credentialed provider run. The two cleanup workflows +share the branch-based group naming used by push/PR runs, with `cancel-in-progress: false`, so cleanup queues behind rather than races an active run. The cancelled duplicate's `e2e-gate` reports the replacement as neutral and sets `run-ci=false`, so it neither leaves a misleading aggregate failure nor starts a second copy of downstream CI. diff --git a/tests/ci-workflow.test.ts b/tests/ci-workflow.test.ts index 70bc188..2225b7a 100644 --- a/tests/ci-workflow.test.ts +++ b/tests/ci-workflow.test.ts @@ -31,6 +31,11 @@ describe('CI workflow contracts', () => { expect(workflow).toContain('provider_result="${{ needs.provider-e2e.result }}"'); }); + it('deduplicates push and PR runs without cancelling manual or scheduled checks', () => { + const independentRunIdentity = "format('{0}-{1}', github.event_name, github.run_id)"; + expect(workflow.split(independentRunIdentity)).toHaveLength(3); + }); + it('does not fail or continue downstream CI when a provider run is replaced', () => { expect(workflow).toContain('if [ "$result" = "cancelled" ]; then'); expect(workflow).toContain('echo "run-ci=false" >> "$GITHUB_OUTPUT"'); From bf33cd288594460dcea0559d6d1abeffb0b4a39e Mon Sep 17 00:00:00 2001 From: tianyao Date: Thu, 27 Aug 2026 07:14:36 +0000 Subject: [PATCH 4/4] docs: record Gitea E2E CI verification --- docs/testing/real-provider-e2e.md | 5 +- feature_list.json | 10 +-- progress.md | 124 +++--------------------------- session-handoff.md | 44 ++++------- 4 files changed, 34 insertions(+), 149 deletions(-) diff --git a/docs/testing/real-provider-e2e.md b/docs/testing/real-provider-e2e.md index 8be3262..756c08b 100644 --- a/docs/testing/real-provider-e2e.md +++ b/docs/testing/real-provider-e2e.md @@ -301,8 +301,9 @@ structurally cannot produce. ## Known gaps -- The new GitHub-hosted `gitea-e2e` job and two-input `e2e-gate` must be confirmed by a real - workflow run before issue #139 is complete; local runs cannot prove GitHub runner behavior. +- A real external fork PR has not yet exercised the fork event context end to end. Workflow + contracts enforce the trust split, and Gitea-only dispatch run 33048613679 proved the equivalent + `Gitea success + credentialed providers skipped` gate path through downstream CI. - Branch-protection required-check configuration (`E2E / gitea`) is a manual follow-up for whoever has admin access to the repo. - The official Obsidian community-plugin scanner rescan (as opposed to this repo's own diff --git a/feature_list.json b/feature_list.json index 74d6b91..51e8783 100644 --- a/feature_list.json +++ b/feature_list.json @@ -1,14 +1,14 @@ { "_note": "GitHub Issues (firstsun-dev/git-files-sync, Project #6) is the source of truth for the full backlog and priority/estimate fields. This file mirrors only the active feature and the next few candidates so an agent session has a local, offline checkpoint — sync it against `gh issue list --repo firstsun-dev/git-files-sync --state open` at the start of a session rather than treating it as authoritative.", - "_lastSync": "2026-08-19: Synced against open GitHub issues; issue #105 is the active architecture refactor.", + "_lastSync": "2026-08-27: Synced against open GitHub issues; issue #139 is active on PR #140.", "features": [ { - "id": "feat-026", - "name": "refactor(sync): separate planning, execution, conflicts, metadata, and UI (issue #105)", - "description": "Preserve sync behavior while extracting SyncStatusView presentation state/controller boundaries and SyncManager scanner/planner/executor/workspace boundaries with regression and integration coverage.", + "id": "feat-027", + "name": "test(e2e): run disposable Gitea safely in local and CI environments (issue #139)", + "description": "Keep local Gitea runs parallel-safe, move the secretless Gitea PR gate to GitHub-hosted runners, and prevent fork code from reaching credentialed self-hosted runners.", "dependencies": [], "status": "in-progress", - "evidence": "Commits dff95db/948df28 on refactor/sync-domain-pipeline: unified sync decisions and CI hardening are covered; 613 tests, local Gitea E2E, and real CI run 32338116598 are green; desktop/mobile smoke pending." + "evidence": "Commits 920adee/18de6e0/b5884fc on PR #140: local serial/parallel Gitea and targeted real CI run 33048613679 are green; awaiting review/merge." }, { "id": "feat-004", diff --git a/progress.md b/progress.md index bb8a526..f39a8c5 100644 --- a/progress.md +++ b/progress.md @@ -4,122 +4,20 @@ Completed work is archived in [archive/](./archive/), one file per calendar mont ## Current State -**Last Updated:** 2026-08-20 -**Active Feature:** feat-026 / issue #105 — sync architecture refactor on `refactor/sync-domain-pipeline`. `SyncPlanner` is now the decision source for normal push, batch pull/preview, single pull, and moves. Edited tracked renames with a free destination plan one move instead of being auto-skipped; remote-only changes pull without false conflicts; real two-sided divergence and occupied move destinations remain conflicts. Post-push CI hardening is locally green; real provider CI plus Obsidian desktop/mobile manual verification remain before declaring the feature complete. -**Parallel Work:** PR #87 (4x Dependabot security alerts via npm overrides) and Issue #57 (live-credential smoke test). +**Last Updated:** 2026-08-27 +**Active Feature:** feat-027 / issue #139 — disposable Gitea local/CI portability and runner trust separation. +**Branch / PR:** `codex/test-gitea-e2e-ci` / PR #140, based on `claude/source-control-foundation` because that stacked branch contains the current E2E baseline. ## Outstanding Items -0a. **Re-enable the gitea leg in CI** (`.github/workflows/ci.yml`, "Determine whether this provider leg should run" step) — disabled 2026-08-13 after two rounds of real-CI-only failures (host-port/127.0.0.1 unreachable from this self-hosted fleet's sibling-container topology, then a curl hang) got fixed but a third run wasn't attempted before the user asked to pause it; harness code (`scripts/e2e-harness.sh`'s gitea path, `e2e/suites/gitea.e2e.test.ts`) is unchanged and passes locally every time (`npm run test:e2e -- --provider gitea`, most recently re-confirmed 14/14 twice this session). While disabled, fork PRs get zero E2E coverage (gitea is normally the only leg that needs no secrets). PR #124 is already open and green with this leg gated off; re-enabling is a follow-up, not a blocker. -1. **feat-025 manual verification** — Tree view code is complete and all automated checks pass; manual Obsidian verification in a real vault remains for user to confirm functionality (tree hierarchy, folder expand/collapse, checkboxes, Show synced toggle). -2. **PR #87** — Dependabot security patches via npm overrides; awaiting review/merge. -3. **Issue #57** — Live-credential smoke test; pre-existing, relevant before pushing major sync work. +1. Review and merge PR #140, then allow the stacked source-control branch to reach `main`. +2. Configure `E2E / gitea` as a required check when branch protection is next updated. +3. Optionally validate the fork-only event path with a controlled external fork PR; workflow contracts and targeted dispatch already prove the same success/skipped gate combination. -## Latest Evidence +## Verification Evidence -- [x] Issue #105 post-push CI hardening (2026-08-20), commit `948df28`: diagnosed run 32336155736 as two exhausted transient-provider attempts rather than a planner regression (GitHub 503/socket close; GitLab deadline exceeded). Increased provider E2E attempts from 2 to 3. A duplicate matrix cancelled by the shared push/PR concurrency group now produces a neutral aggregate gate with `run-ci=false`, so it neither creates a misleading `E2E gate` failure nor starts duplicate downstream CI; real failures still block. SyncManager E2E push preconditions now include `success`, `failed`, and provider `errors` in assertion diagnostics instead of surfacing only a secondary count mismatch. Added workflow contract and diagnostic unit tests and updated the E2E documentation. Verification: `actionlint v1.7.12 .github/workflows/ci.yml` — 0 errors; `npx eslint .` — 0 errors; `npm run build` — clean including Obsidian 1.11 compatibility; `npx vitest run` — 56 files / 613 tests; `npm run test:e2e -- --provider gitea` — 2 files / 14 tests and container cleanup; `git diff --check` — clean. Real CI run 32338116598 passed GitHub/GitLab production E2E, independent verification, cleanup, aggregate gate, Node 22/24 tests, lint, package, and build/release. The initial disabled-Gitea job landed on offline runner `heavenweb-runner-8`; failed-only rerun completed its skip in 11s and the full run concluded success. Provider API checks found no remaining `e2e/pr/127/**` or branch-source E2E refs. AGENTS-required Haiku was unavailable, so verification ran locally and through real CI. +- Commits `920adee`, `18de6e0`, `b5884fc`: Gitea uses a Docker-assigned loopback port and each local invocation owns a `mktemp` workdir; Gitea CI runs on `ubuntu-latest` with `contents: read`; GitHub/GitLab stay on self-hosted runners with fork rejection at job level; manual/scheduled concurrency cannot cancel normal push/PR checks. +- Local: `npx eslint .` — 0 errors; `npm run build` — pass including Obsidian 1.11 compatibility; `npx vitest run` — 56 files / 554 tests; shell syntax/ShellCheck/actionlint/diff checks — pass (known custom `32gb-ram` label excluded from actionlint); Gitea E2E — 3 files / 27 passed, 17 skipped, including two concurrent runs with distinct ports/workdirs and complete cleanup. +- Real CI: targeted Gitea run 33048613679 — Gitea, aggregate gate, shared lint, Node 22/24 tests, package, and build/release all passed. Concurrent push run 33048499785 retained a successful `E2E / gitea` check while the manual run executed, proving concurrency isolation. SonarCloud passed on PR #140. -- [x] Issue #105 unified sync decisions and move regression (2026-08-20): added operation-aware `SyncPlanner.planFor(push|pull)`, `MoveFacts`, and the `move` domain action. Normal push, batch pull and preview, single pull, and tracked moves now consume planner decisions instead of reimplementing SHA conflict checks. Removed `PushCoordinator.queueMove`'s stale-metadata gate, so an edited tracked rename with a free destination appears under Moves and commits once; occupied destinations remain conflicts. Fixed the complementary pull false positive: a remote-only change now pulls, while real two-sided divergence still resolves as conflict. Content-fetched text/binary paths normalize equal bytes to the provider blob SHA before planning, preserving binary and GitLab legacy-baseline behavior. Added planner operation matrix, coordinator move regression, batch pull, and single pull coverage. Verification: `npx eslint .` — 0 errors; `npm run build` — clean including Obsidian 1.11 compatibility; `npx vitest run` — 54 files / 610 tests; `git diff --check` — clean. Manual Obsidian verification remains. - -- [x] Issue #105 architecture implementation (2026-08-19): extracted `SyncStatusRenderer` and `SyncStatusComposition`; `SyncStatusView.ts` is 11.5 KB / 251 lines. Extracted `PullCoordinator` and `PushCoordinator`; `SyncManager.ts` is 13.7 KB / 298 lines and retains its public compatibility API. `SyncManagerWorkspace` now owns refresh/tree-snapshot reuse, push/pull, diff, local/remote deletion, move, metadata mutations, provider URLs and UI-safe workspace info; sync-status UI code no longer reaches provider/tree/settings/vault mutation helpers, and `src/logic/**` has no UI imports. Legacy refresh characterization cases now target the extracted service instead of private View delegates; legacy modal tests explicitly inject the Obsidian interaction adapter. Added real refresh integration plus focused push-coordinator/workspace regression tests. Independent verification: `npx eslint .` — 0 errors; `npm run build` — clean including Obsidian 1.11 compatibility; `npx vitest run` — 54 files / 598 tests; `npm run test:e2e -- --provider gitea` — 2 files / 14 tests with container cleanup; `git diff --check` — clean. Desktop/mobile Obsidian smoke remains manual. - -- [x] Issue #105 architecture slice 3 (2026-08-19): added tested `SyncDiffService` and `SyncStatusNavigator`, so lazy blob loading/cache/content-kind projection is a domain `FileDiff` boundary. Extracted single-file, batch push/pull, local/remote delete, move revert, remote-tree reuse, progress/confirmation, and optimistic-status orchestration into `SyncStatusOperations`; all View row/group events now enter through `SyncStatusController`. The actual View is about 40 KB (down from 58 KB this slice and 80 KB initially). Independent verification: `npx eslint .` — 0 errors; `npm run build` — clean including Obsidian 1.11 compatibility; `npx vitest run` — 52 files / 594 tests; `npm run test:e2e -- --provider gitea` — 2 files / 14 tests with container cleanup; `git diff --check` — clean. Feature remains in progress because renderer composition and the ~50 KB manager facade are still oversized. -- [x] Issue #105 architecture slice 2 (2026-08-19): extracted `SyncStatusRefreshService` for local/remote discovery, hidden files, symlinks, SHA/content classification, out-of-band move reconciliation, and live modify/rename transitions. The actual View fell from about 80 KB to 58 KB while legacy characterization entrypoints remain thin delegates. Added `SyncInteractionPort` plus `ObsidianSyncInteraction`; `logic/sync/SyncManager.ts` no longer imports Modal or Notice classes. Independent verification: `npx eslint .` — 0 errors; `npm run build` — clean including Obsidian 1.11 compatibility; `npx vitest run` — 51 files / 585 tests; `npm run test:e2e -- --provider gitea` — 2 files / 14 tests with container cleanup; `git diff --check` — clean. Feature remains in progress: action orchestration is still View-owned, the manager core is about 50 KB, and desktop/mobile manual smoke tests remain pending. -- [x] Issue #105 architecture slice (2026-08-19): moved compatibility entrypoints to thin re-exports; added presentation state, pure selectors, path-only controller commands, pure planner matrix, scanner, metadata store, push/pull/remote-delete/conflict executors, `SyncManagerWorkspace`, `FileDiff`, and four workspace integration paths. `npx eslint .` — 0 errors; `npm run build` — clean including Obsidian 1.11 compatibility; `npx vitest run` — 51 files / 585 tests passed; `npm run test:e2e -- --provider gitea` — 2 files / 14 tests passed with sandbox cleanup. Feature remains in progress: the implementation files are still oversized (`sync-status/SyncStatusView.ts` ~80 KB, `sync/SyncManager.ts` ~50 KB), domain still imports modal adapters, and manual Obsidian desktop/mobile checks are pending. -- [x] Real-provider E2E Phase 2, PR #124 fully green (2 more follow-up commits, same branch/PR): - (1) NOSONAR placement fix — the previous commit's `# NOSONAR` comments on the gitea-provisioning - `curl` calls landed on the *closing* line of each multi-line statement, but SonarCloud attributes - shell:S5332 to the *opening* `curl` line, which can't carry a trailing comment while also ending - in a `\` continuation; collapsed those two calls to single lines (payload JSON pulled into a - local var first) so the marker lands correctly — confirmed via SonarCloud's issues API (2 of 7 - findings were still OPEN after the first fix, both on the curl lines themselves; 0 after this - one, Security Rating A). (2) Dedup push/pull_request races — `provider-e2e`'s concurrency group - keyed PR runs by PR number and branch-only runs by branch name, so a push to a branch with an - open PR (this branch, since it has an open PR) fired both a `push` and a `pull_request` run in - *different* concurrency groups for the same commit, running fully concurrently against the same - shared GitLab sandbox; reproduced twice (rerunning the `pull_request`-triggered run's GitLab leg - failed both times — first with 3 different real-API errors including `400: Deadline Exceeded`, - then with a plain `testConnection` 120s timeout — while the `push`-triggered run for the - identical commit passed cleanly both times). Fixed by keying the group by branch name alone - (`github.head_ref || github.ref_name`, same expression `E2E_SOURCE_BRANCH` already used) - regardless of trigger event, and updated `e2e-pr-cleanup.yml`/`e2e-branch-cleanup.yml`'s groups - to match (documented as sharing `provider-e2e`'s group so cleanup queues behind rather than races - an active run). Verified end-to-end: pushed the fix, both a `push` and a `pull_request` run fired - again for the same commit as expected, and this time the concurrency group correctly cancelled - one of them instead of letting them race — the surviving run passed 100% clean (GitHub/GitLab/ - Gitea E2E, full CI, SonarCloud A). User explicitly chose "fix the dedup now" over deferring or - just re-running until green, when asked. - Verification: `actionlint` — 0 errors; `npx eslint .` — 0 errors; `npm run build` — clean; - `npx vitest run` — 527 passed; real end-to-end Gitea sandbox run (`npm run test:e2e -- - --provider gitea`) — 14/14 passed, twice, exercising the edited curl calls directly; real CI — - PR #124's surviving run fully green including all three real-provider E2E legs and SonarCloud - Security Rating A. -- [x] Real-provider E2E Phase 2 follow-up fix (same branch/PR #124): `ci.yml`'s `provider-e2e` job - set `E2E_WORKDIR` in job-level `env:` using `${{ runner.temp }}` — `runner` isn't an allowed - context there (only `github`/`inputs`/`matrix`/`needs`/`secrets`/`strategy`/`vars` are), which - makes GitHub Actions reject the whole workflow file at parse time; confirmed via `actionlint` - and via the GitHub API (`jobs_url` for the c8382cb push run returned `total_count: 0` — no job - was ever created). Fixed by computing `E2E_WORKDIR` in an unconditional first step instead, - exporting it through `$GITHUB_ENV` (uses `$RUNNER_TEMP`, the step-level equivalent). Also fixed - the SonarCloud Quality Gate failure (Security Rating D on new code, required ≥ A): - `scripts/e2e-namespace.sh`'s `e2e_branch_hash` used `sha1sum`/`shasum` (CRITICAL, shell:S4790 - weak-hash — not a real security use, just a collision-avoidance digest, but Sonar flags SHA-1 - regardless of context) — switched to `sha256sum`/`shasum -a 256`; five `curl`/log lines in - `scripts/e2e-harness.sh`'s gitea provisioning that talk `http://` to a per-run Docker-bridge-only - container (shell:S5332 clear-text-protocol) — annotated `# NOSONAR` with an inline justification - (address never leaves the run's own Docker network, credentials are freshly random and - discarded at cleanup); `.github/workflows/ci.yml`'s new `npm ci` (githubactions:S6505, missing - `--ignore-scripts`) and `actions/checkout@v6`/`actions/setup-node@v6`/`dorny/paths-filter@v3` in - the two new jobs plus the three new standalone workflow files (githubactions:S7637, unpinned - action refs) — pinned to full commit SHAs, `npm ci` in the new job got `--ignore-scripts` - (husky's `prepare` hook isn't needed in CI). Left the pre-existing `build-artifact` job's - checkout/setup-node/npm ci untouched (not flagged, out of this fix's scope). - Verification: `actionlint` (downloaded v1.7.12 binary) — 0 errors on all 4 workflow files - (aside from an expected false-positive on the `32gb-ram` custom self-hosted label, which - actionlint can't know about); `bash -n` on all 5 changed/touched shell scripts — all parse; - `npx eslint .` — 0 errors; `npm run build` (incl. Obsidian 1.11.0 compat typecheck) — clean; - `npx vitest run` — 527 passed. Not yet re-verified against real CI/SonarCloud (push pending). -- [x] Real-provider E2E Phase 2 (multi-run isolation): added `scripts/e2e-namespace.sh` (single - canonical `e2e/pr///run--` / `e2e/branch/// - run--` identity generator, sourced by every other layer — no branch-naming logic - duplicated anywhere else), `scripts/e2e-namespace-cleanup.sh` (layer 2: deletes a whole PR/branch - namespace), `scripts/e2e-janitor.sh` (layer 3: TTL sweep, default 24h, of any leftover `e2e/**` - branch, generic `git for-each-ref`/`push --delete`, tolerant of already-deleted refs — no - Node-based sweeper reintroduced). Removed `e2e-harness.sh`'s old `sweep` subcommand (superseded - by the janitor) and its ad hoc `gfs-e2e--` naming. `ci.yml`'s `provider-e2e` job - now sets `E2E_WORKDIR` to `$RUNNER_TEMP/git-files-sync-e2e///` - (was a shared `e2e-` dir), passes `E2E_PR_NUMBER`/`E2E_SOURCE_BRANCH` through for - `provision`, and carries a per-source/provider `concurrency` group - (`e2e-pr--`/`e2e-branch--`, `cancel-in-progress: true`) so a - repeated push/rerun cancels its own predecessor instead of both running. Added - `.github/workflows/e2e-pr-cleanup.yml` (`pull_request_target: [closed]`, no `ref:` override on - checkout so it only ever runs this repo's own trusted code/secrets, never the closing PR's - branch) and `e2e-branch-cleanup.yml` (`delete` event) — both share the same concurrency-group - naming as `provider-e2e` with `cancel-in-progress: false` so cleanup queues behind rather than - races an active run. Added `.github/workflows/e2e-janitor.yml` (schedule, every 6h, plus - `workflow_dispatch`). Rewrote `docs/testing/real-provider-e2e.md`'s "Isolation model" section - (namespace scheme, concurrency/cancellation semantics, 3-layer cleanup hierarchy with a Mermaid - diagram, self-hosted workdir isolation) and updated Layout/CI/Cleanup/Known-gaps to match. - Verification: `npx eslint .` — 0 errors; `npm run build` (incl. Obsidian 1.11.0 compat - typecheck) — clean; `npx vitest run` — 527 passed; `python3 -c yaml.safe_load(...)` on all 4 - touched/new workflow YAML files — all parse; `bash -n` on all 4 shell scripts — all parse; - functional dry-runs against throwaway local git repos (not the real sandboxes) for - `e2e_test_branch`/`e2e_branch_id` collision resolution (`feature/foo-bar` vs `feature-foo/bar` - hash to different identities), the janitor's TTL sweep (old branch deleted, recent branch and an - unrelated `feature/keep-me` branch both left untouched), and `e2e-namespace-cleanup.sh`'s prefix - match (`e2e/pr/123/**` matches only that PR's two provider branches, not PR 456 or the - branch-only namespace); **real end-to-end run against a live local Gitea sandbox** - (`npm run test:e2e -- --provider gitea`) with the new harness/namespace code — 14/14 E2E tests - passed including a real Docker provision/seed/cleanup cycle; confirmed - `E2E_PROVIDER=github scripts/e2e-harness.sh provision` still hard-fails on missing - `E2E_GITHUB_OWNER` (never a silent skip) with the new identity plumbing in place. Not yet - exercised against live GitHub/GitLab sandboxes or the real self-hosted runner fleet from this - checkout (no credentials/runner access here) — see `docs/testing/real-provider-e2e.md`'s "Known - gaps". -- [x] Real-provider E2E: pushed to `origin/test/real-provider-e2e`, real CI run against `firstsun-dev/git-files-sync`'s self-hosted fleet (run 31666859288) fully green: `E2E / github` (3m15s) and `E2E / gitlab` (3m54s) both passed for real against live sandboxes, `E2E / github`+`gitlab`+`gitea` gate, and the full downstream `CI` (lint, test Node 22/24, package, build/release) all green. Getting there took 3 fix-and-repush rounds off real CI failures the local-only verification hadn't caught: (1) the generated `GitVerifier`'s git calls had no `GIT_ASKPASS`/`GIT_TERMINAL_PROMPT` in the separate vitest-step process — fixed by persisting them (paths/flags only, not the token itself) into `e2e.env`; (2) gitea provisioning timed out on `127.0.0.1:` — this runner fleet is itself a sibling container of the Docker daemon, so a published host port isn't reachable from it; switched to the container's own bridge IP; (3) that same curl call could hang indefinitely with no `--max-time`, silently blowing past the health-check loop's own retry budget — added `--max-time` everywhere and a retry-with-backoff on `docker inspect` returning an empty IP. Gitea leg then temporarily disabled in CI per user request (still passes locally) — see Outstanding Items. -- [x] Real-provider E2E Phase 1 (Shell/Git harness rewrite): replaced the Node-based `e2e/provision`/`e2e/verifier`/`e2e/providers`/`e2e/shim/{obsidian-request-url,window-timers}`/`scripts/run-e2e*.mjs` (fetch/globalThis/node:child_process/node:crypto in committed `.ts` — the exact APIs `docs/obsidian-scanner-audit.md` flagged) with `scripts/e2e-harness.sh` (provision/seed/verify/cleanup/sweep — Shell + Git CLI: `git push :refs/heads/` for GitHub/GitLab branch isolation, plain `docker`/`curl` for Gitea's disposable container+repo, `GIT_ASKPASS` generated per-run under `$RUNNER_TEMP`/`$E2E_WORKDIR`, never persisted) plus `scripts/run-e2e.sh` (local orchestration wrapper). Node-only glue the suites still need at runtime (real `requestUrl` shim, `window` timer alias, a git-CLI-backed verifier) is generated by `provision` into `$E2E_RUNTIME_DIR` and loaded via runtime-computed dynamic `import()` — never committed — so `e2e/**/*.ts` went back into `tsconfig.json`'s `include`/`eslint.config.mts`'s scope clean. Ported all 4 suites (github/gitlab/gitea/sync-manager) to the new `SyncManager.pushFiles` API and the generated verifier. `npx eslint .` — 0 errors; `npm run build` — clean; `npx vitest run` — 527 passed; **real end-to-end run against a live local Gitea sandbox** (`npm run test:e2e -- --provider gitea`) — 14/14 E2E tests passed (gitea contract suite + SyncManager suite), including a real Docker container provision/seed/cleanup cycle. GitHub/GitLab E2E legs are written and typecheck/lint clean but weren't run live (no sandbox credentials in this environment) — same known gap the pre-Phase-1 harness had, documented in `docs/testing/real-provider-e2e.md`'s "Known gaps". Self-audit of `docs/obsidian-scanner-audit.md`'s grep method against the new tree: zero hits for `fetch`/`globalThis`/`node:crypto`/`node:child_process`/`node:util`/bare-timers in `e2e/**` or `src/**`. -- [x] Real-provider E2E Phase 0 reconcile: merged `origin/main` (scanner-driven E2E removal, v1.5.8) into `test/real-provider-e2e-work`, keeping the old `e2e/**` tree temporarily (added `e2e/**`/`vitest.e2e.config.ts` to `eslint.config.mts` `globalIgnores` as an interim measure — not in `tsconfig.json` `include` either, both to be resolved for real by the Phase 1 harness rewrite), then merged `origin/claude/unify-push-pull-pipeline` (new unified `SyncManager.pushFiles` API) cleanly (disjoint file sets, only `package-lock.json` auto-merged). `npx eslint .` — 0 errors; `npm run build` (incl. Obsidian 1.11.0 compat typecheck) — clean; `npx vitest run` — 527 tests passed. -- [x] `fix(sync): ensure parent dirs exist when reverting file moves` (issue #94): extracted `ensureParentDirs()` to `src/utils/vault-path.ts` and called it before rename in both `revertMove` and `revertMoveGroup`, fixing "folder does not exist" error when reverting moves to deleted parent folders. `npx eslint .` — 0 errors; `npm run build` — clean; `npx vitest run` — 502 tests passed. -- [x] `fix(gitlab): fix sha/revision semantics for optimistic locking` (issue #101, PR #113, merged): `GitFile.sha` now consistently represents blob identity across providers; added `GitFile.revision` for provider-specific write control. - -Full history of completed features (feat-001 through feat-024) archived to [archive/2026-07.md](./archive/2026-07.md). August work archived to [archive/2026-08.md](./archive/2026-08.md). +The AGENTS-required Haiku verifier was unavailable in this environment; verification ran locally and through real CI. diff --git a/session-handoff.md b/session-handoff.md index 9a92258..5f740b6 100644 --- a/session-handoff.md +++ b/session-handoff.md @@ -1,44 +1,30 @@ # Session Handoff -**Date:** 2026-08-20 -**Branch:** `refactor/sync-domain-pipeline` (PR #127) -**Active Feature:** feat-026 / issue #105 — sync architecture refactor +**Date:** 2026-08-27 +**Branch:** `codex/test-gitea-e2e-ci` +**Active Feature:** issue #139 / PR #140 ## Completed This Session -Investigated the failed real-provider CI after the unified planner commit. The move paths passed; -GitHub exhausted two attempts on a 503 and `UND_ERR_SOCKET`, while GitLab exhausted two attempts -on provider deadline errors. The tests then surfaced secondary count/existence assertions that -hid those original request failures. +Implemented the local/CI Gitea split in commits `920adee`, `18de6e0`, and `b5884fc`. Local Gitea now uses a Docker-assigned loopback port, collision-safe container identity, and a unique temporary workdir. CI runs secretless Gitea on `ubuntu-latest`, keeps credentialed GitHub/GitLab E2E on self-hosted runners, and rejects fork PRs before allocating those runners. -Hardened CI with three provider attempts, explicit push-result diagnostics in SyncManager E2E, -and workflow contract coverage. When the shared push/PR concurrency group cancels a duplicate -matrix, its aggregate gate now reports the replacement neutrally and emits `run-ci=false`, so it -does not leave an additional aggregate red check or run downstream CI twice. Real failures remain -blocking. Updated the real-provider E2E documentation to match. - -Committed as `948df28` (`fix(ci): harden provider e2e failures`) and pushed to -`origin/refactor/sync-domain-pipeline`. The pre-existing untracked `.codex-gitlab.env` remains -untouched. +Real CI exposed and then verified two follow-up fixes: downstream CI needs `always()` to cross an intentionally skipped provider job after the successful aggregate gate, and manual/scheduled runs need independent concurrency identities so they cannot cancel a normal PR's required Gitea check. ## Verification Evidence ```text -npx eslint . -> PASS, 0 errors -npm run build -> PASS, incl. Obsidian 1.11 compatibility -npx vitest run -> PASS, 56 files / 613 tests -npm run test:e2e -- --provider gitea -> PASS, 2 files / 14 tests; container removed -actionlint v1.7.12 .github/workflows/ci.yml -> PASS, 0 errors -git diff --check -> PASS -real CI run 32338116598 -> PASS after failed-only rerun of a disabled Gitea leg assigned to an offline runner -GitHub/GitLab sandbox branch query -> PASS, no e2e/pr/127 or source-branch refs remain +npx eslint . -> PASS, 0 errors +npm run build -> PASS, including Obsidian 1.11 compatibility +npx vitest run -> PASS, 56 files / 554 tests +local Gitea E2E -> PASS, 3 files / 27 passed / 17 skipped +two concurrent local Gitea runs -> PASS, distinct ports/workdirs, no leftovers +bash -n + ShellCheck + actionlint + git diff --check -> PASS +real targeted CI run 33048613679 -> PASS through Gitea, gate, shared CI, package, build/release +PR #140 SonarCloud -> PASS ``` -The AGENTS-required Haiku verifier was unavailable in this environment, so verification ran -locally in this session. +The AGENTS-required Haiku verifier was unavailable, so verification ran locally and in real CI. ## Exact Next Step -Complete the remaining Obsidian desktop/mobile move smoke tests. Verify moving and editing a -tracked file appears under Moves and applies as one remote move, while an occupied remote -destination remains a skipped conflict. +Review and merge PR #140. The full push run 33048499785 may still be finishing the unchanged GitHub/GitLab live-provider legs; its Gitea and SonarCloud checks are already green.