diff --git a/.agents/skills/paperclip-page/README.md b/.agents/skills/paperclip-page/README.md index 4ba859741b4..cfe2d878827 100644 --- a/.agents/skills/paperclip-page/README.md +++ b/.agents/skills/paperclip-page/README.md @@ -73,8 +73,8 @@ Required for live publishes: export AWS_REGION=us-east-1 export PAPERCLIP_PAGE_BUCKET=paperclip-pages-prod export PAPERCLIP_PAGE_BASE_URL=https://pages.paperclip.ing -export AWS_ACCESS_KEY_ID=... -export AWS_SECRET_ACCESS_KEY=... +export PAPERCLIP_PAGE_AWS_ACCESS_KEY_ID=... +export PAPERCLIP_PAGE_AWS_SECRET_ACCESS_KEY=... ``` Optional: @@ -82,15 +82,35 @@ Optional: ```bash export PAPERCLIP_PAGE_DEFAULT_PREFIX="" export PAPERCLIP_PAGE_AWS_PROFILE=paperclip-page-uploader +export PAPERCLIP_PAGE_AWS_SESSION_TOKEN=... # only with the namespaced key pair ``` +Credential resolution order inside `publish.sh`: + +1. `PAPERCLIP_PAGE_AWS_ACCESS_KEY_ID` + `PAPERCLIP_PAGE_AWS_SECRET_ACCESS_KEY` + (scoped to the helper's `aws` calls; the surrounding process identity is + untouched) +2. `PAPERCLIP_PAGE_AWS_PROFILE`, passed to `aws` as `--profile` (ambient + `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY` / `AWS_SESSION_TOKEN` / + `AWS_PROFILE` are stripped from the helper's `aws` calls so the named + profile always wins) +3. The ambient AWS credential chain + +Setting both the namespaced key pair and `PAPERCLIP_PAGE_AWS_PROFILE` is an +error. + Recommended Paperclip secret names: - `paperclip-page-aws-access-key-id` - `paperclip-page-aws-secret-access-key` -Bind those secrets into publisher agents as `AWS_ACCESS_KEY_ID` and -`AWS_SECRET_ACCESS_KEY`. Do not reuse Paperclip's internal S3 attachment/object +Bind those secrets into publisher agents as `PAPERCLIP_PAGE_AWS_ACCESS_KEY_ID` +and `PAPERCLIP_PAGE_AWS_SECRET_ACCESS_KEY`. Never bind them as the global +`AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY` names: static env keys override +`AWS_PROFILE` in the AWS CLI and every SDK, so global names silently switch the +whole agent run — and every subprocess — to the page-uploader identity and +break access to anything the uploader cannot reach (Secrets Manager, STS role +use, other buckets). Do not reuse Paperclip's internal S3 attachment/object storage credentials. ## AWS Setup @@ -484,16 +504,18 @@ pnpm paperclipai secrets create \ --value-env PAPERCLIP_PAGE_AWS_SECRET_ACCESS_KEY ``` -Bind runtime env to publishing agents: +Bind runtime env to publishing agents. Use the namespaced names — never the +global `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY`, which would shadow the +host `AWS_PROFILE` identity for the entire agent run: ```json { - "AWS_ACCESS_KEY_ID": { + "PAPERCLIP_PAGE_AWS_ACCESS_KEY_ID": { "type": "secret_ref", "secretId": "", "version": "latest" }, - "AWS_SECRET_ACCESS_KEY": { + "PAPERCLIP_PAGE_AWS_SECRET_ACCESS_KEY": { "type": "secret_ref", "secretId": "", "version": "latest" diff --git a/.agents/skills/paperclip-page/SKILL.md b/.agents/skills/paperclip-page/SKILL.md index e9e74cfe085..df77444874d 100644 --- a/.agents/skills/paperclip-page/SKILL.md +++ b/.agents/skills/paperclip-page/SKILL.md @@ -19,10 +19,20 @@ host, for example `https://pages.paperclip.ing//`. - `PAPERCLIP_PAGE_BUCKET` - `PAPERCLIP_PAGE_BASE_URL` - `AWS_REGION` - - AWS credentials via Paperclip Secrets or an approved AWS vault + - `PAPERCLIP_PAGE_AWS_ACCESS_KEY_ID` and `PAPERCLIP_PAGE_AWS_SECRET_ACCESS_KEY` + with the page-uploader credentials from Paperclip Secrets - Optional environment variables: - `PAPERCLIP_PAGE_DEFAULT_PREFIX` - - `PAPERCLIP_PAGE_AWS_PROFILE` + - `PAPERCLIP_PAGE_AWS_PROFILE` (alternative to the namespaced key pair) + - `PAPERCLIP_PAGE_AWS_SESSION_TOKEN` (only together with the namespaced key + pair) + +Do not bind the page-uploader credentials as global `AWS_ACCESS_KEY_ID` / +`AWS_SECRET_ACCESS_KEY`: static env keys take precedence over `AWS_PROFILE` in +every AWS SDK, so global names silently replace the host identity for every +process in the agent run. The namespaced variables scope the uploader identity +to this helper only. The ambient credential chain still works as a fallback +when none of the `PAPERCLIP_PAGE_AWS_*` credential variables are set. ## Workflow diff --git a/.agents/skills/paperclip-page/scripts/publish.sh b/.agents/skills/paperclip-page/scripts/publish.sh index 2aa3a356add..086003a1e11 100755 --- a/.agents/skills/paperclip-page/scripts/publish.sh +++ b/.agents/skills/paperclip-page/scripts/publish.sh @@ -15,6 +15,15 @@ Required environment for live publish: Optional environment: PAPERCLIP_PAGE_DEFAULT_PREFIX, PAPERCLIP_PAGE_AWS_PROFILE + PAPERCLIP_PAGE_AWS_ACCESS_KEY_ID, PAPERCLIP_PAGE_AWS_SECRET_ACCESS_KEY, + PAPERCLIP_PAGE_AWS_SESSION_TOKEN + +Credential resolution for aws calls made by this helper: + 1. PAPERCLIP_PAGE_AWS_ACCESS_KEY_ID + PAPERCLIP_PAGE_AWS_SECRET_ACCESS_KEY + (used only by this helper; ambient AWS_PROFILE/AWS_* identity is untouched) + 2. PAPERCLIP_PAGE_AWS_PROFILE (passed as --profile; ambient AWS_* identity + variables are stripped from the helper's aws calls) + 3. Ambient AWS credential chain (env keys, profile, instance role) Options: --slug SLUG Lowercase URL slug. Allowed: a-z, 0-9, hyphen. @@ -125,9 +134,26 @@ join_prefix() { } aws_base_args=() +aws_env_unset=() +aws_env_overrides=() aws_cli() { - aws "${aws_base_args[@]}" "$@" + local name pair + if [[ ${#aws_env_unset[@]} -gt 0 || ${#aws_env_overrides[@]} -gt 0 ]]; then + # Scope the page-uploader identity to this helper's aws calls only, and + # drop the ambient identity variables that would otherwise mix with or + # shadow the configured credential source. Apply the overrides with shell + # builtins in a subshell — passing them to an external `env` command would + # expose the credential values in its argv (world-readable via + # /proc//cmdline) while it runs. + ( + for name in "${aws_env_unset[@]}"; do unset "$name"; done + for pair in "${aws_env_overrides[@]}"; do export "$pair"; done + exec aws "${aws_base_args[@]}" "$@" + ) + else + aws "${aws_base_args[@]}" "$@" + fi } object_exists() { @@ -297,6 +323,18 @@ default_prefix="$(normalize_default_prefix "${PAPERCLIP_PAGE_DEFAULT_PREFIX:-}") [[ -n "$base_url" ]] || die "PAPERCLIP_PAGE_BASE_URL is required" base_url="$(normalize_base_url "$base_url")" +page_access_key_id="${PAPERCLIP_PAGE_AWS_ACCESS_KEY_ID:-}" +page_secret_access_key="${PAPERCLIP_PAGE_AWS_SECRET_ACCESS_KEY:-}" +if [[ -n "$page_access_key_id" || -n "$page_secret_access_key" ]]; then + [[ -n "$page_access_key_id" && -n "$page_secret_access_key" ]] || + die "PAPERCLIP_PAGE_AWS_ACCESS_KEY_ID and PAPERCLIP_PAGE_AWS_SECRET_ACCESS_KEY must be set together" + [[ -z "${PAPERCLIP_PAGE_AWS_PROFILE:-}" ]] || + die "set PAPERCLIP_PAGE_AWS_PROFILE or the PAPERCLIP_PAGE_AWS_* key pair, not both" +fi +if [[ -n "${PAPERCLIP_PAGE_AWS_SESSION_TOKEN:-}" && -z "$page_access_key_id" ]]; then + die "PAPERCLIP_PAGE_AWS_SESSION_TOKEN requires the PAPERCLIP_PAGE_AWS_* key pair" +fi + explicit_slug=0 if [[ -n "$slug_arg" ]]; then explicit_slug=1 @@ -310,8 +348,18 @@ if [[ "$dry_run" == "0" ]]; then require_command curl [[ -n "$region" ]] || die "AWS_REGION is required for live publish" aws_base_args=(--region "$region") - if [[ -n "${PAPERCLIP_PAGE_AWS_PROFILE:-}" ]]; then + if [[ -n "$page_access_key_id" ]]; then + aws_env_unset=(AWS_PROFILE AWS_SESSION_TOKEN) + aws_env_overrides=( + AWS_ACCESS_KEY_ID="$page_access_key_id" + AWS_SECRET_ACCESS_KEY="$page_secret_access_key" + ) + if [[ -n "${PAPERCLIP_PAGE_AWS_SESSION_TOKEN:-}" ]]; then + aws_env_overrides+=(AWS_SESSION_TOKEN="$PAPERCLIP_PAGE_AWS_SESSION_TOKEN") + fi + elif [[ -n "${PAPERCLIP_PAGE_AWS_PROFILE:-}" ]]; then aws_base_args+=(--profile "$PAPERCLIP_PAGE_AWS_PROFILE") + aws_env_unset=(AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN AWS_PROFILE) fi fi diff --git a/.agents/skills/paperclip-page/scripts/publish.test.mjs b/.agents/skills/paperclip-page/scripts/publish.test.mjs index 8ef4b47bd2a..bb8384bfb0b 100644 --- a/.agents/skills/paperclip-page/scripts/publish.test.mjs +++ b/.agents/skills/paperclip-page/scripts/publish.test.mjs @@ -131,6 +131,215 @@ test("rejects hidden files in the source tree", () => { assert.match(result.output, /hidden files and dot paths are not allowed/); }); +test("namespaced page keys require both halves of the pair", () => { + const result = runPublish( + [createSite(), "--slug", "demo-page", "--dry-run"], + { PAPERCLIP_PAGE_AWS_ACCESS_KEY_ID: "AKIAPAGEUPLOADER" }, + ); + + assert.notEqual(result.status, 0); + assert.match(result.output, /must be set together/); +}); + +test("namespaced session token requires the namespaced key pair", () => { + const result = runPublish( + [createSite(), "--slug", "demo-page", "--dry-run"], + { PAPERCLIP_PAGE_AWS_SESSION_TOKEN: "page-session-token" }, + ); + + assert.notEqual(result.status, 0); + assert.match(result.output, /requires the PAPERCLIP_PAGE_AWS_\* key pair/); +}); + +test("namespaced page keys conflict with PAPERCLIP_PAGE_AWS_PROFILE", () => { + const result = runPublish( + [createSite(), "--slug", "demo-page", "--dry-run"], + { + PAPERCLIP_PAGE_AWS_ACCESS_KEY_ID: "AKIAPAGEUPLOADER", + PAPERCLIP_PAGE_AWS_SECRET_ACCESS_KEY: "page-secret", + PAPERCLIP_PAGE_AWS_PROFILE: "paperclip-page-uploader", + }, + ); + + assert.notEqual(result.status, 0); + assert.match(result.output, /not both/); +}); + +test("namespaced page keys are scoped to the helper's aws calls", () => { + const siteDir = createSite(); + const binDir = mkdtempSync(join(tmpdir(), "paperclip-page-bin-")); + tempDirs.add(binDir); + const envDump = join(binDir, "aws-env.txt"); + + writeExecutable( + join(binDir, "aws"), + `#!/usr/bin/env bash +set -euo pipefail +{ + echo "AWS_ACCESS_KEY_ID=\${AWS_ACCESS_KEY_ID:-}" + echo "AWS_SECRET_ACCESS_KEY=\${AWS_SECRET_ACCESS_KEY:-}" + echo "AWS_SESSION_TOKEN=\${AWS_SESSION_TOKEN:-}" + echo "AWS_PROFILE=\${AWS_PROFILE:-}" +} >"${envDump}" +while [[ "$1" == "--region" || "$1" == "--profile" ]]; do + shift 2 +done +if [[ "$1" == "s3api" ]]; then + echo "None" + exit 0 +fi +if [[ "$1" == "s3" && "$2" == "sync" ]]; then + exit 0 +fi +echo "unexpected aws call: $*" >&2 +exit 1 +`, + ); + writeExecutable( + join(binDir, "curl"), + `#!/usr/bin/env bash +exit 0 +`, + ); + + const result = runPublish([siteDir, "--slug", "demo-page"], { + AWS_REGION: "us-east-1", + PATH: `${binDir}:${process.env.PATH}`, + AWS_ACCESS_KEY_ID: "AKIAAMBIENTIDENTITY", + AWS_SECRET_ACCESS_KEY: "ambient-secret", + AWS_SESSION_TOKEN: "ambient-session-token", + AWS_PROFILE: "ambient-profile", + PAPERCLIP_PAGE_AWS_ACCESS_KEY_ID: "AKIAPAGEUPLOADER", + PAPERCLIP_PAGE_AWS_SECRET_ACCESS_KEY: "page-secret", + }); + + assert.equal(result.status, 0); + + const seen = readFileSync(envDump, "utf8"); + assert.match(seen, /^AWS_ACCESS_KEY_ID=AKIAPAGEUPLOADER$/m); + assert.match(seen, /^AWS_SECRET_ACCESS_KEY=page-secret$/m); + assert.match(seen, /^AWS_SESSION_TOKEN=$/m); + assert.match(seen, /^AWS_PROFILE=$/m); +}); + +test("credential values never pass through an external env command's argv", () => { + const siteDir = createSite(); + const binDir = mkdtempSync(join(tmpdir(), "paperclip-page-bin-")); + tempDirs.add(binDir); + const envArgvDump = join(binDir, "env-argv.txt"); + + writeExecutable( + join(binDir, "aws"), + `#!/usr/bin/env bash +set -euo pipefail +while [[ "$1" == "--region" || "$1" == "--profile" ]]; do + shift 2 +done +if [[ "$1" == "s3api" ]]; then + echo "None" + exit 0 +fi +if [[ "$1" == "s3" && "$2" == "sync" ]]; then + exit 0 +fi +echo "unexpected aws call: $*" >&2 +exit 1 +`, + ); + writeExecutable( + join(binDir, "curl"), + `#!/usr/bin/env bash +exit 0 +`, + ); + // Shim env: record every argv it is invoked with, then behave normally. + // Credentials in that argv would be world-readable via /proc//cmdline. + writeExecutable( + join(binDir, "env"), + `#!/bin/bash +printf '%s\\n' "$@" >>"${envArgvDump}" +exec /usr/bin/env "$@" +`, + ); + + const result = runPublish([siteDir, "--slug", "demo-page"], { + AWS_REGION: "us-east-1", + PATH: `${binDir}:${process.env.PATH}`, + PAPERCLIP_PAGE_AWS_ACCESS_KEY_ID: "AKIAPAGEUPLOADER", + PAPERCLIP_PAGE_AWS_SECRET_ACCESS_KEY: "page-secret-argv-canary", + PAPERCLIP_PAGE_AWS_SESSION_TOKEN: "page-session-argv-canary", + }); + + assert.equal(result.status, 0); + + const argvSeen = existsSync(envArgvDump) ? readFileSync(envArgvDump, "utf8") : ""; + assert.doesNotMatch(argvSeen, /page-secret-argv-canary/); + assert.doesNotMatch(argvSeen, /page-session-argv-canary/); + assert.doesNotMatch(argvSeen, /AKIAPAGEUPLOADER/); +}); + +test("PAPERCLIP_PAGE_AWS_PROFILE strips ambient static credentials", () => { + const siteDir = createSite(); + const binDir = mkdtempSync(join(tmpdir(), "paperclip-page-bin-")); + tempDirs.add(binDir); + const envDump = join(binDir, "aws-env.txt"); + + writeExecutable( + join(binDir, "aws"), + `#!/usr/bin/env bash +set -euo pipefail +profile="" +while [[ "$1" == "--region" || "$1" == "--profile" ]]; do + if [[ "$1" == "--profile" ]]; then + profile="$2" + fi + shift 2 +done +{ + echo "PROFILE_ARG=$profile" + echo "AWS_ACCESS_KEY_ID=\${AWS_ACCESS_KEY_ID:-}" + echo "AWS_SECRET_ACCESS_KEY=\${AWS_SECRET_ACCESS_KEY:-}" + echo "AWS_SESSION_TOKEN=\${AWS_SESSION_TOKEN:-}" + echo "AWS_PROFILE=\${AWS_PROFILE:-}" +} >"${envDump}" +if [[ "$1" == "s3api" ]]; then + echo "None" + exit 0 +fi +if [[ "$1" == "s3" && "$2" == "sync" ]]; then + exit 0 +fi +echo "unexpected aws call: $*" >&2 +exit 1 +`, + ); + writeExecutable( + join(binDir, "curl"), + `#!/usr/bin/env bash +exit 0 +`, + ); + + const result = runPublish([siteDir, "--slug", "demo-page"], { + AWS_REGION: "us-east-1", + PATH: `${binDir}:${process.env.PATH}`, + AWS_ACCESS_KEY_ID: "AKIAAMBIENTIDENTITY", + AWS_SECRET_ACCESS_KEY: "ambient-secret", + AWS_SESSION_TOKEN: "ambient-session-token", + AWS_PROFILE: "ambient-profile", + PAPERCLIP_PAGE_AWS_PROFILE: "paperclip-page-uploader", + }); + + assert.equal(result.status, 0); + + const seen = readFileSync(envDump, "utf8"); + assert.match(seen, /^PROFILE_ARG=paperclip-page-uploader$/m); + assert.match(seen, /^AWS_ACCESS_KEY_ID=$/m); + assert.match(seen, /^AWS_SECRET_ACCESS_KEY=$/m); + assert.match(seen, /^AWS_SESSION_TOKEN=$/m); + assert.match(seen, /^AWS_PROFILE=$/m); +}); + test("live publish writes state before URL verification", () => { const siteDir = createSite(); const binDir = mkdtempSync(join(tmpdir(), "paperclip-page-bin-")); diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ba9c25e4009..2e41dd7657b 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -6,6 +6,13 @@ on: - "master" tags: - "v*" + - "nightly/v*" + - "beta/v*" + # Release workflows push lane tags with GITHUB_TOKEN, and GitHub suppresses + # push-triggered runs for those, so release.yml dispatches this workflow at + # the new tag ref instead. The tag mapping below keys off github.ref either + # way. + workflow_dispatch: permissions: contents: read @@ -40,7 +47,19 @@ jobs: id: build-version run: | set -euo pipefail - version="$(git describe --tags --match 'v*' --long --dirty 2>/dev/null || true)" + case "${GITHUB_REF}" in + refs/tags/nightly/v*) + # Lane tags carry the exact published version; stamp it verbatim + # instead of describing drift from the nearest stable tag. + version="${GITHUB_REF#refs/tags/nightly/v}" + ;; + refs/tags/beta/v*) + version="${GITHUB_REF#refs/tags/beta/v}" + ;; + *) + version="$(git describe --tags --match 'v*' --long --dirty 2>/dev/null || true)" + ;; + esac echo "version=${version}" >> "$GITHUB_OUTPUT" echo "Stamping build version: ${version:-}" @@ -131,15 +150,21 @@ jobs: echo "last=${last}" >> "$GITHUB_OUTPUT" echo "count=${count}" >> "$GITHUB_OUTPUT" + # Lane tag mapping: master pushes publish `:canary`, nightly/v* tags + # publish `:nightly`, and only stable v* tags move `:latest` and the + # versioned tags. `:sha-` is published on every build. - name: Docker meta id: meta uses: docker/metadata-action@v6 with: images: ghcr.io/${{ github.repository }} tags: | - type=raw,value=latest,enable={{is_default_branch}} - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} + type=raw,value=canary,enable=${{ github.ref == 'refs/heads/master' }} + type=raw,value=nightly,enable=${{ startsWith(github.ref, 'refs/tags/nightly/v') }} + type=raw,value=beta,enable=${{ startsWith(github.ref, 'refs/tags/beta/v') }} + type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v') }} + type=semver,pattern={{version}},enable=${{ startsWith(github.ref, 'refs/tags/v') }} + type=semver,pattern={{major}}.{{minor}},enable=${{ startsWith(github.ref, 'refs/tags/v') }} type=sha labels: | io.github.paperclipai.schema.last-migration=${{ steps.schema.outputs.last }} @@ -197,7 +222,19 @@ jobs: id: build-version run: | set -euo pipefail - version="$(git describe --tags --match 'v*' --long --dirty 2>/dev/null || true)" + case "${GITHUB_REF}" in + refs/tags/nightly/v*) + # Lane tags carry the exact published version; stamp it verbatim + # instead of describing drift from the nearest stable tag. + version="${GITHUB_REF#refs/tags/nightly/v}" + ;; + refs/tags/beta/v*) + version="${GITHUB_REF#refs/tags/beta/v}" + ;; + *) + version="$(git describe --tags --match 'v*' --long --dirty 2>/dev/null || true)" + ;; + esac echo "version=${version}" >> "$GITHUB_OUTPUT" echo "Stamping build version: ${version:-}" @@ -288,8 +325,9 @@ jobs: echo "last=${last}" >> "$GITHUB_OUTPUT" echo "count=${count}" >> "$GITHUB_OUTPUT" - # Published under the same tag set with a `-cloud` suffix - # (sha--cloud, latest-cloud, -cloud). + # Published under the same lane tag set as the self-hosted image, with a + # `-cloud` suffix (canary-cloud, nightly-cloud, latest-cloud, + # -cloud, sha--cloud). - name: Docker meta (cloud) id: meta-cloud uses: docker/metadata-action@v6 @@ -298,9 +336,12 @@ jobs: flavor: | suffix=-cloud,onlatest=true tags: | - type=raw,value=latest,enable={{is_default_branch}} - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} + type=raw,value=canary,enable=${{ github.ref == 'refs/heads/master' }} + type=raw,value=nightly,enable=${{ startsWith(github.ref, 'refs/tags/nightly/v') }} + type=raw,value=beta,enable=${{ startsWith(github.ref, 'refs/tags/beta/v') }} + type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v') }} + type=semver,pattern={{version}},enable=${{ startsWith(github.ref, 'refs/tags/v') }} + type=semver,pattern={{major}}.{{minor}},enable=${{ startsWith(github.ref, 'refs/tags/v') }} type=sha labels: | io.github.paperclipai.schema.last-migration=${{ steps.schema.outputs.last }} diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index ef4aa39dc60..ad57d60d866 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -183,8 +183,20 @@ jobs: group_label: server (5/5) shard_index: 4 shard_count: 5 + # workspaces-a was the slowest check in the fully-green PR run + # 31371439296 (2026-08-10) at 319s, with the ui project's single + # vitest invocation accounting for ~224s and the paperclipai CLI + # ~37s. Two shards use Vitest's native --shard on each project's + # file list (ui: 439 files, cli: 54), bringing each job to roughly + # half the suite time (~130s + setup) without a duration manifest. - group: general-workspaces-a - group_label: workspaces-a + group_label: workspaces-a (1/2) + shard_index: 0 + shard_count: 2 + - group: general-workspaces-a + group_label: workspaces-a (2/2) + shard_index: 1 + shard_count: 2 - group: general-workspaces-b group_label: workspaces-b diff --git a/.github/workflows/release-smoke.yml b/.github/workflows/release-smoke.yml index 923bb896a3e..5ec66da8a69 100644 --- a/.github/workflows/release-smoke.yml +++ b/.github/workflows/release-smoke.yml @@ -10,6 +10,8 @@ on: type: choice options: - canary + - nightly + - beta - latest host_port: description: Host port for the Docker smoke container @@ -69,6 +71,7 @@ jobs: HOST_PORT="${{ inputs.host_port }}" \ DATA_DIR="$RUNNER_TEMP/release-smoke-data" \ PAPERCLIPAI_VERSION="${{ inputs.paperclip_version }}" \ + SMOKE_READY_TIMEOUT_SECONDS=420 \ SMOKE_DETACH=true \ SMOKE_METADATA_FILE="$metadata_file" \ ./scripts/docker-onboard-smoke.sh diff --git a/.github/workflows/release-verify.yml b/.github/workflows/release-verify.yml index cac52d75b5d..df146e7faac 100644 --- a/.github/workflows/release-verify.yml +++ b/.github/workflows/release-verify.yml @@ -64,8 +64,16 @@ jobs: group_label: server (3/3) shard_index: 2 shard_count: 3 + # Keep parity with pr.yml: workspaces-a is split with Vitest's + # native --shard because the ui project dominates the lane. - group: general-workspaces-a - group_label: workspaces-a + group_label: workspaces-a (1/2) + shard_index: 0 + shard_count: 2 + - group: general-workspaces-a + group_label: workspaces-a (2/2) + shard_index: 1 + shard_count: 2 - group: general-workspaces-b group_label: workspaces-b diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d46cf1fb594..c8e74ea146f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,10 +4,22 @@ on: push: branches: - master + schedule: + # Nightly cut at 09:00 UTC, after the workday's merges have settled. + - cron: "0 9 * * *" workflow_dispatch: inputs: + channel: + description: Release channel to publish + required: true + type: choice + options: + - stable + - beta + - nightly + default: stable source_ref: - description: Commit SHA, branch, or tag to publish as stable + description: (stable) Commit SHA, branch, or tag to publish as stable required: true type: string default: master @@ -15,8 +27,16 @@ on: description: Enter a UTC date in YYYY-MM-DD format, for example 2026-03-18. Do not enter a version string. The workflow will resolve that date to a stable version such as 2026.318.0, then 2026.318.1 for the next same-day stable. required: false type: string + source_version: + description: For nightly, the explicit canary version to promote (empty selects the newest canary on master). For beta, the explicit nightly version to promote (empty selects the newest nightly on master). + required: false + type: string + skip_soak_justification: + description: (stable) Written justification for publishing a stable whose source has not soaked as a beta for 3 days. Leave empty for normal releases. + required: false + type: string dry_run: - description: Preview the stable release without publishing + description: Preview the release without publishing required: true type: boolean default: false @@ -97,15 +117,526 @@ jobs: fi git push origin "refs/tags/${tag}" + # ----- Nightly lane ----------------------------------------------------- + # Once a night (or on a forced nightly dispatch), promote the newest master + # commit that already shipped a green canary: smoke-test that exact + # published canary first, then republish the same commit under the nightly + # identity. The candidate commit already passed release-verify during its + # canary publish, so the nightly publish skips re-verification. + + select_nightly: + if: github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.channel == 'nightly') + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + outputs: + proceed: ${{ steps.select.outputs.proceed }} + sha: ${{ steps.select.outputs.sha }} + canary_version: ${{ steps.select.outputs.canary_version }} + steps: + - name: Checkout repository + uses: actions/checkout@v7 + with: + ref: master + fetch-depth: 0 + + - name: Select nightly candidate + id: select + env: + EXPLICIT_CANARY_VERSION: ${{ github.event_name == 'workflow_dispatch' && inputs.source_version || '' }} + run: | + set -euo pipefail + + git fetch origin --tags --prune --quiet + + skip() { + echo "proceed=false" >> "$GITHUB_OUTPUT" + { + echo "## Nightly skipped" + echo "" + echo "$1" + } >> "$GITHUB_STEP_SUMMARY" + echo "Nightly skipped: $1" + } + + if [ -n "${EXPLICIT_CANARY_VERSION:-}" ]; then + tag="canary/v${EXPLICIT_CANARY_VERSION}" + sha="$(git rev-list -n 1 "$tag" 2>/dev/null || true)" + if [ -z "$sha" ]; then + echo "Error: tag $tag does not exist." >&2 + exit 1 + fi + else + # Newest canary-tagged commit on master. Canary tags are pushed + # only after a successful canary publish, so tag presence is the + # green-publish signal. Walk master newest-first and stop at the + # first commit that carries a canary tag. + sha="$(grep -m1 -F \ + -f <(git for-each-ref 'refs/tags/canary/v*' --format='%(objectname)') \ + <(git rev-list origin/master -n 500) || true)" + + if [ -z "$sha" ]; then + skip "No canary/v* tag found on the last 500 commits of master." + exit 0 + fi + + tag="$(git tag --points-at "$sha" | grep '^canary/v' | sort -V | tail -1)" + fi + + canary_version="${tag#canary/v}" + + existing_nightly="$(git tag --points-at "$sha" | grep '^nightly/v' | head -1 || true)" + if [ -n "$existing_nightly" ]; then + skip "Candidate \`$sha\` (canary \`$canary_version\`) already shipped as \`$existing_nightly\`." + exit 0 + fi + + # Promotions run the release tooling of the source commit, so the + # source must already understand the nightly channel. (Literal match + # of release.sh's channel case arm; if that line is reformatted this + # fails closed and should be updated alongside it.) + if ! git show "${sha}:scripts/release.sh" | grep -qF 'canary|nightly'; then + echo "Error: source commit $sha predates nightly release tooling; promote a newer canary." >&2 + exit 1 + fi + + echo "proceed=true" >> "$GITHUB_OUTPUT" + echo "sha=$sha" >> "$GITHUB_OUTPUT" + echo "canary_version=$canary_version" >> "$GITHUB_OUTPUT" + { + echo "## Nightly candidate" + echo "" + echo "- Source SHA: \`$sha\`" + echo "- Source canary: \`$canary_version\`" + } >> "$GITHUB_STEP_SUMMARY" + + # Gate the promotion on the release smoke suite, run against the exact + # published canary artifact that would become tonight's nightly. Red smoke + # means no nightly tonight. Skipped for dry-run dispatches. + smoke_nightly: + needs: select_nightly + if: needs.select_nightly.outputs.proceed == 'true' && !(github.event_name == 'workflow_dispatch' && inputs.dry_run) + uses: ./.github/workflows/release-smoke.yml + with: + paperclip_version: ${{ needs.select_nightly.outputs.canary_version }} + artifact_name: nightly-release-smoke + + publish_nightly: + needs: [select_nightly, smoke_nightly] + # Publish when smoke passed, or when smoke was deliberately skipped by a + # dry-run dispatch (the publish itself is a dry-run in that case). + if: >- + !cancelled() && + needs.select_nightly.outputs.proceed == 'true' && + (needs.smoke_nightly.result == 'success' || + (needs.smoke_nightly.result == 'skipped' && github.event_name == 'workflow_dispatch' && inputs.dry_run)) + runs-on: ubuntu-latest + timeout-minutes: 45 + environment: npm-canary + # The workflow-level concurrency group is per event, so a forced dispatch + # nightly could otherwise overlap the scheduled one and race it to the + # same next -nightly.N version. Serialize actual nightly publishes across + # events here; release.sh additionally refuses to double-publish a commit + # that already carries a nightly tag. + concurrency: + group: release-publish-nightly + cancel-in-progress: false + permissions: + contents: write + id-token: write + actions: write + + steps: + - name: Checkout repository + uses: actions/checkout@v7 + with: + ref: ${{ needs.select_nightly.outputs.sha }} + fetch-depth: 0 + + - name: Setup pnpm + uses: pnpm/action-setup@v6 + with: + version: 9.15.4 + + - name: Setup Node.js + uses: actions/setup-node@v7 + with: + node-version: 24 + cache: pnpm + + - name: Validate release package manifest + run: node ./scripts/release-package-map.mjs check + + - name: Install dependencies + run: pnpm install --no-frozen-lockfile + + - name: Restore tracked install-time changes + run: git checkout -- pnpm-lock.yaml + + - name: Configure git author + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + + - name: Publish nightly + env: + GITHUB_ACTIONS: "true" + run: | + args=(nightly --skip-verify) + if [ "${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }}" = "true" ]; then + args+=(--dry-run) + fi + ./scripts/release.sh "${args[@]}" + + - name: Dump npm debug logs + if: failure() + run: | + shopt -s nullglob + for f in "$HOME"/.npm/_logs/*.log; do + echo "===== $f =====" + tail -n 300 "$f" | sed -E \ + -e 's#((authorization|_authToken|_auth|node_auth_token|npm_token)"?[[:space:]]*[:=][[:space:]]*"?)(Bearer[[:space:]]+)?[^",[:space:]]+#\1***REDACTED***#Ig' + done + + - name: Push nightly tag + if: ${{ !(github.event_name == 'workflow_dispatch' && inputs.dry_run) }} + run: | + tag="$(git tag --points-at HEAD | grep '^nightly/v' | head -1)" + if [ -z "$tag" ]; then + echo "Error: no nightly tag points at HEAD after release." >&2 + exit 1 + fi + git push origin "refs/tags/${tag}" + + # Tag pushes made with GITHUB_TOKEN do not fire docker.yml's tag + # trigger (GitHub suppresses workflow runs caused by GITHUB_TOKEN + # pushes), so dispatch the image build at the new tag explicitly. + - name: Build Docker images for the nightly tag + if: ${{ !(github.event_name == 'workflow_dispatch' && inputs.dry_run) }} + env: + GH_TOKEN: ${{ github.token }} + run: | + tag="$(git tag --points-at HEAD | grep '^nightly/v' | head -1)" + { + echo "## Nightly published" + echo "" + echo "- Source SHA: \`${{ needs.select_nightly.outputs.sha }}\`" + echo "- Source canary: \`${{ needs.select_nightly.outputs.canary_version }}\`" + echo "- Published nightly: \`${tag#nightly/v}\`" + echo "- Docker build dispatched at \`${tag}\`" + } >> "$GITHUB_STEP_SUMMARY" + gh workflow run docker.yml --ref "refs/tags/${tag}" --repo "$GITHUB_REPOSITORY" + + # ----- Beta lane -------------------------------------------------------- + # Beta is a manual, human-approved promotion of a nightly. Unlike the + # scheduled nightly lane, a beta dispatch is explicit operator intent, so + # selection problems fail the run loudly instead of skipping quietly. The + # publish runs behind the npm-beta environment, whose required reviewers + # are the approval gate. + + select_beta: + if: github.event_name == 'workflow_dispatch' && inputs.channel == 'beta' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + outputs: + sha: ${{ steps.select.outputs.sha }} + nightly_version: ${{ steps.select.outputs.nightly_version }} + steps: + - name: Checkout repository + uses: actions/checkout@v7 + with: + ref: master + fetch-depth: 0 + + - name: Select beta candidate + id: select + env: + EXPLICIT_NIGHTLY_VERSION: ${{ inputs.source_version }} + run: | + set -euo pipefail + + git fetch origin --tags --prune --quiet + + if [ -n "${EXPLICIT_NIGHTLY_VERSION:-}" ]; then + tag="nightly/v${EXPLICIT_NIGHTLY_VERSION}" + sha="$(git rev-list -n 1 "$tag" 2>/dev/null || true)" + if [ -z "$sha" ]; then + echo "Error: tag $tag does not exist." >&2 + exit 1 + fi + else + # Newest nightly-tagged commit on master. + sha="$(grep -m1 -F \ + -f <(git for-each-ref 'refs/tags/nightly/v*' --format='%(objectname)') \ + <(git rev-list origin/master -n 2000) || true)" + + if [ -z "$sha" ]; then + echo "Error: no nightly/v* tag found on the last 2000 commits of master. Publish a nightly first, or pass source_version." >&2 + exit 1 + fi + + tag="$(git tag --points-at "$sha" | grep '^nightly/v' | sort -V | tail -1)" + fi + + nightly_version="${tag#nightly/v}" + + existing_beta="$(git tag --points-at "$sha" | grep '^beta/v' | head -1 || true)" + if [ -n "$existing_beta" ]; then + echo "Error: candidate $sha (nightly $nightly_version) already shipped as $existing_beta." >&2 + exit 1 + fi + + # Promotions run the release tooling of the source commit, so the + # source must already understand the beta channel. (Literal match of + # release.sh's channel case arm; if that line is reformatted this + # fails closed and should be updated alongside it.) + if ! git show "${sha}:scripts/release.sh" | grep -qF 'canary|nightly|beta|stable)'; then + echo "Error: source commit $sha predates beta release tooling; promote a newer nightly whose source contains the beta channel." >&2 + exit 1 + fi + + echo "sha=$sha" >> "$GITHUB_OUTPUT" + echo "nightly_version=$nightly_version" >> "$GITHUB_OUTPUT" + { + echo "## Beta candidate" + echo "" + echo "- Source SHA: \`$sha\`" + echo "- Source nightly: \`$nightly_version\`" + } >> "$GITHUB_STEP_SUMMARY" + + publish_beta: + needs: select_beta + runs-on: ubuntu-latest + timeout-minutes: 45 + environment: npm-beta + # Serialize beta publishes so two dispatches cannot race to the same next + # -beta.N version; release.sh additionally refuses to double-publish a + # commit that already carries a beta tag. + concurrency: + group: release-publish-beta + cancel-in-progress: false + permissions: + contents: write + id-token: write + actions: write + outputs: + beta_version: ${{ steps.result.outputs.beta_version }} + + steps: + - name: Checkout repository + uses: actions/checkout@v7 + with: + ref: ${{ needs.select_beta.outputs.sha }} + fetch-depth: 0 + + - name: Setup pnpm + uses: pnpm/action-setup@v6 + with: + version: 9.15.4 + + - name: Setup Node.js + uses: actions/setup-node@v7 + with: + node-version: 24 + cache: pnpm + + - name: Validate release package manifest + run: node ./scripts/release-package-map.mjs check + + - name: Install dependencies + run: pnpm install --no-frozen-lockfile + + - name: Restore tracked install-time changes + run: git checkout -- pnpm-lock.yaml + + - name: Configure git author + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + + - name: Publish beta + env: + GITHUB_ACTIONS: "true" + run: | + args=(beta --skip-verify) + if [ "${{ inputs.dry_run }}" = "true" ]; then + args+=(--dry-run) + fi + ./scripts/release.sh "${args[@]}" + + - name: Dump npm debug logs + if: failure() + run: | + shopt -s nullglob + for f in "$HOME"/.npm/_logs/*.log; do + echo "===== $f =====" + tail -n 300 "$f" | sed -E \ + -e 's#((authorization|_authToken|_auth|node_auth_token|npm_token)"?[[:space:]]*[:=][[:space:]]*"?)(Bearer[[:space:]]+)?[^",[:space:]]+#\1***REDACTED***#Ig' + done + + - name: Push beta tag + if: ${{ !inputs.dry_run }} + run: | + tag="$(git tag --points-at HEAD | grep '^beta/v' | head -1)" + if [ -z "$tag" ]; then + echo "Error: no beta tag points at HEAD after release." >&2 + exit 1 + fi + git push origin "refs/tags/${tag}" + + # Tag pushes made with GITHUB_TOKEN do not fire docker.yml's tag + # trigger (GitHub suppresses workflow runs caused by GITHUB_TOKEN + # pushes), so dispatch the image build at the new tag explicitly. + - name: Build Docker images for the beta tag + id: result + if: ${{ !inputs.dry_run }} + env: + GH_TOKEN: ${{ github.token }} + run: | + tag="$(git tag --points-at HEAD | grep '^beta/v' | head -1)" + echo "beta_version=${tag#beta/v}" >> "$GITHUB_OUTPUT" + { + echo "## Beta published" + echo "" + echo "- Source SHA: \`${{ needs.select_beta.outputs.sha }}\`" + echo "- Source nightly: \`${{ needs.select_beta.outputs.nightly_version }}\`" + echo "- Published beta: \`${tag#beta/v}\`" + echo "- Docker build dispatched at \`${tag}\`" + } >> "$GITHUB_STEP_SUMMARY" + gh workflow run docker.yml --ref "refs/tags/${tag}" --repo "$GITHUB_REPOSITORY" + + # Post-publish verification: run the release smoke suite against the exact + # beta version that was just published. + smoke_beta: + needs: publish_beta + if: ${{ !inputs.dry_run }} + uses: ./.github/workflows/release-smoke.yml + with: + paperclip_version: ${{ needs.publish_beta.outputs.beta_version }} + artifact_name: beta-release-smoke + + # ----- Stable lane ------------------------------------------------------ + + # Stable releases promote a soaked beta. The preflight enforces that the + # source commit shipped as a beta at least 3 days ago (measured from the + # npm publish time of that beta version), unless a written justification + # is provided. Dry runs report soak state without blocking. + # Resolves source_ref to an immutable commit exactly once; every downstream + # stable job consumes that SHA. Otherwise a branch or movable tag that + # advances mid-run could be soak-checked at one commit and verified or + # published at another. + preflight_stable: + if: github.event_name == 'workflow_dispatch' && inputs.channel == 'stable' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + outputs: + sha: ${{ steps.soak.outputs.sha }} + steps: + - name: Checkout repository + uses: actions/checkout@v7 + with: + ref: master + fetch-depth: 0 + + - name: Check beta soak + id: soak + env: + SOURCE_REF: ${{ inputs.source_ref }} + JUSTIFICATION: ${{ inputs.skip_soak_justification }} + DRY_RUN: ${{ inputs.dry_run }} + run: | + set -euo pipefail + + git fetch origin --tags --prune --quiet + + sha="$(git rev-parse --verify "${SOURCE_REF}^{commit}" 2>/dev/null || true)" + if [ -z "$sha" ]; then + git fetch origin "$SOURCE_REF" --quiet || true + sha="$(git rev-parse --verify "FETCH_HEAD^{commit}" 2>/dev/null || true)" + fi + if [ -z "$sha" ]; then + echo "Error: could not resolve source_ref '$SOURCE_REF' to a commit." >&2 + exit 1 + fi + + echo "sha=$sha" >> "$GITHUB_OUTPUT" + { + echo "## Stable source pinned" + echo "" + echo "- source_ref: \`${SOURCE_REF}\` -> \`$sha\`" + } >> "$GITHUB_STEP_SUMMARY" + + fail_or_justify() { + if [ -n "${JUSTIFICATION:-}" ]; then + { + echo "## Stable soak gate bypassed" + echo "" + echo "$1" + echo "" + echo "Justification: ${JUSTIFICATION}" + } >> "$GITHUB_STEP_SUMMARY" + echo "::warning::Soak gate bypassed: $1" + return 0 + fi + if [ "${DRY_RUN}" = "true" ]; then + echo "::warning::Soak gate would block a real release: $1" + { + echo "## Stable soak gate (dry run)" + echo "" + echo "A real release would be blocked: $1" + } >> "$GITHUB_STEP_SUMMARY" + return 0 + fi + echo "Error: $1" >&2 + echo "Pass skip_soak_justification with a written reason to release anyway." >&2 + exit 1 + } + + beta_tag="$(git tag --points-at "$sha" | grep '^beta/v' | sort -V | tail -1 || true)" + if [ -z "$beta_tag" ]; then + fail_or_justify "source commit $sha never shipped as a beta (no beta/v* tag)." + exit 0 + fi + + beta_version="${beta_tag#beta/v}" + publish_time="$(npm view "paperclipai@${beta_version}" time --json 2>/dev/null \ + | node -e 'let d="";process.stdin.on("data",c=>d+=c).on("end",()=>{const t=JSON.parse(d);process.stdout.write(typeof t === "string" ? t : (t[process.argv[1]] ?? ""))})' "$beta_version" || true)" + if [ -z "$publish_time" ]; then + fail_or_justify "could not determine the npm publish time of beta ${beta_version}." + exit 0 + fi + + age_seconds="$(node -e 'process.stdout.write(String(Math.floor((Date.now() - Date.parse(process.argv[1])) / 1000)))' "$publish_time")" + min_seconds=$((3 * 24 * 60 * 60)) + age_days="$(node -e 'process.stdout.write((Number(process.argv[1]) / 86400).toFixed(1))' "$age_seconds")" + + if [ "$age_seconds" -lt "$min_seconds" ]; then + fail_or_justify "beta ${beta_version} has only soaked ${age_days} days (minimum is 3)." + exit 0 + fi + + { + echo "## Stable soak gate passed" + echo "" + echo "- Source beta: \`${beta_version}\`" + echo "- Soak time: ${age_days} days" + } >> "$GITHUB_STEP_SUMMARY" + verify_stable: - if: github.event_name == 'workflow_dispatch' + if: github.event_name == 'workflow_dispatch' && inputs.channel == 'stable' + needs: preflight_stable uses: ./.github/workflows/release-verify.yml with: - ref: ${{ inputs.source_ref }} + ref: ${{ needs.preflight_stable.outputs.sha }} preview_stable: - if: github.event_name == 'workflow_dispatch' && inputs.dry_run - needs: verify_stable + if: github.event_name == 'workflow_dispatch' && inputs.channel == 'stable' && inputs.dry_run + needs: [preflight_stable, verify_stable] runs-on: ubuntu-latest timeout-minutes: 45 permissions: @@ -116,7 +647,7 @@ jobs: uses: actions/checkout@v7 with: fetch-depth: 0 - ref: ${{ inputs.source_ref }} + ref: ${{ needs.preflight_stable.outputs.sha }} - name: Setup pnpm uses: pnpm/action-setup@v6 @@ -146,21 +677,22 @@ jobs: ./scripts/release.sh "${args[@]}" publish_stable: - if: github.event_name == 'workflow_dispatch' && !inputs.dry_run - needs: verify_stable + if: github.event_name == 'workflow_dispatch' && inputs.channel == 'stable' && !inputs.dry_run + needs: [preflight_stable, verify_stable] runs-on: ubuntu-latest timeout-minutes: 45 environment: npm-stable permissions: contents: write id-token: write + actions: write steps: - name: Checkout repository uses: actions/checkout@v7 with: fetch-depth: 0 - ref: ${{ inputs.source_ref }} + ref: ${{ needs.preflight_stable.outputs.sha }} - name: Setup pnpm uses: pnpm/action-setup@v6 @@ -213,6 +745,25 @@ jobs: fi git push origin "refs/tags/${tag}" + # Tag pushes made with GITHUB_TOKEN do not fire docker.yml's tag + # trigger (GitHub suppresses workflow runs caused by GITHUB_TOKEN + # pushes), so dispatch the image build at the new tag explicitly. This + # is what moves Docker `:latest` and publishes the versioned stable + # image tags. + - name: Build Docker images for the stable tag + env: + GH_TOKEN: ${{ github.token }} + run: | + tag="$(git tag --points-at HEAD | grep '^v' | head -1)" + if gh workflow run docker.yml --ref "refs/tags/${tag}" --repo "$GITHUB_REPOSITORY"; then + echo "Dispatched docker.yml at ${tag}." + else + # Older source commits may predate docker.yml's workflow_dispatch + # trigger; the dispatch then fails while the npm release is + # already complete and correct. + echo "::warning::Could not dispatch docker.yml at ${tag}. Run docker.yml manually at that tag to publish the stable images." + fi + - name: Create GitHub Release env: GH_TOKEN: ${{ github.token }} diff --git a/doc/CHANNELS.md b/doc/CHANNELS.md new file mode 100644 index 00000000000..b99ae33a3d8 --- /dev/null +++ b/doc/CHANNELS.md @@ -0,0 +1,89 @@ +# Release Channels + +Paperclip ships on four channels. Pick the one that matches your appetite for +freshness versus stability — switching is just a matter of which version you +install. + +| Channel | What it is | Updates | npm | Docker | +| --- | --- | --- | --- | --- | +| `stable` | The recommended release | every week or two | `paperclipai@latest` | `ghcr.io/paperclipai/paperclip:latest` | +| `beta` | Release candidates soaking before stable | when promoted | `paperclipai@beta` | `ghcr.io/paperclipai/paperclip:beta` | +| `nightly` | Yesterday's merges, smoke-tested as a unit | once a night | `paperclipai@nightly` | `ghcr.io/paperclipai/paperclip:nightly` | +| `canary` | Every merge to `master`, as it happens | many times a day | `paperclipai@canary` | `ghcr.io/paperclipai/paperclip:canary` | + +## Choosing a channel + +**stable** is the right choice for almost everyone. It only moves when a +release has been explicitly vetted and promoted by a maintainer, and every +stable must first soak as a beta for at least 3 days. + +**beta** is for people who want the next stable early. A beta is a nightly +that a maintainer hand-picked and explicitly promoted behind an approval +gate, and it is re-smoked after publishing. Betas are the release candidates: +what you run on beta today is what stable becomes a few days later. + +**nightly** is for people who want new features quickly but not the churn of +tracking every merge. Once a night, the newest master build that published +green is run through the full release smoke suite (real Docker container, real +onboarding flow, browser-driven). Only if that passes does it ship as the +nightly. If smoke fails, there is no nightly that night — the channel never +ships a build that failed its checks. + +**canary** is the bleeding edge: it publishes on every merge to `master`. +It is primarily the lane that continuously exercises our release automation, +but it's available to anyone who wants the newest bits and accepts the risk. + +## Installing from a channel + +npm / npx: + +```bash +npx paperclipai@latest onboard # stable +npx paperclipai@beta onboard +npx paperclipai@nightly onboard +npx paperclipai@canary onboard +``` + +Docker: + +```bash +docker pull ghcr.io/paperclipai/paperclip:latest # stable +docker pull ghcr.io/paperclipai/paperclip:beta +docker pull ghcr.io/paperclipai/paperclip:nightly +docker pull ghcr.io/paperclipai/paperclip:canary +``` + +Every image is also published as `:sha-` for exact pinning, and +stable images additionally get `:YYYY.MDD.P` version tags. + +## Switching channels + +Channel choice is per-install: install from a different tag and you're on that +channel. Moving forward (stable → nightly) is always safe. Moving backward +(nightly → stable) can mean running an older schema than your data was created +with — treat a downgrade like a restore and keep a backup of your data +directory before switching down. + +## Reading version strings + +The version tells you which channel a build came from: + +- `2026.807.0` — stable, published Aug 7 2026 +- `2026.807.0-beta.0` — beta promoted on Aug 7 2026 +- `2026.807.0-nightly.0` — nightly cut on Aug 7 2026 +- `2026.807.0-canary.4` — the fifth canary for the Aug 7 line + +Each promotion republishes the exact source commit of the previous lane's +build: a nightly shares its source SHA with a canary, and a beta with a +nightly. The version dates the promotion, and the shared SHA is visible in +the release job summaries and as git tags on the commit. + +One quirk to be aware of: npm's semver ordering compares prerelease names +alphabetically, so `-beta.N` sorts below `-canary.N`, which sorts below +`-nightly.N` for the same base version. This never matters when installing by +dist-tag (the recommended way), only if you write version ranges by hand. + +## For maintainers + +The publishing mechanics, promotion flow, and release checklist live in +[`RELEASING.md`](RELEASING.md). diff --git a/doc/RELEASE-AUTOMATION-SETUP.md b/doc/RELEASE-AUTOMATION-SETUP.md index d21975c106f..e8131595f8b 100644 --- a/doc/RELEASE-AUTOMATION-SETUP.md +++ b/doc/RELEASE-AUTOMATION-SETUP.md @@ -113,9 +113,10 @@ Goal: ## 4. Create GitHub Environments -Create two environments in the GitHub repository: +Create three environments in the GitHub repository: - `npm-canary` +- `npm-beta` - `npm-stable` Path: @@ -141,6 +142,35 @@ Reasoning: - every push to `master` should be able to publish a canary automatically - no human approval should be required for canaries +The scheduled nightly lane also publishes under `npm-canary`: it is the same +trust level (fully automated, no human gate), its runs execute on `master` so +the branch rule is satisfied, and reusing the environment means the nightly +lane required no new environments and no npm trusted-publisher changes +(publishing still happens from `release.yml`, see section 2.2). + +## 5.1. Configure `npm-beta` + +Recommended settings for `npm-beta`: + +- environment name: `npm-beta` +- required reviewers: at least one maintainer +- prevent self-review: enabled when your team size allows it +- wait timer: none +- deployment branches and tags: + - selected branches only + - allow `master` + +Reasoning: + +- beta promotions are deliberate human decisions; the required reviewer on + this environment is the promotion gate +- create this environment before the first `channel: beta` dispatch. If the + workflow runs first, GitHub auto-creates the environment with no + protection rules, and that first beta would publish without approval + +Like nightly, beta publishing lives in `release.yml`, so no npm +trusted-publisher changes are needed (see section 2.2). + ## 6. Configure `npm-stable` Recommended settings for `npm-stable`: diff --git a/doc/RELEASING.md b/doc/RELEASING.md index da4143dfae6..a77ca18729f 100644 --- a/doc/RELEASING.md +++ b/doc/RELEASING.md @@ -5,9 +5,15 @@ Maintainer runbook for shipping Paperclip across npm, GitHub, and the website-fa The release model is now commit-driven: 1. Every push to `master` publishes a canary automatically. -2. Stable releases are manually promoted from a chosen tested commit or canary tag. -3. Stable release notes live in `releases/vYYYY.MDD.P.md`. -4. Only stable releases get GitHub Releases. +2. Once a night, the newest master commit with a green canary publish is + smoke-tested and republished as the nightly. +3. Betas are manual, human-approved promotions of a chosen nightly. +4. Stable releases promote a beta that has soaked for at least 3 days + (bypass requires a written justification). +5. Stable release notes live in `releases/vYYYY.MDD.P.md`. +6. Only stable releases get GitHub Releases. + +The user-facing guide to the channels is [`CHANNELS.md`](CHANNELS.md). ## Versioning Model @@ -15,12 +21,20 @@ Paperclip uses calendar versions that still fit semver syntax: - stable: `YYYY.MDD.P` - canary: `YYYY.MDD.P-canary.N` +- nightly: `YYYY.MDD.P-nightly.N` +- beta: `YYYY.MDD.P-beta.N` Examples: - first stable on March 18, 2026: `2026.318.0` - second stable on March 18, 2026: `2026.318.1` - fourth canary for the `2026.318.1` line: `2026.318.1-canary.3` +- first nightly cut on March 18, 2026: `2026.318.1-nightly.0` +- first beta promoted on March 18, 2026: `2026.318.1-beta.0` + +A promotion republishes the exact source commit of the previous lane's build +(canary → nightly → beta); the version dates the promotion, not the source +build. Important constraints: @@ -41,16 +55,27 @@ Every stable release has four separate surfaces: A stable release is done only when all four surfaces are handled. -Canaries only cover the first two surfaces plus an internal traceability tag. +Canaries, nightlies, and betas only cover the first two surfaces plus an +internal traceability tag. ## Core Invariants - canaries publish from `master` -- stables publish from an explicitly chosen source ref +- nightlies republish a commit that already shipped a canary (the commit must + carry a `canary/v*` tag), and only after the release smoke suite passes + against that exact published canary +- betas republish a commit that already shipped a nightly (the commit must + carry a `nightly/v*` tag), behind the `npm-beta` approval gate, and the + published beta is re-smoked +- stables publish from an explicitly chosen source ref, which must have + shipped as a beta at least 3 days earlier unless a written justification + is provided - tags point at the original source commit, not a generated release commit - stable notes are always `releases/vYYYY.MDD.P.md` -- canaries never create GitHub Releases -- canaries never require changelog generation +- canaries, nightlies, and betas never create GitHub Releases +- canaries, nightlies, and betas never require changelog generation +- Docker `:latest` moves only on stable releases; master builds publish + `:canary`, nightly builds `:nightly`, and beta builds `:beta` ## TL;DR @@ -78,6 +103,69 @@ npx paperclipai@canary onboard npx paperclipai@canary onboard --data-dir "$(mktemp -d /tmp/paperclip-canary.XXXXXX)" ``` +### Nightly + +A scheduled job in [`.github/workflows/release.yml`](../.github/workflows/release.yml) +runs once a night at 09:00 UTC. + +It: + +- selects the newest commit on `master` that carries a `canary/v*` tag (the + tag is pushed only after a successful canary publish, so it is the + green-publish signal) +- skips with a job-summary reason when there is no new candidate or the + candidate already shipped as a nightly +- runs the release smoke suite ([`release-smoke.yml`](../.github/workflows/release-smoke.yml)) + against that exact published canary version — red smoke means no nightly + tonight +- republishes the same source commit as `YYYY.MDD.P-nightly.N` under the npm + dist-tag `nightly` (the commit was already verified by its canary run, so + verification is not repeated) +- creates and pushes the git tag `nightly/vYYYY.MDD.P-nightly.N` +- dispatches [`docker.yml`](../.github/workflows/docker.yml) at that tag to + publish the `:nightly` images + +To force a nightly outside the schedule (recovery, or promoting a specific +canary), dispatch `release.yml` with `channel: nightly`. Leave +`source_version` empty for automatic selection, or set it to an exact +canary version. `dry_run: true` previews the publish and skips smoke, the tag +push, and the Docker dispatch. + +Users install nightlies with: + +```bash +npx paperclipai@nightly onboard +``` + +### Beta + +Betas are manual promotions. Dispatch +[`release.yml`](../.github/workflows/release.yml) with `channel: beta`. + +- leave `source_version` empty to promote the newest nightly, or set it to an + exact nightly version such as `2026.807.0-nightly.0` +- the selection job resolves the nightly's source commit and fails loudly if + it does not exist or already shipped as a beta +- the publish waits for approval in the **`npm-beta` environment** — its + required reviewers are the promotion gate +- promotions run the release tooling of the source commit, so the source + nightly must postdate the beta channel's introduction; the selection job + rejects older sources with a clear error (in practice every nightly cut + after the beta tooling merged qualifies) +- the same commit is republished as `YYYY.MDD.P-beta.N` under the npm + dist-tag `beta`, tagged `beta/vYYYY.MDD.P-beta.N`, and `docker.yml` is + dispatched at that tag to publish the `:beta` images +- after publishing, the release smoke suite runs against the exact published + beta version as verification +- `dry_run: true` previews the publish and skips the tag push, Docker + dispatch, and post-publish smoke + +Users install betas with: + +```bash +npx paperclipai@beta onboard +``` + ### Stable Use [`.github/workflows/release.yml`](../.github/workflows/release.yml) from the Actions tab with the manual `workflow_dispatch` inputs. @@ -86,20 +174,32 @@ Use [`.github/workflows/release.yml`](../.github/workflows/release.yml) from the Inputs: +- `channel` + - `stable` (the default) for a stable release; `beta` and `nightly` run + those lanes instead (see above) - `source_ref` - commit SHA, branch, or tag - `stable_date` - optional UTC date override in `YYYY-MM-DD` - enter a date like `2026-03-18`, not a version like `2026.318.0` +- `skip_soak_justification` + - written reason for releasing a stable whose source has not soaked as a + beta for 3 days; leave empty for normal releases - `dry_run` - preview only when true +The stable preflight enforces the beta soak: the source commit must carry a +`beta/v*` tag whose npm publish time is at least 3 days old. If it is not, +the run fails unless `skip_soak_justification` is provided; the justification +is echoed into the job summary. Dry runs report soak state without blocking. + Before running stable: -1. pick the canary commit or tag you trust -2. resolve the target stable version with `./scripts/release.sh stable --date "$(date +%F)" --print-version` -3. create or update `releases/vYYYY.MDD.P.md` on that source ref -4. run the stable workflow from that source ref +1. pick the beta you are promoting (its source commit is the `source_ref`) +2. confirm the beta has soaked for 3 days with no open blockers +3. resolve the target stable version with `./scripts/release.sh stable --date "$(date +%F)" --print-version` +4. create or update `releases/vYYYY.MDD.P.md` on that source ref +5. run the stable workflow from that source ref Example: @@ -113,8 +213,27 @@ The workflow: - computes the next stable patch slot for the chosen UTC date - publishes `YYYY.MDD.P` under npm dist-tag `latest` - creates git tag `vYYYY.MDD.P` +- dispatches [`docker.yml`](../.github/workflows/docker.yml) at that tag to + publish `:latest` and the versioned stable images - creates or updates the GitHub Release from `releases/vYYYY.MDD.P.md` +## Docker Image Tags + +[`docker.yml`](../.github/workflows/docker.yml) publishes both the self-hosted +image and the `-cloud` variant with the same lane mapping: + +| Build ref | Tags | +| --- | --- | +| `master` push | `:canary`, `:sha-` | +| `nightly/v*` tag | `:nightly`, `:sha-` | +| `beta/v*` tag | `:beta`, `:sha-` | +| `v*` tag (stable) | `:latest`, `:YYYY.MDD.P`, `:YYYY.MDD`, `:sha-` | + +Lane tags are pushed by release workflows using `GITHUB_TOKEN`, and GitHub +suppresses push-triggered workflow runs for those pushes. The release jobs +therefore dispatch `docker.yml` explicitly at the new tag ref; the tag +mapping keys off `github.ref` either way. + ## Local Commands ### Preview a canary locally @@ -123,6 +242,24 @@ The workflow: ./scripts/release.sh canary --dry-run ``` +### Preview a nightly locally + +Requires HEAD to be a commit that already shipped a canary (it must carry a +`canary/v*` tag): + +```bash +./scripts/release.sh nightly --dry-run +``` + +### Preview a beta locally + +Requires HEAD to be a commit that already shipped a nightly (it must carry a +`nightly/v*` tag): + +```bash +./scripts/release.sh beta --dry-run +``` + ### Preview a stable locally ```bash @@ -185,9 +322,15 @@ Automated browser smoke is also available: ```bash gh workflow run release-smoke.yml -f paperclip_version=canary +gh workflow run release-smoke.yml -f paperclip_version=nightly +gh workflow run release-smoke.yml -f paperclip_version=beta gh workflow run release-smoke.yml -f paperclip_version=latest ``` +The nightly lane runs this same suite automatically against its candidate +before publishing, and the beta lane runs it against the published beta as +post-publish verification. + Minimum checks: - `npx paperclipai@canary onboard` installs @@ -224,6 +367,29 @@ Instead: 3. wait for the next automatic canary 4. rerun smoke testing +### If the nightly skipped or failed + +A skipped nightly is working as designed — the job summary names the reason +(no new green candidate, candidate already shipped, or red smoke). Nothing was +published, so there is nothing to clean up. + +To recover after fixing the cause, either wait for the next scheduled run or +force one: dispatch `release.yml` with `channel: nightly` (optionally pinning +`source_version` to a specific canary). + +If the nightly published to npm but the tag push or Docker dispatch failed, +push the `nightly/v*` tag manually and run `docker.yml` at that tag. + +### If a beta looks bad during soak + +Do not promote it to stable. Fix forward: land the fix on `master`, let it +ship through canary and nightly, and promote a new beta. The soak clock +starts over for the new beta. + +If the published beta is actively harmful to beta users, move the `beta` +dist-tag back to the previous beta version with `npm dist-tag add` per +package, and re-point the `:beta` Docker tags at the previous beta's images. + ### If stable npm publish succeeds but tag push or GitHub release creation fails This is a partial release. npm is already live. diff --git a/doc/SPEC-implementation.md b/doc/SPEC-implementation.md index e11298ef817..64b58326d13 100644 --- a/doc/SPEC-implementation.md +++ b/doc/SPEC-implementation.md @@ -1032,6 +1032,7 @@ Dashboard payload must include: The current app also exposes V1-supporting surfaces for: +- company-scoped summary slots for projects, the workspaces overview, project workspaces, and individual execution workspaces; execution-workspace slots are keyed by execution workspace id so a new workspace never inherits another workspace's summary - issue thread interactions (`suggest_tasks`, `ask_user_questions`, `request_confirmation`) - issue approvals, issue references/search, labels, read state, inbox/archive state, and work products - company search through `GET /companies/:companyId/search` plus agent-oriented bulk extraction through diff --git a/docker/Dockerfile.onboard-smoke b/docker/Dockerfile.onboard-smoke index 7b13756b81d..63689c6c8ae 100644 --- a/docker/Dockerfile.onboard-smoke +++ b/docker/Dockerfile.onboard-smoke @@ -37,4 +37,9 @@ WORKDIR /home/paperclip/workspace EXPOSE 3100 USER paperclip -CMD ["bash", "-lc", "set -euo pipefail; mkdir -p \"$PAPERCLIP_HOME\"; npx --yes \"paperclipai@${PAPERCLIPAI_VERSION}\" onboard --yes --data-dir \"$PAPERCLIP_HOME\""] +# --bind lan is required: `onboard --yes` without an explicit --bind prefers +# trusted-local defaults, which write a loopback bind into the instance config +# and ignore the deployment env vars. A loopback listener inside the container +# is unreachable through Docker's port mapping, so the smoke health check can +# never pass without this flag. +CMD ["bash", "-lc", "set -euo pipefail; mkdir -p \"$PAPERCLIP_HOME\"; npx --yes \"paperclipai@${PAPERCLIPAI_VERSION}\" onboard --yes --bind lan --data-dir \"$PAPERCLIP_HOME\""] diff --git a/packages/adapters/codex-local/src/server/__fixtures__/README.md b/packages/adapters/codex-local/src/server/__fixtures__/README.md new file mode 100644 index 00000000000..8cbb76777b5 --- /dev/null +++ b/packages/adapters/codex-local/src/server/__fixtures__/README.md @@ -0,0 +1,38 @@ +# Device-login sample fixtures + +These fixtures hold redacted, real Codex device-login output. A capture step ran +`codex login --device-auth` inside a Daytona sandbox and recorded the output. The +capture step redacted every secret before it kept or posted the text. The parser +tests read these fixtures. The tests never read a live secret. + +## Source + +- Capture date (UTC): `2026-08-08`. +- Daytona image: `cr.app.daytona.io/sbox/daytona-6a8d60e245981dd72d0e647e6afae46e0fd7b8bdfdb29b08120747509a39dc33:daytona`. +- Sandbox id: `3bd22e18-2103-4227-b918-6c375fd30b49` (region `us`). +- Codex home: a new throwaway directory for each independent run. + +## Files + +| File | Row | Condition | Expected parse result | +|---|---|---|---| +| `device-login-sample.txt` | A — normal prompt | `timeout 60 codex login --device-auth` | a URL and a code | +| `device-login-edge.txt` | D — error / retry | offline run with an unreachable local proxy | `null` | + +Two more rows from the capture are not committed as fixtures. Row B (timeout +tail) printed no extra Codex line; the external `timeout` process ended the +command with exit status `124`. Row C (`codex login --help`) and Row E +(`codex --version`) are not device-login prompts. + +## Redaction + +The capture step transformed the real one-time code to question marks and kept +the shape. So `device-login-sample.txt` holds `????-?????` in the code position. +The real code alphabet is a Codex detail and the capture did not keep it. The +parser matches the grounded structure of the code — four characters, a hyphen, +then five characters — and does not invent an alphabet. The token class is bound +to alphanumerics and the redaction sentinel `?`, so the committed sample parses +to a code without a real secret in the repository. + +The capture step checked the final text for common credential field names and +for an unredacted device-code pattern. It found no match. diff --git a/packages/adapters/codex-local/src/server/__fixtures__/device-login-edge.txt b/packages/adapters/codex-local/src/server/__fixtures__/device-login-edge.txt new file mode 100644 index 00000000000..a846c118206 --- /dev/null +++ b/packages/adapters/codex-local/src/server/__fixtures__/device-login-edge.txt @@ -0,0 +1 @@ +Error logging in with device code: error sending request for url (https://auth.openai.com/api/accounts/deviceauth/) diff --git a/packages/adapters/codex-local/src/server/__fixtures__/device-login-sample.txt b/packages/adapters/codex-local/src/server/__fixtures__/device-login-sample.txt new file mode 100644 index 00000000000..9cf72aa8518 --- /dev/null +++ b/packages/adapters/codex-local/src/server/__fixtures__/device-login-sample.txt @@ -0,0 +1,5 @@ +1. Open this link in your browser and sign in to your account +https://auth.openai.com/codex/device +2. Enter this one-time code (expires in 15 minutes) +????-????? +Device codes are a common phishing target. Never share this code. diff --git a/packages/adapters/codex-local/src/server/device-login-export.test.ts b/packages/adapters/codex-local/src/server/device-login-export.test.ts new file mode 100644 index 00000000000..616b23b1354 --- /dev/null +++ b/packages/adapters/codex-local/src/server/device-login-export.test.ts @@ -0,0 +1,284 @@ +import { chmod, lstat, mkdir, mkdtemp, readFile, rm, stat, symlink, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; + +import { + MAX_AUTH_JSON_BYTES, + deriveProofHome, + installDeviceLoginCredential, + removeProofHome, + resolveProofHomeRoot, +} from "./device-login-export.js"; +import { resolveManagedCodexHomeDir, resolveSharedCodexHomeDir } from "./codex-home.js"; + +const COMPANY = "company-a"; +const RUN = "run-1234"; +const NEWER = "2026-07-09T02:00:00Z"; +const OLDER = "2026-07-09T01:00:00Z"; +const ACCOUNT = "acct-42"; +const OTHER_ACCOUNT = "acct-99"; +const TOKEN_SENTINEL = "SENTINEL_TOKEN_XYZ"; + +describe("device-login credential export", () => { + const cleanupDirs: string[] = []; + + afterEach(async () => { + while (cleanupDirs.length > 0) { + const dir = cleanupDirs.pop(); + if (!dir) continue; + await chmod(dir, 0o700).catch(() => undefined); + await rm(dir, { recursive: true, force: true }).catch(() => undefined); + } + }); + + async function makeInstanceRoot(): Promise { + const dir = await mkdtemp(path.join(os.tmpdir(), "paperclip-codex-proof-")); + cleanupDirs.push(dir); + return dir; + } + + function envFor(instanceHome: string, extra: Record = {}): NodeJS.ProcessEnv { + return { PAPERCLIP_HOME: instanceHome, PAPERCLIP_INSTANCE_ID: "default", ...extra }; + } + + function subscriptionAuth(input: { accountId: string; lastRefresh?: string; marker?: string }): Buffer { + const suffix = input.marker ?? input.accountId; + return Buffer.from( + JSON.stringify({ + tokens: { + id_token: `id-token-${suffix}`, + access_token: `access-token-${suffix}`, + refresh_token: `${TOKEN_SENTINEL}-${suffix}`, + account_id: input.accountId, + }, + ...(input.lastRefresh ? { last_refresh: input.lastRefresh } : {}), + }), + ); + } + + const noopLog = (_line: string): void => {}; + + it("deriveProofHome returns a unique, run-scoped path under a company-scoped root", async () => { + const home = await makeInstanceRoot(); + const env = envFor(home); + const root = resolveProofHomeRoot(env, COMPANY); + expect(root).toBe( + path.resolve(home, "instances", "default", "companies", COMPANY, "codex-device-login-proof"), + ); + const a = deriveProofHome({ env, companyId: COMPANY, runId: RUN }); + const b = deriveProofHome({ env, companyId: COMPANY, runId: RUN }); + expect(a.startsWith(root + path.sep)).toBe(true); + expect(a).toContain(RUN); + expect(a).not.toBe(b); // unique per call + // Never the shared or the managed home. + expect(a).not.toBe(resolveSharedCodexHomeDir(env)); + expect(a).not.toBe(resolveManagedCodexHomeDir(env, COMPANY)); + }); + + it("export_creates_root_and_home_at_mode_0700", async () => { + const home = await makeInstanceRoot(); + const env = envFor(home); + const proofHome = deriveProofHome({ env, companyId: COMPANY, runId: RUN }); + await installDeviceLoginCredential({ + sandboxAuthBytes: subscriptionAuth({ accountId: ACCOUNT, lastRefresh: NEWER }), + proofHome, + env, + companyId: COMPANY, + log: noopLog, + }); + const root = resolveProofHomeRoot(env, COMPANY); + expect((await stat(root)).mode & 0o777).toBe(0o700); + expect((await stat(proofHome)).mode & 0o777).toBe(0o700); + }); + + it("export_seeds_empty_proof_home_with_mode_0600", async () => { + const home = await makeInstanceRoot(); + const env = envFor(home); + const proofHome = deriveProofHome({ env, companyId: COMPANY, runId: RUN }); + const outcome = await installDeviceLoginCredential({ + sandboxAuthBytes: subscriptionAuth({ accountId: ACCOUNT, lastRefresh: NEWER }), + proofHome, + env, + companyId: COMPANY, + log: noopLog, + }); + expect(outcome).toBe("seeded"); + const authPath = path.join(proofHome, "auth.json"); + expect((await stat(authPath)).mode & 0o777).toBe(0o600); + const written = JSON.parse(await readFile(authPath, "utf8")); + expect(written.tokens.account_id).toBe(ACCOUNT); + }); + + it("export_rejects_default_shared_and_managed_home", async () => { + const home = await makeInstanceRoot(); + const env = envFor(home, { CODEX_HOME: path.join(home, "shared-codex") }); + const bytes = subscriptionAuth({ accountId: ACCOUNT, lastRefresh: NEWER }); + const shared = resolveSharedCodexHomeDir(env); + const managed = resolveManagedCodexHomeDir(env, COMPANY); + const agentManaged = path.resolve(managed, "..", "agents", "agent-1", "codex-home"); + for (const target of [shared, managed, agentManaged]) { + await expect( + installDeviceLoginCredential({ sandboxAuthBytes: bytes, proofHome: target, env, companyId: COMPANY, log: noopLog }), + ).rejects.toThrow(); + } + }); + + it("export_rejects_symlink_or_non_regular_path", async () => { + const home = await makeInstanceRoot(); + const env = envFor(home); + const root = resolveProofHomeRoot(env, COMPANY); + await mkdir(root, { recursive: true, mode: 0o700 }); + const bytes = subscriptionAuth({ accountId: ACCOUNT, lastRefresh: NEWER }); + + // A symlinked proof home is rejected. + const realDir = path.join(home, "elsewhere"); + await mkdir(realDir, { recursive: true, mode: 0o700 }); + const linkedHome = path.join(root, `${RUN}-linked`); + await symlink(realDir, linkedHome); + await expect( + installDeviceLoginCredential({ sandboxAuthBytes: bytes, proofHome: linkedHome, env, companyId: COMPANY, log: noopLog }), + ).rejects.toThrow(); + + // A proof home whose auth.json is a symlink is rejected. + const symAuthHome = path.join(root, `${RUN}-symauth`); + await mkdir(symAuthHome, { recursive: true, mode: 0o700 }); + await symlink(path.join(home, "target.json"), path.join(symAuthHome, "auth.json")); + await expect( + installDeviceLoginCredential({ sandboxAuthBytes: bytes, proofHome: symAuthHome, env, companyId: COMPANY, log: noopLog }), + ).rejects.toThrow(); + }); + + it("export_rejects_api_key_malformed_or_oversized_payload", async () => { + const home = await makeInstanceRoot(); + const env = envFor(home); + const apiKey = Buffer.from(JSON.stringify({ OPENAI_API_KEY: "sk-secret-key" })); + const malformed = Buffer.from("this is not json {"); + const oversized = Buffer.concat([ + subscriptionAuth({ accountId: ACCOUNT, lastRefresh: NEWER }), + Buffer.alloc(MAX_AUTH_JSON_BYTES + 10, 0x20), + ]); + for (const bytes of [apiKey, malformed, oversized]) { + const proofHome = deriveProofHome({ env, companyId: COMPANY, runId: RUN }); + await expect( + installDeviceLoginCredential({ sandboxAuthBytes: bytes, proofHome, env, companyId: COMPANY, log: noopLog }), + ).rejects.toThrow(); + // No file was written on a rejected payload. + await expect(stat(path.join(proofHome, "auth.json"))).rejects.toThrow(); + } + }); + + it("export_updates_home_with_strictly_newer_same_identity", async () => { + const home = await makeInstanceRoot(); + const env = envFor(home); + const proofHome = deriveProofHome({ env, companyId: COMPANY, runId: RUN }); + const authPath = path.join(proofHome, "auth.json"); + await installDeviceLoginCredential({ + sandboxAuthBytes: subscriptionAuth({ accountId: ACCOUNT, lastRefresh: OLDER, marker: "old" }), + proofHome, + env, + companyId: COMPANY, + log: noopLog, + }); + const outcome = await installDeviceLoginCredential({ + sandboxAuthBytes: subscriptionAuth({ accountId: ACCOUNT, lastRefresh: NEWER, marker: "new" }), + proofHome, + env, + companyId: COMPANY, + log: noopLog, + }); + expect(outcome).toBe("updated"); + const written = JSON.parse(await readFile(authPath, "utf8")); + expect(written.last_refresh).toBe(NEWER); + expect(written.tokens.refresh_token).toContain("new"); + }); + + it("export_keeps_home_on_older_or_different_identity", async () => { + const home = await makeInstanceRoot(); + const env = envFor(home); + const proofHome = deriveProofHome({ env, companyId: COMPANY, runId: RUN }); + const authPath = path.join(proofHome, "auth.json"); + await installDeviceLoginCredential({ + sandboxAuthBytes: subscriptionAuth({ accountId: ACCOUNT, lastRefresh: NEWER, marker: "keep" }), + proofHome, + env, + companyId: COMPANY, + log: noopLog, + }); + // Older same identity: kept. + const olderOutcome = await installDeviceLoginCredential({ + sandboxAuthBytes: subscriptionAuth({ accountId: ACCOUNT, lastRefresh: OLDER, marker: "older" }), + proofHome, + env, + companyId: COMPANY, + log: noopLog, + }); + expect(olderOutcome).toBe("kept"); + // Different identity, even newer: kept. + const otherOutcome = await installDeviceLoginCredential({ + sandboxAuthBytes: subscriptionAuth({ accountId: OTHER_ACCOUNT, lastRefresh: NEWER, marker: "other" }), + proofHome, + env, + companyId: COMPANY, + log: noopLog, + }); + expect(otherOutcome).toBe("kept"); + const written = JSON.parse(await readFile(authPath, "utf8")); + expect(written.tokens.account_id).toBe(ACCOUNT); + expect(written.tokens.refresh_token).toContain("keep"); + }); + + it("export_logs_contain_no_token_bytes", async () => { + const home = await makeInstanceRoot(); + const env = envFor(home); + const proofHome = deriveProofHome({ env, companyId: COMPANY, runId: RUN }); + const logs: string[] = []; + await installDeviceLoginCredential({ + sandboxAuthBytes: subscriptionAuth({ accountId: ACCOUNT, lastRefresh: NEWER }), + proofHome, + env, + companyId: COMPANY, + log: (line) => { + logs.push(line); + }, + }); + const haystack = logs.join("\n"); + expect(haystack).not.toContain(TOKEN_SENTINEL); + expect(haystack).not.toContain(ACCOUNT); + }); + + it("cleanup_removes_proof_home_and_leaves_default_home_unchanged", async () => { + const home = await makeInstanceRoot(); + const env = envFor(home, { CODEX_HOME: path.join(home, "shared-codex") }); + // A sentinel in a fake shared/default home must survive the cleanup. + const shared = resolveSharedCodexHomeDir(env); + await mkdir(shared, { recursive: true, mode: 0o700 }); + const sentinel = path.join(shared, "auth.json"); + await writeFile(sentinel, JSON.stringify({ tokens: { account_id: "host", refresh_token: "host" } }), { mode: 0o600 }); + + const proofHome = deriveProofHome({ env, companyId: COMPANY, runId: RUN }); + await installDeviceLoginCredential({ + sandboxAuthBytes: subscriptionAuth({ accountId: ACCOUNT, lastRefresh: NEWER }), + proofHome, + env, + companyId: COMPANY, + log: noopLog, + }); + expect((await lstat(proofHome)).isDirectory()).toBe(true); + + await removeProofHome(proofHome, { env, companyId: COMPANY }); + await expect(lstat(proofHome)).rejects.toThrow(); + // The fake default home is untouched. + expect(await readFile(sentinel, "utf8")).toContain("host"); + }); + + it("removeProofHome refuses a path outside the proof root", async () => { + const home = await makeInstanceRoot(); + const env = envFor(home); + const outside = path.join(home, "not-a-proof-home"); + await mkdir(outside, { recursive: true }); + await expect(removeProofHome(outside, { env, companyId: COMPANY })).rejects.toThrow(); + // The outside directory is untouched. + expect((await lstat(outside)).isDirectory()).toBe(true); + }); +}); diff --git a/packages/adapters/codex-local/src/server/device-login-export.ts b/packages/adapters/codex-local/src/server/device-login-export.ts new file mode 100644 index 00000000000..b72f16749e8 --- /dev/null +++ b/packages/adapters/codex-local/src/server/device-login-export.ts @@ -0,0 +1,290 @@ +import { chmod, lstat, mkdir, rm } from "node:fs/promises"; +import path from "node:path"; +import { randomUUID } from "node:crypto"; +import { resolvePaperclipInstanceRootForAdapter } from "@paperclipai/adapter-utils/server-utils"; +import { readSubscriptionAccountId, writeCodexAuthCacheEntry } from "./codex-auth-cache.js"; +import { copyBackCodexAuth } from "./codex-auth-copyback.js"; +import { + codexHomeHasUsableAuth, + resolveManagedCodexHomeDir, + resolveSharedCodexHomeDir, +} from "./codex-home.js"; + +// The credential export. It installs a device-login credential into a unique, +// run-scoped, private proof home. It handles the empty-home first-login case and +// a later strictly-newer, same-identity update. It removes the proof home on +// every terminal path. +// +// Security (Control 2): the export never accepts an arbitrary host-home +// argument. It derives a unique, run-scoped proof home under a dedicated +// company-scoped root. It rejects the default, the shared, and every managed +// Codex home. It calls `lstat` on the proof-home path and rejects a symlink or a +// non-regular file. It creates the root and the proof home at mode 0700. It +// stages and renames `auth.json` at mode 0600 under a directory lock (through the +// reused seed writer and copy-back helpers). It enforces a bounded-size, +// subscription-only auth shape before any write, so it rejects an API-key, a +// malformed, and an oversized payload. It never logs token bytes. + +const PROOF_ROOT_DIR_NAME = "codex-device-login-proof"; +// A private directory (owner rwx only). 0o700 has no group or other bits. +const PRIVATE_DIR_MODE = 0o700; +// The managed Codex home directory always uses this name (see +// `resolveManagedCodexHomeDir`). The export rejects any target with this name, +// so a shared company home and a per-agent home are both refused. +const MANAGED_HOME_DIR_NAME = "codex-home"; +const AUTH_FILE_NAME = "auth.json"; + +// A bounded size for the credential payload. A real subscription `auth.json` is +// a few kilobytes. The export refuses a larger payload before any parse or write. +export const MAX_AUTH_JSON_BYTES = 64 * 1024; + +export type InstallDeviceLoginOutcome = "seeded" | "updated" | "kept"; + +function nonEmpty(value: string | undefined): string | null { + return typeof value === "string" && value.trim().length > 0 ? value.trim() : null; +} + +/** + * Sanitizes one raw value to a single safe path segment. Rejects an empty value, + * a relative segment, a path separator, and a NUL byte, so the value can never + * become a path traversal. Returns the trimmed, safe segment. + */ +function requireSafeSegment(value: string, label: string): string { + const trimmed = typeof value === "string" ? value.trim() : ""; + if (trimmed.length === 0) throw new Error(`device-login export: ${label} is empty`); + if (trimmed === "." || trimmed === "..") { + throw new Error(`device-login export: ${label} is a relative path segment`); + } + if (trimmed.includes("/") || trimmed.includes("\\") || trimmed.includes("\0")) { + throw new Error(`device-login export: ${label} contains a path separator`); + } + if (path.basename(trimmed) !== trimmed) { + throw new Error(`device-login export: ${label} is not a single path segment`); + } + return trimmed; +} + +/** Reduces a run id to a safe, bounded path segment. Never throws. */ +function toSafeRunSegment(value: string | null): string { + const cleaned = (value ?? "").trim().replace(/[^A-Za-z0-9_-]+/g, "-").slice(0, 80); + return cleaned.length > 0 ? cleaned : "run"; +} + +/** + * Resolves the company-scoped proof-home root under the same isolation boundary + * as the managed Codex home. The root is always company-scoped, so a proof home + * can never cross a company boundary. `companyId` is required and is sanitized to + * a single safe path segment. + */ +export function resolveProofHomeRoot(env: NodeJS.ProcessEnv = process.env, companyId: string): string { + const safeCompanyId = requireSafeSegment(companyId, "companyId"); + const instanceRoot = resolvePaperclipInstanceRootForAdapter({ + homeDir: nonEmpty(env.PAPERCLIP_HOME) ?? undefined, + instanceId: nonEmpty(env.PAPERCLIP_INSTANCE_ID) ?? undefined, + env, + }); + return path.resolve(instanceRoot, "companies", safeCompanyId, PROOF_ROOT_DIR_NAME); +} + +export interface DeriveProofHomeInput { + env?: NodeJS.ProcessEnv; + companyId?: string; + runId?: string; +} + +/** + * Returns a unique, run-scoped proof-home path under the company-scoped root. + * The path carries the run id and a fresh random suffix, so two calls never + * collide and every proof home is tied to its run. + */ +export function deriveProofHome(input: DeriveProofHomeInput = {}): string { + const env = input.env ?? process.env; + const companyId = requireSafeSegment( + input.companyId ?? nonEmpty(env.PAPERCLIP_COMPANY_ID) ?? "", + "companyId", + ); + const runSegment = toSafeRunSegment(input.runId ?? nonEmpty(env.PAPERCLIP_RUN_ID)); + const root = resolveProofHomeRoot(env, companyId); + return path.resolve(root, `${runSegment}-${randomUUID()}`); +} + +/** + * Rejects a target that is the shared, the default, or a managed Codex home, or + * that is not strictly under the company-scoped proof root. This guard runs + * before any filesystem write, so the export never writes outside its own root. + */ +function assertProofHomeIsSafeTarget( + resolved: string, + env: NodeJS.ProcessEnv, + companyId: string, +): void { + const shared = path.resolve(resolveSharedCodexHomeDir(env)); + if (resolved === shared) { + throw new Error("device-login export: refused the shared or default Codex home"); + } + const managed = path.resolve(resolveManagedCodexHomeDir(env, companyId)); + if (resolved === managed) { + throw new Error("device-login export: refused the managed company Codex home"); + } + if (path.basename(resolved) === MANAGED_HOME_DIR_NAME) { + throw new Error("device-login export: refused a managed Codex home"); + } + const root = resolveProofHomeRoot(env, companyId); + if (!resolved.startsWith(root + path.sep)) { + throw new Error("device-login export: the proof home must be under the company-scoped proof root"); + } +} + +/** + * Enforces the bounded-size, subscription-only auth shape. Returns the + * subscription `account_id` on success. Rejects an empty, an oversized, an + * API-key, and a malformed payload. Never puts token bytes into the error. + */ +function assertUsableSubscriptionShape(bytes: Buffer): void { + if (bytes.length === 0) { + throw new Error("device-login export: refused an empty auth payload"); + } + if (bytes.length > MAX_AUTH_JSON_BYTES) { + throw new Error("device-login export: refused an oversized auth payload"); + } + const accountId = readSubscriptionAccountId(bytes); + if (!accountId) { + // Covers an API-key payload, a malformed payload, and an unusable payload. + throw new Error("device-login export: refused a non-subscription auth payload"); + } +} + +/** + * Ensures one directory exists and is private (mode 0700). Uses `lstat` (not + * `stat`), so the export never writes through a planted symlink. Fails closed + * when the existing path is a symlink or a non-directory. + */ +async function ensurePrivateDir(dir: string): Promise { + const existing = await lstat(dir).catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return null; + throw error; + }); + if (existing) { + if (existing.isSymbolicLink() || !existing.isDirectory()) { + throw new Error("device-login export: the proof-home path is a symlink or a non-directory"); + } + await chmod(dir, PRIVATE_DIR_MODE); + return; + } + await mkdir(dir, { mode: PRIVATE_DIR_MODE }); + await chmod(dir, PRIVATE_DIR_MODE); +} + +/** + * Rejects an existing `auth.json` that is a symlink or a non-regular file. An + * absent file is the normal first-install case. + */ +async function assertAuthPathIsRegularOrAbsent(authPath: string): Promise { + const existing = await lstat(authPath).catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return null; + throw error; + }); + if (existing && (existing.isSymbolicLink() || !existing.isFile())) { + throw new Error("device-login export: refused a symlink or a non-regular auth.json"); + } +} + +export interface InstallDeviceLoginCredentialInput { + /** The sandbox `auth.json` bytes read back from the login sandbox. */ + sandboxAuthBytes: Buffer; + /** The run-scoped proof home from {@link deriveProofHome}. */ + proofHome: string; + /** A non-leaking progress sink. It receives only fixed status lines. */ + log: (line: string) => void | Promise; + env?: NodeJS.ProcessEnv; + companyId?: string; +} + +/** + * Installs the sandbox credential into the run-scoped proof home. Seeds an empty + * home. Applies a strictly-newer, same-identity update to a non-empty home. + * Keeps the home otherwise. Enforces Control 2 before any write: the target + * safety guard, the auth-shape gate, and the path-safety guard all run first. + * Never logs token bytes. + */ +export async function installDeviceLoginCredential( + input: InstallDeviceLoginCredentialInput, +): Promise { + const { sandboxAuthBytes, log } = input; + const env = input.env ?? process.env; + const companyId = requireSafeSegment( + input.companyId ?? nonEmpty(env.PAPERCLIP_COMPANY_ID) ?? "", + "companyId", + ); + const proofHome = path.resolve(input.proofHome); + + // 1. Target safety (pure path logic). Reject a shared, default, or managed + // home, or a path outside the proof root, before any filesystem work. + assertProofHomeIsSafeTarget(proofHome, env, companyId); + + // 2. Auth-shape gate. Reject an empty, oversized, API-key, or malformed + // payload before any directory is created. + assertUsableSubscriptionShape(sandboxAuthBytes); + + // 3. Path safety. Create the root and the proof home at mode 0700, each + // guarded by `lstat`. + const root = resolveProofHomeRoot(env, companyId); + await mkdir(path.dirname(root), { recursive: true }); + await ensurePrivateDir(root); + await ensurePrivateDir(proofHome); + + const authPath = path.join(proofHome, AUTH_FILE_NAME); + await assertAuthPathIsRegularOrAbsent(authPath); + + // 4. Decide first install versus update. + const hadUsableAuth = await codexHomeHasUsableAuth(proofHome); + if (!hadUsableAuth) { + // First install: the copy-back predicate fails closed on an absent + // destination, so the seed writer fills the empty home. The seed writer + // stages and renames `auth.json` at mode 0600 under a directory lock. + const outcome = await writeCodexAuthCacheEntry({ + sandboxAuthBytes, + cacheEntryPath: authPath, + log, + }); + return outcome === "written" ? "seeded" : "kept"; + } + + // Update: install the credential only when it is strictly newer for the same + // subscription identity. `copyBackCodexAuth` runs the same decision predicate + // and the same 0600 staged rename under the directory lock. + const outcome = await copyBackCodexAuth({ + readSandboxAuth: async () => sandboxAuthBytes, + hostAuthPath: authPath, + log, + env, + }); + return outcome === "copied" ? "updated" : "kept"; +} + +export interface RemoveProofHomeInput { + env?: NodeJS.ProcessEnv; + companyId?: string; +} + +/** + * Removes the proof home. Refuses a path outside the company-scoped proof root, + * so a cleanup can never delete the shared, the default, or a managed home. A + * missing proof home is a benign no-op. + */ +export async function removeProofHome( + proofHome: string, + input: RemoveProofHomeInput = {}, +): Promise { + const env = input.env ?? process.env; + const companyId = requireSafeSegment( + input.companyId ?? nonEmpty(env.PAPERCLIP_COMPANY_ID) ?? "", + "companyId", + ); + const resolved = path.resolve(proofHome); + const root = resolveProofHomeRoot(env, companyId); + if (!resolved.startsWith(root + path.sep)) { + throw new Error("device-login export: refused to remove a path outside the proof root"); + } + await rm(resolved, { recursive: true, force: true }); +} diff --git a/packages/adapters/codex-local/src/server/device-login-parse.test.ts b/packages/adapters/codex-local/src/server/device-login-parse.test.ts new file mode 100644 index 00000000000..cfd6fa9b981 --- /dev/null +++ b/packages/adapters/codex-local/src/server/device-login-parse.test.ts @@ -0,0 +1,212 @@ +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; +import { parseDeviceLoginPrompt } from "./device-login-parse.js"; + +const fixturesDir = path.join(path.dirname(fileURLToPath(import.meta.url)), "__fixtures__"); + +function readFixture(name: string): string { + return readFileSync(path.join(fixturesDir, name), "utf8"); +} + +const EXACT_URL = "https://auth.openai.com/codex/device"; + +describe("parseDeviceLoginPrompt", () => { + it("parse_returns_url_and_code_from_sample", () => { + const result = parseDeviceLoginPrompt(readFixture("device-login-sample.txt")); + expect(result).not.toBeNull(); + expect(result?.url).toBe(EXACT_URL); + // The committed sample keeps the capture-time redaction of the code. The + // parser extracts the four-hyphen-five structure without a real secret. + expect(result?.code).toBe("????-?????"); + }); + + it("parse_returns_url_and_code_from_ansi_colored_output", () => { + // Codex CLI 0.128.0 and later add ANSI color (SGR) sequences around the URL + // and the code. A live sandbox run confirmed this format. The parser removes + // the Control Sequence Introducer sequences and reads the tokens the same as + // plain output. + const cyan = "\x1b[36m"; + const bold = "\x1b[1m"; + const reset = "\x1b[0m"; + const text = [ + "1. Open this link in your browser and sign in to your account", + `${cyan}${EXACT_URL}${reset}`, + "2. Enter this one-time code (expires in 15 minutes)", + `${bold}ABCD-EFGHJ${reset}`, + ].join("\n"); + const result = parseDeviceLoginPrompt(text); + expect(result).not.toBeNull(); + expect(result?.url).toBe(EXACT_URL); + expect(result?.code).toBe("ABCD-EFGHJ"); + }); + + it("parse_returns_null_when_prompt_absent", () => { + const text = "Some unrelated log line\nNothing to see here\n"; + expect(parseDeviceLoginPrompt(text)).toBeNull(); + }); + + it("parse_returns_null_for_url_with_query_or_fragment", () => { + const withQuery = [ + "Open this link", + "https://auth.openai.com/codex/device?foo=bar", + "ABCD-EFGHJ", + ].join("\n"); + const withFragment = [ + "Open this link", + "https://auth.openai.com/codex/device#section", + "ABCD-EFGHJ", + ].join("\n"); + expect(parseDeviceLoginPrompt(withQuery)).toBeNull(); + expect(parseDeviceLoginPrompt(withFragment)).toBeNull(); + }); + + it("parse_returns_null_for_wrong_origin_or_path", () => { + const wrongOrigin = [ + "Open this link", + "https://auth.example.com/codex/device", + "ABCD-EFGHJ", + ].join("\n"); + const wrongPath = [ + "Open this link", + "https://auth.openai.com/codex/device/extra", + "ABCD-EFGHJ", + ].join("\n"); + const httpScheme = [ + "Open this link", + "http://auth.openai.com/codex/device", + "ABCD-EFGHJ", + ].join("\n"); + expect(parseDeviceLoginPrompt(wrongOrigin)).toBeNull(); + expect(parseDeviceLoginPrompt(wrongPath)).toBeNull(); + expect(parseDeviceLoginPrompt(httpScheme)).toBeNull(); + }); + + it("parse_returns_null_for_malformed_short_code", () => { + // Each input carries the code preamble after the URL, so the test exercises + // the code-structure check and not the preamble check. + const preamble = "2. Enter this one-time code (expires in 15 minutes)"; + const shortCode = [EXACT_URL, preamble, "ABC-EFGHJ"].join("\n"); // 3 then 5 + const longCode = [EXACT_URL, preamble, "ABCDE-EFGHJ"].join("\n"); // 5 then 5 + const noHyphen = [EXACT_URL, preamble, "ABCDEFGHJ"].join("\n"); + const noCode = [EXACT_URL, preamble, "no code on this line"].join("\n"); + expect(parseDeviceLoginPrompt(shortCode)).toBeNull(); + expect(parseDeviceLoginPrompt(longCode)).toBeNull(); + expect(parseDeviceLoginPrompt(noHyphen)).toBeNull(); + expect(parseDeviceLoginPrompt(noCode)).toBeNull(); + }); + + it("parse_binds_code_to_url_and_ignores_a_code_before_the_url", () => { + // A code-shaped token appears before the URL. The parser reads the code only + // after the URL, so it uses the code that follows the URL. + const text = [ + "WXYZ-98765", + "1. Open this link in your browser and sign in to your account", + EXACT_URL, + "2. Enter this one-time code (expires in 15 minutes)", + "ABCD-EFGHJ", + ].join("\n"); + const result = parseDeviceLoginPrompt(text); + expect(result).not.toBeNull(); + expect(result?.code).toBe("ABCD-EFGHJ"); + }); + + it("parse_returns_null_when_only_code_is_before_the_url", () => { + // The only code-shaped token sits before the URL. No code follows the URL, + // so the parser rejects the output instead of binding the earlier token. + const text = ["WXYZ-98765", "Open this link", EXACT_URL].join("\n"); + expect(parseDeviceLoginPrompt(text)).toBeNull(); + }); + + it("parse_returns_null_when_code_is_far_after_the_url", () => { + // A code-shaped token sits far below the URL, past the proximity window. The + // parser does not bind a distant token to the URL. + const filler = "unrelated log line\n".repeat(40); + const text = [EXACT_URL, filler, "ABCD-EFGHJ"].join("\n"); + expect(parseDeviceLoginPrompt(text)).toBeNull(); + }); + + it("parse_ignores_a_code_between_the_url_and_the_code_preamble", () => { + // A code-shaped token sits after the URL but before the "one-time code" + // preamble line. The parser anchors the code on the preamble, so it ignores + // the earlier token and returns the real code after the preamble. + const text = [ + "1. Open this link in your browser and sign in to your account", + EXACT_URL, + "session id WXYZ-98765 for this attempt", + "2. Enter this one-time code (expires in 15 minutes)", + "ABCD-EFGHJ", + ].join("\n"); + const result = parseDeviceLoginPrompt(text); + expect(result).not.toBeNull(); + expect(result?.code).toBe("ABCD-EFGHJ"); + }); + + it("parse_ignores_a_code_that_shares_the_preamble_line", () => { + // A code-shaped token shares the preamble line with the "one-time code" + // phrase. The parser advances past the whole preamble line, so it ignores the + // token on that line and returns the code from the dedicated code line below. + const text = [ + EXACT_URL, + "2. Enter this one-time code (ref WXYZ-98765, expires in 15 minutes)", + "ABCD-EFGHJ", + ].join("\n"); + const result = parseDeviceLoginPrompt(text); + expect(result).not.toBeNull(); + expect(result?.code).toBe("ABCD-EFGHJ"); + }); + + it("parse_returns_null_when_a_noise_line_follows_the_preamble", () => { + // A line that mixes a code-shaped token with other text sits right after the + // preamble line. The parser reads the code only from a dedicated code line, so + // it never surfaces the embedded token. It rejects the output instead. This + // proves the parser binds the code to the prompt-code line and not to any + // nearby code-shaped token. + const text = [ + EXACT_URL, + "2. Enter this one-time code (expires in 15 minutes)", + "your session token is WXYZ-98765 here", + "ABCD-EFGHJ", + ].join("\n"); + expect(parseDeviceLoginPrompt(text)).toBeNull(); + }); + + it("parse_returns_null_when_the_code_preamble_is_absent", () => { + // A valid code follows the URL, but the "one-time code" preamble is absent. + // The parser requires the preamble, so it rejects the output. This binds the + // code to the prompt structure and not to a bare code-shaped token. + const text = [ + "1. Open this link in your browser and sign in to your account", + EXACT_URL, + "ABCD-EFGHJ", + ].join("\n"); + expect(parseDeviceLoginPrompt(text)).toBeNull(); + }); + + it("parse_ignores_token_like_text", () => { + // Token-like noise without the exact device URL must not yield a prompt. + const text = [ + "tokens received", + "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.payload.signature", + "sk-proj-ABCD1234ABCD1234ABCD1234", + "ABCD-EFGHJ", + ].join("\n"); + expect(parseDeviceLoginPrompt(text)).toBeNull(); + }); + + it("parse_returns_null_for_edge_sample", () => { + // The grounded edge row carries a URL, but a wrong path and a wrong origin + // segment, so the parser rejects it. + expect(parseDeviceLoginPrompt(readFixture("device-login-edge.txt"))).toBeNull(); + }); + + it("keeps the url and the code out of a thrown error", () => { + // A non-string input is a programming error, but the message must never + // carry secret-bearing input. The parser returns null instead of throwing. + // @ts-expect-error deliberate wrong type + expect(parseDeviceLoginPrompt(undefined)).toBeNull(); + // @ts-expect-error deliberate wrong type + expect(parseDeviceLoginPrompt(12345)).toBeNull(); + }); +}); diff --git a/packages/adapters/codex-local/src/server/device-login-parse.ts b/packages/adapters/codex-local/src/server/device-login-parse.ts new file mode 100644 index 00000000000..86dc3fb83e4 --- /dev/null +++ b/packages/adapters/codex-local/src/server/device-login-parse.ts @@ -0,0 +1,166 @@ +// The device-login output parser. It reads the Codex `login --device-auth` +// output and returns the login URL and the one-time code, or null. +// +// Security (Control 1 — strict validation): the parser accepts only the exact +// origin and path of the device-login URL. It rejects any query, any fragment, a +// different origin, and a different path. It accepts only the short-code +// structure `XXXX-XXXXX` (four characters, a hyphen, then five characters). The +// parser never logs the URL, the code, or any input byte, and it keeps them out +// of every thrown error. The parser is a pure function. + +export interface DeviceLoginPrompt { + url: string; + code: string; +} + +// The one and only accepted device-login URL. The parser returns this exact +// constant string on a match, so the output is never a caller-controlled value. +export const DEVICE_LOGIN_URL = "https://auth.openai.com/codex/device"; + +// Codex CLI 0.128.0 and later wrap the login URL and the one-time code in ANSI +// color sequences. A color sequence is a Control Sequence Introducer (CSI): the +// ESC control byte, a `[`, zero or more parameter bytes (0x30-0x3F), zero or +// more intermediate bytes (0x20-0x2F), and one final byte (0x40-0x7E). The +// parser removes every CSI sequence first, so a colored URL or code reads the +// same as a plain one. The strip only normalizes the input; the URL and the +// code still pass the strict validation below. +// eslint-disable-next-line no-control-regex +const ANSI_CSI_RE = /\x1b\[[0-?]*[ -/]*[@-~]/g; + +// A candidate URL token is a run of non-space characters that starts with an +// http or https scheme. The parser validates each candidate with the `URL` +// class; the regular expression only splits tokens out of the text. +const URL_TOKEN_RE = /https?:\/\/\S+/g; + +// Trailing punctuation that prose commonly puts right after a URL. The parser +// strips only these characters. It never strips `?` or `#`, so a URL with a +// query or a fragment stays malformed and the parser rejects it. +const TRAILING_PUNCTUATION_RE = /[)\].,;:!]+$/; + +// The one-time code structure on a dedicated line: four characters, a hyphen, +// then five characters, and nothing else on the line. The Codex prompt prints +// the code alone on the line that comes right after the preamble line. So the +// pattern matches the whole line. A code-shaped token that shares a line with +// other text cannot match. The real code alphabet is a Codex detail that the +// grounded capture did not keep (the capture redacted the code to `?`). So the +// parser binds the token class to alphanumerics and the redaction sentinel `?`. +const CODE_LINE_RE = /^([A-Za-z0-9?]{4}-[A-Za-z0-9?]{5})$/; + +// The prompt line that introduces the one-time code. The Codex prompt prints +// the phrase "one-time code" on the line right before the code. The parser +// anchors the code search on this phrase. So the code binds to the prompt line +// that introduces it. A code-shaped token that sits between the URL and this +// phrase cannot bind, because the parser reads the code only after the phrase. +const CODE_PREAMBLE_RE = /one-time code/i; + +// The maximum number of characters between the end of the URL and the start of +// the code preamble. The Codex prompt prints the preamble about one line after +// the URL. The parser looks for the preamble only inside this window after the +// URL. So a preamble far away in the output cannot bind to the URL. +const MAX_URL_TO_PREAMBLE_GAP = 256; + +// The maximum number of characters after the end of the preamble line that the +// parser reads for the code. The Codex prompt prints the code on the line right +// after the preamble line. The parser looks for the code only inside this window +// after the preamble line. The window is large enough for the real code line and +// small enough to reject distant noise. +const MAX_PREAMBLE_TO_CODE_GAP = 128; + +// The result of a URL search: the canonical URL and the index of the first +// character after the matched URL token. The code search starts at this index. +interface DeviceUrlMatch { + url: string; + end: number; +} + +/** + * Returns the exact device-login URL and its end index when `text` holds it as + * a standalone token with the exact origin `https://auth.openai.com` and the + * exact path `/codex/device` and no query, fragment, or credentials. Returns + * null otherwise. Returns the canonical {@link DEVICE_LOGIN_URL} constant on a + * match. The `end` index is the position of the first character after the + * matched token in `text`. + */ +function findExactDeviceUrl(text: string): DeviceUrlMatch | null { + for (const match of text.matchAll(URL_TOKEN_RE)) { + const token = match[0]; + const cleaned = token.replace(TRAILING_PUNCTUATION_RE, ""); + let parsed: URL; + try { + parsed = new URL(cleaned); + } catch { + continue; + } + if ( + parsed.protocol === "https:" && + parsed.host === "auth.openai.com" && + parsed.pathname === "/codex/device" && + parsed.search === "" && + parsed.hash === "" && + parsed.username === "" && + parsed.password === "" + ) { + return { url: DEVICE_LOGIN_URL, end: (match.index ?? 0) + token.length }; + } + } + return null; +} + +/** + * Returns the one-time code when `text` holds the code preamble after + * `fromIndex` and a dedicated code line after that preamble line. The parser + * first finds the "one-time code" preamble in a window after the URL. It then + * advances past the end of the preamble line and reads the first non-blank line + * after it. That line must hold only the `XXXX-XXXXX` code. So the code binds to + * the dedicated code line that the prompt prints right after the preamble line. + * A code-shaped token between the URL and the preamble cannot bind. A code-shaped + * token that shares the preamble line cannot bind. A code-shaped token inside a + * line that also holds other text cannot bind. Returns null when the preamble is + * absent, when the preamble has no line break after it, or when the first + * non-blank line after the preamble line is not a code line. + */ +function findShortCode(text: string, fromIndex: number): string | null { + const preambleWindow = text.slice(fromIndex, fromIndex + MAX_URL_TO_PREAMBLE_GAP); + const preambleMatch = CODE_PREAMBLE_RE.exec(preambleWindow); + if (!preambleMatch) return null; + // Advance to the end of the preamble line. The code sits on the next line, so + // the parser reads the code only after the preamble line ends. A code-shaped + // token on the preamble line cannot bind. + const preambleEnd = fromIndex + preambleMatch.index + preambleMatch[0].length; + const lineBreak = text.indexOf("\n", preambleEnd); + if (lineBreak === -1) return null; + const codeWindow = text.slice(lineBreak + 1, lineBreak + 1 + MAX_PREAMBLE_TO_CODE_GAP); + // Read the first non-blank line after the preamble line. It must hold only the + // code. So the parser binds the code to the dedicated code line, and it rejects + // a line that mixes the code with other text. + for (const line of codeWindow.split("\n")) { + const trimmed = line.trim(); + if (trimmed.length === 0) continue; + const match = CODE_LINE_RE.exec(trimmed); + return match ? match[1] : null; + } + return null; +} + +/** + * Parses Codex device-login output. Removes ANSI color sequences first, so + * colored output from Codex CLI 0.128.0 and later reads the same as plain + * output. The parser reads the URL first, then finds the "one-time code" + * preamble after the URL, then reads the code from the dedicated code line right + * after the preamble line. So the code binds to the prompt line that introduces + * it, and a code-shaped token between the URL and the preamble, on the preamble + * line, or mixed with other text on a later line, cannot form a prompt. Returns + * the login URL and the one-time code when both are present and valid. Returns + * null for any other input, including a non-string input, an absent prompt, a URL + * with a query or a fragment, a wrong origin or path, and a malformed short code. + * Never throws on input, and never puts the URL or the code into a log or an error. + */ +export function parseDeviceLoginPrompt(text: string): DeviceLoginPrompt | null { + if (typeof text !== "string" || text.length === 0) return null; + const clean = text.replace(ANSI_CSI_RE, ""); + const urlMatch = findExactDeviceUrl(clean); + if (!urlMatch) return null; + const code = findShortCode(clean, urlMatch.end); + if (!code) return null; + return { url: urlMatch.url, code }; +} diff --git a/packages/adapters/codex-local/src/server/device-login-runner.test.ts b/packages/adapters/codex-local/src/server/device-login-runner.test.ts new file mode 100644 index 00000000000..7a149581499 --- /dev/null +++ b/packages/adapters/codex-local/src/server/device-login-runner.test.ts @@ -0,0 +1,197 @@ +import { describe, expect, it, vi } from "vitest"; +import { + CODEX_DEVICE_LOGIN_COMMAND, + runDeviceLogin, + type SandboxLoginDriver, +} from "./device-login-runner.js"; + +const REAL_SHAPED_URL = "https://auth.openai.com/codex/device"; +const REAL_SHAPED_CODE = "WXYZ-12345"; +const TOKEN_SENTINEL = "SENTINEL_TOKEN_ABC123"; +const AUTH_BYTES = Buffer.from( + JSON.stringify({ tokens: { account_id: "acc", refresh_token: TOKEN_SENTINEL } }), +); + +interface FakeDriverOptions { + chunks?: string[]; + exitCode?: number; + hang?: boolean; + execError?: Error; + readError?: Error; + authBytes?: Buffer; +} + +function createFakeDriver(options: FakeDriverOptions = {}) { + const disposeCalls = { count: 0 }; + const driver: SandboxLoginDriver = { + async execStreaming(_command, onStdout) { + if (options.execError) throw options.execError; + for (const chunk of options.chunks ?? []) { + onStdout(chunk); + } + if (options.hang) { + // Never resolve. The runner must fall back to its timeout or signal. + await new Promise(() => {}); + } + return { exitCode: options.exitCode ?? 0 }; + }, + async readFile() { + if (options.readError) throw options.readError; + return options.authBytes ?? AUTH_BYTES; + }, + async dispose() { + disposeCalls.count += 1; + }, + }; + return { driver, disposeCalls }; +} + +describe("runDeviceLogin", () => { + it("runner_reports_prompt_then_success", async () => { + const { driver, disposeCalls } = createFakeDriver({ + // The prompt spans two chunks; the runner must join them and fire once. + // The chunk boundary falls inside the prompt, after the code preamble. + chunks: [ + `1. Open this link\n${REAL_SHAPED_URL}\n2. Enter this one-time code (expires in 15 minutes)\n`, + `${REAL_SHAPED_CODE}\nDone.\n`, + ], + exitCode: 0, + }); + const onPrompt = vi.fn(); + const onCredential = vi.fn(); + const result = await runDeviceLogin(driver, { + onPrompt, + onCredential, + authPath: "/home/.codex/auth.json", + timeoutMs: 1000, + }); + expect(result.outcome).toBe("success"); + expect(result.exitCode).toBe(0); + expect(result.promptSurfaced).toBe(true); + expect(onPrompt).toHaveBeenCalledTimes(1); + expect(onPrompt).toHaveBeenCalledWith({ url: REAL_SHAPED_URL, code: REAL_SHAPED_CODE }); + expect(onCredential).toHaveBeenCalledTimes(1); + expect(onCredential).toHaveBeenCalledWith(AUTH_BYTES); + expect(disposeCalls.count).toBe(1); + }); + + it("runner_surfaces_prompt_after_a_large_pre_prompt_stream", async () => { + // The sandbox streams a large volume of output before the prompt. The runner + // bounds the parse buffer, so the memory stays limited. The prompt arrives at + // the end of the stream, so the trailing window still holds it and the runner + // surfaces it. + const noise = "unrelated sandbox log line\n".repeat(20000); + const { driver } = createFakeDriver({ + chunks: [ + noise, + `1. Open this link\n${REAL_SHAPED_URL}\n2. Enter this one-time code (expires in 15 minutes)\n${REAL_SHAPED_CODE}\nDone.\n`, + ], + exitCode: 0, + }); + const onPrompt = vi.fn(); + const result = await runDeviceLogin(driver, { onPrompt, timeoutMs: 1000 }); + expect(result.outcome).toBe("success"); + expect(result.promptSurfaced).toBe(true); + expect(onPrompt).toHaveBeenCalledTimes(1); + expect(onPrompt).toHaveBeenCalledWith({ url: REAL_SHAPED_URL, code: REAL_SHAPED_CODE }); + }); + + it("runner_surfaces_prompt_at_the_start_of_one_large_chunk", async () => { + // One stdout callback supplies more than the retained-buffer limit. The + // prompt sits near the start, and a large volume of output follows it. The + // runner parses the whole chunk before it trims the retained window, so it + // still finds the early prompt and never drops it. + const trailingNoise = "unrelated sandbox log line\n".repeat(20000); + const { driver } = createFakeDriver({ + chunks: [ + `1. Open this link\n${REAL_SHAPED_URL}\n2. Enter this one-time code (expires in 15 minutes)\n${REAL_SHAPED_CODE}\nDone.\n${trailingNoise}`, + ], + exitCode: 0, + }); + const onPrompt = vi.fn(); + const result = await runDeviceLogin(driver, { onPrompt, timeoutMs: 1000 }); + expect(result.outcome).toBe("success"); + expect(result.promptSurfaced).toBe(true); + expect(onPrompt).toHaveBeenCalledTimes(1); + expect(onPrompt).toHaveBeenCalledWith({ url: REAL_SHAPED_URL, code: REAL_SHAPED_CODE }); + }); + + it("runner_disposes_sandbox_on_timeout", async () => { + const { driver, disposeCalls } = createFakeDriver({ hang: true }); + const onPrompt = vi.fn(); + const result = await runDeviceLogin(driver, { onPrompt, timeoutMs: 20 }); + expect(result.outcome).toBe("timeout"); + expect(disposeCalls.count).toBe(1); + }); + + it("runner_disposes_sandbox_on_cancellation", async () => { + const { driver, disposeCalls } = createFakeDriver({ hang: true }); + const controller = new AbortController(); + const onPrompt = vi.fn(); + const promise = runDeviceLogin(driver, { + onPrompt, + timeoutMs: 5000, + signal: controller.signal, + }); + controller.abort(); + const result = await promise; + expect(result.outcome).toBe("cancelled"); + expect(disposeCalls.count).toBe(1); + }); + + it("runner_reports_failure_on_nonzero_exit", async () => { + const { driver, disposeCalls } = createFakeDriver({ exitCode: 7 }); + const onPrompt = vi.fn(); + const result = await runDeviceLogin(driver, { onPrompt, timeoutMs: 1000 }); + expect(result.outcome).toBe("failure"); + expect(result.exitCode).toBe(7); + expect(disposeCalls.count).toBe(1); + }); + + it("runner_logs_and_result_contain_no_url_code_or_token_sentinel", async () => { + const logs: string[] = []; + const { driver } = createFakeDriver({ + chunks: [ + `${REAL_SHAPED_URL}\n${REAL_SHAPED_CODE}\n`, + `refresh_token=${TOKEN_SENTINEL}\n`, + ], + exitCode: 0, + }); + const result = await runDeviceLogin(driver, { + onPrompt: () => {}, + timeoutMs: 1000, + log: (line) => { + logs.push(line); + }, + }); + const haystack = `${logs.join("\n")}\n${JSON.stringify(result)}`; + expect(haystack).not.toContain(REAL_SHAPED_URL); + expect(haystack).not.toContain(REAL_SHAPED_CODE); + expect(haystack).not.toContain(TOKEN_SENTINEL); + }); + + it("runner_error_messages_contain_no_url_or_code", async () => { + const { driver, disposeCalls } = createFakeDriver({ + // A driver error whose message embeds secret-bearing text. The runner must + // never let that message reach its own thrown error. + execError: new Error(`network failure while streaming ${REAL_SHAPED_URL} ${REAL_SHAPED_CODE}`), + }); + let caught: unknown; + try { + await runDeviceLogin(driver, { onPrompt: () => {}, timeoutMs: 1000 }); + } catch (error) { + caught = error; + } + expect(caught).toBeInstanceOf(Error); + const message = (caught as Error).message; + expect(message).not.toContain(REAL_SHAPED_URL); + expect(message).not.toContain(REAL_SHAPED_CODE); + // The driver is still disposed on the error path. + expect(disposeCalls.count).toBe(1); + }); + + it("exposes the default login command", () => { + expect(CODEX_DEVICE_LOGIN_COMMAND).toContain("codex"); + expect(CODEX_DEVICE_LOGIN_COMMAND).toContain("--device-auth"); + }); +}); diff --git a/packages/adapters/codex-local/src/server/device-login-runner.ts b/packages/adapters/codex-local/src/server/device-login-runner.ts new file mode 100644 index 00000000000..2eccde880bf --- /dev/null +++ b/packages/adapters/codex-local/src/server/device-login-runner.ts @@ -0,0 +1,206 @@ +import { parseDeviceLoginPrompt, type DeviceLoginPrompt } from "./device-login-parse.js"; + +// The device-login runner. It runs the Codex device-login command through an +// injected {@link SandboxLoginDriver}, surfaces the login prompt one time in +// memory, and handles a timeout and a cancellation. The runner always disposes +// the driver. +// +// Security (Control 1 — secret handling): the runner treats every byte of the +// sandbox stream as secret-bearing, untrusted input. It parses the stream in an +// in-memory buffer only. It drops the buffer as soon as it finds the prompt. It +// never forwards the raw text to a log or an artifact, and it never stores the +// raw text on the result. The runner reports only a fixed, non-secret status. It +// passes the prompt one time through the in-memory `onPrompt` callback. It passes +// the credential bytes one time through the in-memory `onCredential` callback. +// The runner keeps the URL, the code, and any token byte out of every log line +// and every thrown error. + +/** The default Codex device-login command. */ +export const CODEX_DEVICE_LOGIN_COMMAND = "codex login --device-auth"; + +/** + * The maximum number of characters the runner keeps for the next chunk. The + * Codex prompt is small and puts the URL and the code close together. A sandbox + * can stream a large volume of output before the prompt. So after each parse the + * runner keeps only the most recent characters up to this limit. The retained + * buffer cannot grow without a bound across many chunks. The limit is far larger + * than the prompt, so the trailing window never drops a real prompt that spans a + * chunk boundary. + */ +const MAX_PARSE_BUFFER_CHARS = 64 * 1024; + +/** + * The sandbox side of the device-login run. The runner never calls Daytona + * directly; a caller injects a concrete driver. A production driver binds these + * three methods to a non-persisting Daytona exec path, a file read, and a + * sandbox delete. + */ +export interface SandboxLoginDriver { + /** + * Runs `command` in the sandbox and streams standard output to `onStdout` in + * memory. Resolves with the command exit code when the command ends. A driver + * must not persist the raw output to any durable log. + */ + execStreaming(command: string, onStdout: (chunk: string) => void): Promise<{ exitCode: number | null }>; + /** Reads the bytes of one file from the sandbox. */ + readFile(path: string): Promise; + /** Deletes the sandbox and releases its resources. */ + dispose(): Promise; +} + +/** Receives the parsed prompt one time in memory. The caller displays it. */ +export type DeviceLoginPromptSink = (prompt: DeviceLoginPrompt) => void; + +export type DeviceLoginOutcome = "success" | "failure" | "timeout" | "cancelled"; + +/** The runner result. It never carries a URL, a code, or a token byte. */ +export interface DeviceLoginResult { + outcome: DeviceLoginOutcome; + exitCode: number | null; + promptSurfaced: boolean; +} + +export interface RunDeviceLoginOptions { + /** The login command. Defaults to {@link CODEX_DEVICE_LOGIN_COMMAND}. */ + command?: string; + /** Receives the parsed prompt one time in memory. The caller displays it. */ + onPrompt: DeviceLoginPromptSink; + /** + * Receives the sandbox `auth.json` bytes one time in memory on success. The + * runner reads the bytes with {@link SandboxLoginDriver.readFile} before it + * disposes the driver. Set together with {@link authPath}. + */ + onCredential?: (authBytes: Buffer) => void | Promise; + /** The sandbox path of the credential file to read on success. */ + authPath?: string; + /** The host-side timeout in milliseconds. */ + timeoutMs: number; + /** An optional cancellation signal. */ + signal?: AbortSignal; + /** A non-leaking progress sink. It receives only fixed status lines. */ + log?: (line: string) => void; +} + +type RaceResult = + | { kind: "exit"; exitCode: number | null } + | { kind: "timeout" } + | { kind: "cancelled" }; + +/** + * Races the streaming exec against the timeout and the cancellation signal. The + * exec result resolves the race; the timeout and the signal resolve the race + * with a terminal status. A driver error rejects the race, so the caller can + * convert it to a fixed, non-secret error. A late exec rejection after the race + * already settled is consumed here, so it never becomes an unhandled rejection. + */ +function raceExec( + exec: Promise<{ exitCode: number | null }>, + timeoutMs: number, + signal: AbortSignal | undefined, +): Promise { + return new Promise((resolve, reject) => { + let settled = false; + const cleanup = () => { + clearTimeout(timer); + if (signal) signal.removeEventListener("abort", onAbort); + }; + const finish = (run: () => void) => { + if (settled) return; + settled = true; + cleanup(); + run(); + }; + const timer = setTimeout(() => finish(() => resolve({ kind: "timeout" })), timeoutMs); + const onAbort = () => finish(() => resolve({ kind: "cancelled" })); + if (signal) signal.addEventListener("abort", onAbort, { once: true }); + exec.then( + (value) => finish(() => resolve({ kind: "exit", exitCode: value.exitCode })), + (error) => finish(() => reject(error)), + ); + }); +} + +/** + * Runs the device-login command through `driver`. Surfaces the prompt one time + * through `onPrompt`. On success it reads the credential and surfaces the bytes + * one time through `onCredential`. Returns a fixed status. Always disposes the + * driver. Never logs the raw stream, and never puts a URL, a code, or a token + * into a log line, the result, or a thrown error. + */ +export async function runDeviceLogin( + driver: SandboxLoginDriver, + options: RunDeviceLoginOptions, +): Promise { + const { onPrompt, onCredential, authPath, timeoutMs, signal } = options; + const command = options.command ?? CODEX_DEVICE_LOGIN_COMMAND; + const log = options.log ?? (() => {}); + + let promptSurfaced = false; + // The in-memory parse buffer. The runner drops it as soon as it finds the + // prompt, so the secret-bearing stream never lives longer than one parse. The + // runner parses the full buffer, and it includes the whole new chunk. So a + // prompt at the start of one large chunk still parses. The runner bounds only + // the buffer that it keeps for the next chunk to {@link MAX_PARSE_BUFFER_CHARS}. + // The runner keeps the trailing window and drops the oldest characters. The + // prompt puts the URL and the code close together, so the trailing window + // always holds a real prompt that spans a chunk boundary. + let buffer = ""; + const onStdout = (chunk: string): void => { + if (promptSurfaced) return; + buffer += chunk; + const prompt = parseDeviceLoginPrompt(buffer); + if (prompt) { + promptSurfaced = true; + buffer = ""; + onPrompt(prompt); + return; + } + if (buffer.length > MAX_PARSE_BUFFER_CHARS) { + buffer = buffer.slice(buffer.length - MAX_PARSE_BUFFER_CHARS); + } + }; + + try { + if (signal?.aborted) { + log("[paperclip] Device login cancelled before start."); + return { outcome: "cancelled", exitCode: null, promptSurfaced }; + } + + const exec = driver.execStreaming(command, onStdout); + const raced = await raceExec(exec, timeoutMs, signal); + + if (raced.kind === "timeout") { + log("[paperclip] Device login timed out; disposing the sandbox."); + return { outcome: "timeout", exitCode: null, promptSurfaced }; + } + if (raced.kind === "cancelled") { + log("[paperclip] Device login cancelled; disposing the sandbox."); + return { outcome: "cancelled", exitCode: null, promptSurfaced }; + } + + const exitCode = raced.exitCode; + if (exitCode !== 0) { + log("[paperclip] Device login command ended with a non-zero exit code."); + return { outcome: "failure", exitCode, promptSurfaced }; + } + + if (onCredential && authPath) { + const authBytes = await driver.readFile(authPath); + await onCredential(authBytes); + } + log("[paperclip] Device login command ended successfully."); + return { outcome: "success", exitCode, promptSurfaced }; + } catch { + // Convert any driver error to a fixed, non-secret error. The original error + // may embed streamed bytes, so the runner never propagates its message. + throw new Error("device login failed: the sandbox login command errored."); + } finally { + // Always dispose the driver. A dispose error must not leak or mask the + // result, so the runner swallows it and logs a fixed line. + try { + await driver.dispose(); + } catch { + log("[paperclip] Device login: the sandbox dispose step errored."); + } + } +} diff --git a/packages/shared/src/constants.ts b/packages/shared/src/constants.ts index 1e476a26ec7..54e1252e6fe 100644 --- a/packages/shared/src/constants.ts +++ b/packages/shared/src/constants.ts @@ -221,7 +221,12 @@ export const ISSUE_HARNESS_KINDS = ["skill_test"] as const; export type IssueHarnessKind = (typeof ISSUE_HARNESS_KINDS)[number]; export const MAX_ISSUE_REQUEST_DEPTH = 1024; -export const SUMMARY_SLOT_SCOPE_KINDS = ["project", "workspaces_overview", "project_workspace"] as const; +export const SUMMARY_SLOT_SCOPE_KINDS = [ + "project", + "workspaces_overview", + "project_workspace", + "execution_workspace", +] as const; export type SummarySlotScopeKind = (typeof SUMMARY_SLOT_SCOPE_KINDS)[number]; export const SUMMARY_SLOT_KEYS = ["header"] as const; export type SummarySlotKey = (typeof SUMMARY_SLOT_KEYS)[number]; diff --git a/packages/shared/src/portability-zip.test.ts b/packages/shared/src/portability-zip.test.ts index f128b701b85..a6b20335edd 100644 --- a/packages/shared/src/portability-zip.test.ts +++ b/packages/shared/src/portability-zip.test.ts @@ -270,7 +270,7 @@ describe("readZipArchive", () => { it("bounds a highly compressible DEFLATE entry at the per-entry decompressed limit", async () => { // Compresses tiny but expands to 8 KiB; a 1 KiB cap must reject it before it - // materializes. Real packages sit far under the 256 MB production default. + // materializes. Real packages sit far under the 512 MB production default. const bomb = new TextEncoder().encode("a".repeat(8 * 1024)); const archive = buildZip([{ path: "bomb.txt", bytes: bomb, method: 8 }], "paperclip-demo"); await expect( diff --git a/packages/shared/src/portability-zip.ts b/packages/shared/src/portability-zip.ts index ea531512ed1..f4b77e7e5de 100644 --- a/packages/shared/src/portability-zip.ts +++ b/packages/shared/src/portability-zip.ts @@ -20,10 +20,13 @@ const strictTextDecoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: tru // malicious highly-compressible package cannot exhaust server memory: a // per-entry ceiling (passed to zlib as maxOutputLength, so it fails before // over-allocating) plus an aggregate ceiling across all entries. Both sit far -// above any real company package (inline JSON was historically capped at 64MB) -// yet far below what a bomb would need. -export const MAX_ZIP_ENTRY_DECOMPRESSED_BYTES = 256 * 1024 * 1024; -export const MAX_ZIP_TOTAL_DECOMPRESSED_BYTES = 512 * 1024 * 1024; +// above any real company package yet far below what a bomb would need. These +// are only fallback defaults: the server import route passes explicit limits +// scaled from its configured upload cap. The per-entry ceiling stays at 512 MB +// even for large caps because a text or base64 entry beyond that cannot +// materialize as a JS string (V8's string length limit) regardless. +export const MAX_ZIP_ENTRY_DECOMPRESSED_BYTES = 512 * 1024 * 1024; +export const MAX_ZIP_TOTAL_DECOMPRESSED_BYTES = 1024 * 1024 * 1024; export const binaryContentTypeByExtension: Record = { ".gif": "image/gif", diff --git a/packages/shared/src/summary-slot.test.ts b/packages/shared/src/summary-slot.test.ts index 46c1de5f496..5f8e222dea4 100644 --- a/packages/shared/src/summary-slot.test.ts +++ b/packages/shared/src/summary-slot.test.ts @@ -20,7 +20,7 @@ describe("summary slot shared contract", () => { ); }); - it("allows scoped project and project-workspace header slots", () => { + it("allows scoped project, project-workspace, and execution-workspace header slots", () => { expect(summarySlotScopeSelectorSchema.parse({ scopeKind: "project", scopeId, @@ -32,6 +32,12 @@ describe("summary slot shared contract", () => { scopeId, slotKey: "header", })).toEqual({ scopeKind: "project_workspace", scopeId, slotKey: "header" }); + + expect(summarySlotScopeSelectorSchema.parse({ + scopeKind: "execution_workspace", + scopeId, + slotKey: "header", + })).toEqual({ scopeKind: "execution_workspace", scopeId, slotKey: "header" }); }); it("treats workspaces_overview as a company-scoped singleton", () => { @@ -56,6 +62,10 @@ describe("summary slot shared contract", () => { scopeKind: "project_workspace", slotKey: "header", })).toThrow("project_workspace summary slots require scopeId"); + expect(() => summarySlotScopeSelectorSchema.parse({ + scopeKind: "execution_workspace", + slotKey: "header", + })).toThrow("execution_workspace summary slots require scopeId"); }); it("validates summary write payload revision and generation metadata", () => { diff --git a/packages/shared/src/types/company-portability.ts b/packages/shared/src/types/company-portability.ts index 3cc0eb4a6f0..a32486b42fe 100644 --- a/packages/shared/src/types/company-portability.ts +++ b/packages/shared/src/types/company-portability.ts @@ -200,6 +200,14 @@ export interface CompanyPortabilityIssueManifestEntry { workProducts?: CompanyPortabilityIssueWorkProductManifestEntry[]; monitor?: CompanyPortabilityIssueMonitorManifestEntry | null; attachments?: CompanyPortabilityIssueAttachmentManifestEntry[]; + /** Slug of the parent task when it is part of the same bundle (schemaVersion >= 7). */ + parentSlug?: string | null; + /** Preserved source timestamps as ISO strings (schemaVersion >= 7); absent in older bundles. */ + createdAt?: string | null; + updatedAt?: string | null; + startedAt?: string | null; + completedAt?: string | null; + cancelledAt?: string | null; metadata: Record | null; } diff --git a/packages/shared/src/validators/company-portability.ts b/packages/shared/src/validators/company-portability.ts index eb766e8ecf2..dcd1bdc4db5 100644 --- a/packages/shared/src/validators/company-portability.ts +++ b/packages/shared/src/validators/company-portability.ts @@ -222,6 +222,12 @@ export const portabilityIssueManifestEntrySchema = z.object({ workProducts: z.array(portabilityIssueWorkProductManifestEntrySchema).default([]), monitor: portabilityIssueMonitorManifestEntrySchema.nullable().default(null), attachments: z.array(portabilityIssueAttachmentManifestEntrySchema).default([]), + parentSlug: z.string().min(1).nullable().optional(), + createdAt: z.string().datetime().nullable().optional(), + updatedAt: z.string().datetime().nullable().optional(), + startedAt: z.string().datetime().nullable().optional(), + completedAt: z.string().datetime().nullable().optional(), + cancelledAt: z.string().datetime().nullable().optional(), metadata: z.record(z.string(), z.unknown()).nullable(), }); diff --git a/packages/skills-catalog/catalog/bundled/paperclip-operations/summarize-status/SKILL.md b/packages/skills-catalog/catalog/bundled/paperclip-operations/summarize-status/SKILL.md index fe14c3840f6..fb303e99d33 100644 --- a/packages/skills-catalog/catalog/bundled/paperclip-operations/summarize-status/SKILL.md +++ b/packages/skills-catalog/catalog/bundled/paperclip-operations/summarize-status/SKILL.md @@ -15,7 +15,7 @@ tags: # Summarize status -You are the Summarizer. Turn the current state of a Paperclip scope — a project, the workspaces overview, or a single project workspace — into a short, honest, human-readable Markdown summary and write it back to that scope's **summary slot** as a new revision. +You are the Summarizer. Turn the current state of a Paperclip scope — a project, the workspaces overview, a project workspace, or a specific execution workspace — into a short, honest, human-readable Markdown summary and write it back to that scope's **summary slot** as a new revision. **Open with what the reader needs to do.** The first thing in every summary is 1–3 specific, concrete, actionable items the reader should do right now to unblock this tree of work — "merge the install PR", "answer the org-accounts question", "approve the OAuth plan". Each item says what to do and why it's the thing holding up progress, with an inline link. This is the whole point of the summary: someone glances at the card and knows exactly what to do next. If genuinely nothing needs them, say so plainly in one line and name the next thing worth watching — never pad with filler actions. @@ -27,7 +27,7 @@ This is a **read-and-report** loop. You never change the underlying issues, work ## When to use -- A summary-generation issue is assigned to you naming a scope (`project`, `workspaces_overview`, or `project_workspace`) and slot (`header`). +- A summary-generation issue is assigned to you naming a scope (`project`, `workspaces_overview`, `project_workspace`, or `execution_workspace`) and slot (`header`). - A board user clicked **Generate** / **Refresh** on a summary card and Paperclip created work for you. - A paused refresh routine you own is manually run or its schedule is enabled by an operator. @@ -41,8 +41,8 @@ This is a **read-and-report** loop. You never change the underlying issues, work From the generation issue / run context: -- `scopeKind` — `project`, `workspaces_overview`, or `project_workspace`. -- `scopeId` — the project or project-workspace id. Omitted for `workspaces_overview` (it has no scopeId). +- `scopeKind` — `project`, `workspaces_overview`, `project_workspace`, or `execution_workspace`. +- `scopeId` — the project, project-workspace, or execution-workspace id. Omitted for `workspaces_overview` (it has no scopeId). - `slotKey` — currently always `header`. - `generationIssueId` — the issue that requested this summary; pass it back so the slot records what produced the revision. - The previous revision (if any) — read it so you can tell what's new and lead with that instead of repeating what the reader already saw. @@ -55,6 +55,7 @@ Use these routes directly. Do not guess unscoped `/api/issues` or alternate summ - Read the current slot: `GET /api/companies/{companyId}/summary-slots/{scopeKind}/{slotKey}?scopeId=...` - Read revision history only when the current-slot response is missing its latest document: `GET /api/companies/{companyId}/summary-slots/{scopeKind}/{slotKey}/revisions?scopeId=...` - Gather project issues: `GET /api/companies/{companyId}/issues?projectId=...` +- Gather execution-workspace issues: `GET /api/companies/{companyId}/issues?executionWorkspaceId=...` - Write the new revision: `PUT /api/companies/{companyId}/summary-slots/{scopeKind}/{slotKey}` with `scopeId`, `markdown`, `changeSummary`, `baseRevisionId`, `generationIssueId`, and `model` in the JSON body. For `workspaces_overview`, omit `scopeId` from the read query and send it as `null` in the write body. All calls use the run-scoped Paperclip API URL and bearer token already present in the environment. diff --git a/packages/skills-catalog/generated/catalog.json b/packages/skills-catalog/generated/catalog.json index ad63e175690..d126606ea16 100644 --- a/packages/skills-catalog/generated/catalog.json +++ b/packages/skills-catalog/generated/catalog.json @@ -172,11 +172,11 @@ { "path": "SKILL.md", "kind": "skill", - "sizeBytes": 8682, - "sha256": "c5f459ced4e97e6ae33c3ffbe7b3fb7d4c1fe7bfb0121187f0c2f9000a670b37" + "sizeBytes": 8877, + "sha256": "4cb3d177f5d16e3cc2052d2f80e7fb245ab95e79c1e947abdc84cd4ee3d8c61e" } ], - "contentHash": "sha256:32d2f231a35fc3a658b244f13dd726b2f2bc642db6d3512559fdf9a2b680838d" + "contentHash": "sha256:3e49ee2b8d83d1371fb0f59197bb96c46b64f722540405f46d4a5a756fbe1b89" }, { "id": "paperclipai:bundled:paperclip-operations:task-planning", diff --git a/scripts/__tests__/release-dry-run-notes.test.mjs b/scripts/__tests__/release-dry-run-notes.test.mjs index 2e82a61a0fd..64f70fcc0fe 100644 --- a/scripts/__tests__/release-dry-run-notes.test.mjs +++ b/scripts/__tests__/release-dry-run-notes.test.mjs @@ -38,10 +38,24 @@ get_current_stable_version() { printf '2026.709.0\\n'; } utc_date_iso() { printf '2026-07-10\\n'; } list_public_package_info() { printf 'cli\\tpaperclipai\\t0.0.0\\n'; } next_stable_version() { printf '2026.710.0\\n'; } -next_canary_version() { printf '2026.710.0-canary.0\\n'; } +next_prerelease_version() { printf '2026.710.0-%s.0\\n' "$1"; } release_notes_file() { printf '%s/releases/v%s.md\\n' "$REPO_ROOT" "$1"; } stable_tag_name() { printf 'v%s\\n' "$1"; } -canary_tag_name() { printf 'canary/v%s\\n' "$1"; } +prerelease_tag_name() { printf '%s/v%s\\n' "$1" "$2"; } +require_channel_tag_at_head() { + if [ "\${FAKE_MISSING_CHANNEL_TAG:-}" = "$1" ]; then + echo "Error: HEAD has no $1/v* tag; this channel only publishes commits that already shipped a $1 release." >&2 + exit 1 + fi + echo "[fixture] require_channel_tag_at_head $1" +} +require_channel_tag_absent_at_head() { + if [ "\${FAKE_PRESENT_CHANNEL_TAG:-}" = "$1" ]; then + echo "Error: HEAD already shipped as $1/v2026.710.0-$1.0; delete that tag first if you really want to republish this commit on the $1 channel." >&2 + exit 1 + fi + echo "[fixture] require_channel_tag_absent_at_head $1" +} require_clean_worktree() { :; } require_npm_publish_auth() { :; } git_local_tag_exists() { return 1; } @@ -108,7 +122,7 @@ exit 0 return { binDir, callLog, fixtureDir, script: join(scriptsDir, "release.sh") }; } -function runRelease(args) { +function runRelease(args, extraEnv = {}) { const fixture = createReleaseFixture(); const result = spawnSync(fixture.script, args, { cwd: fixture.fixtureDir, @@ -117,6 +131,7 @@ function runRelease(args) { ...process.env, PATH: `${fixture.binDir}:${process.env.PATH}`, FAKE_CALL_LOG: fixture.callLog, + ...extraEnv, }, }); @@ -148,3 +163,67 @@ test("stable publish still requires release notes before publish work starts", ( assert.doesNotMatch(result.output, /==> Step 2\/7: Building workspace artifacts/); assert.doesNotMatch(result.calls, /^pnpm /m); }); + +test("nightly dry-run publishes under the nightly identity without release notes", () => { + const result = runRelease(["nightly", "--skip-verify", "--dry-run"]); + + assert.equal(result.status, 42); + assert.match(result.output, /\[fixture\] require_channel_tag_at_head canary/); + assert.match(result.output, /Nightly version: 2026\.710\.0-nightly\.0/); + assert.match(result.output, /Dist-tag: nightly/); + assert.match(result.output, /Git tag: nightly\/v2026\.710\.0-nightly\.0/); + assert.doesNotMatch(result.output, /stable release notes file is required/); + assert.match(result.calls, /^pnpm build$/m); +}); + +test("nightly refuses commits that never shipped a canary", () => { + const result = runRelease(["nightly", "--skip-verify", "--dry-run"], { + FAKE_MISSING_CHANNEL_TAG: "canary", + }); + + assert.equal(result.status, 1); + assert.match(result.output, /HEAD has no canary\/v\* tag/); + assert.doesNotMatch(result.calls, /^pnpm /m); +}); + +test("nightly refuses commits that already shipped as a nightly", () => { + const result = runRelease(["nightly", "--skip-verify", "--dry-run"], { + FAKE_PRESENT_CHANNEL_TAG: "nightly", + }); + + assert.equal(result.status, 1); + assert.match(result.output, /HEAD already shipped as nightly\/v/); + assert.doesNotMatch(result.calls, /^pnpm /m); +}); + +test("beta dry-run publishes under the beta identity without release notes", () => { + const result = runRelease(["beta", "--skip-verify", "--dry-run"]); + + assert.equal(result.status, 42); + assert.match(result.output, /\[fixture\] require_channel_tag_at_head nightly/); + assert.match(result.output, /Beta version: 2026\.710\.0-beta\.0/); + assert.match(result.output, /Dist-tag: beta/); + assert.match(result.output, /Git tag: beta\/v2026\.710\.0-beta\.0/); + assert.doesNotMatch(result.output, /stable release notes file is required/); + assert.match(result.calls, /^pnpm build$/m); +}); + +test("beta refuses commits that already shipped as a beta", () => { + const result = runRelease(["beta", "--skip-verify", "--dry-run"], { + FAKE_PRESENT_CHANNEL_TAG: "beta", + }); + + assert.equal(result.status, 1); + assert.match(result.output, /HEAD already shipped as beta\/v/); + assert.doesNotMatch(result.calls, /^pnpm /m); +}); + +test("beta refuses commits that never shipped a nightly", () => { + const result = runRelease(["beta", "--skip-verify", "--dry-run"], { + FAKE_MISSING_CHANNEL_TAG: "nightly", + }); + + assert.equal(result.status, 1); + assert.match(result.output, /HEAD has no nightly\/v\* tag/); + assert.doesNotMatch(result.calls, /^pnpm /m); +}); diff --git a/scripts/__tests__/release-verify-workflow.test.mjs b/scripts/__tests__/release-verify-workflow.test.mjs index b5911ab4e8b..02de233cb67 100644 --- a/scripts/__tests__/release-verify-workflow.test.mjs +++ b/scripts/__tests__/release-verify-workflow.test.mjs @@ -17,13 +17,50 @@ test("release workflow delegates stable and canary verification to the reusable releaseWorkflow, /verify_canary:\n\s+if: github\.event_name == 'push'\n\s+uses: \.\/\.github\/workflows\/release-verify\.yml\n\s+with:\n\s+ref: \$\{\{ github\.sha \}\}/, ); + // The stable lane is gated on the stable channel since the nightly lane + // was added; a `needs:` line (for example a preflight job) may sit between + // the gate and the delegation. + // The stable preflight resolves source_ref to an immutable SHA exactly + // once; verification must consume that pin, not re-resolve the ref. assert.match( releaseWorkflow, - /verify_stable:\n\s+if: github\.event_name == 'workflow_dispatch'\n\s+uses: \.\/\.github\/workflows\/release-verify\.yml\n\s+with:\n\s+ref: \$\{\{ inputs\.source_ref \}\}/, + /verify_stable:\n\s+if: github\.event_name == 'workflow_dispatch' && inputs\.channel == 'stable'\n(?:\s+needs: [^\n]+\n)?\s+uses: \.\/\.github\/workflows\/release-verify\.yml\n\s+with:\n\s+ref: \$\{\{ needs\.preflight_stable\.outputs\.sha \}\}/, ); assert.doesNotMatch(releaseWorkflow, /verify_(?:canary|stable):[\s\S]*?pnpm test:run(?:\n|$)/); }); +test("onboard smoke container binds beyond loopback so the mapped port is reachable", () => { + const dockerfile = readFileSync(path.join(repoRoot, "docker/Dockerfile.onboard-smoke"), "utf8"); + + // `onboard --yes` without an explicit --bind prefers trusted-local + // defaults and writes a loopback bind, which Docker port mapping cannot + // reach. The smoke container must pin a non-loopback preset. + assert.match(dockerfile, /onboard --yes --bind lan/); +}); + +test("promotion selection guards against sources that predate their channel tooling", () => { + const releaseWorkflow = readWorkflow("release.yml"); + + // Promotions run the source commit's release.sh, so selection must reject + // sources whose tooling does not know the target channel yet. + assert.match(releaseWorkflow, /git show "\$\{sha\}:scripts\/release\.sh" \| grep -qF 'canary\|nightly'/); + assert.match(releaseWorkflow, /git show "\$\{sha\}:scripts\/release\.sh" \| grep -qF 'canary\|nightly\|beta\|stable\)'/); +}); + +test("release smoke workflow extends the container readiness budget for CI", () => { + const smokeWorkflow = readWorkflow("release-smoke.yml"); + const harness = readFileSync(path.join(repoRoot, "scripts/docker-onboard-smoke.sh"), "utf8"); + + // CI containers cold-install paperclipai and embedded postgres, so the + // workflow must extend the harness's local-default readiness budget. + assert.match(smokeWorkflow, /SMOKE_READY_TIMEOUT_SECONDS=\d+/); + const ciBudget = Number(smokeWorkflow.match(/SMOKE_READY_TIMEOUT_SECONDS=(\d+)/)[1]); + assert.ok(ciBudget >= 300, `CI readiness budget ${ciBudget}s should be at least 300s`); + + assert.match(harness, /SMOKE_READY_TIMEOUT_SECONDS="\$\{SMOKE_READY_TIMEOUT_SECONDS:-\d+\}"/); + assert.match(harness, /wait_for_http "\$PAPERCLIP_PUBLIC_URL\/api\/health" "\$SMOKE_READY_TIMEOUT_SECONDS" 1/); +}); + test("release verify workflow covers the same split test surface as stable PR verification", () => { const verifyWorkflow = readWorkflow("release-verify.yml"); @@ -47,6 +84,15 @@ test("release verify workflow covers the same split test surface as stable PR ve assert.match(verifyWorkflow, new RegExp(`shard_index: ${shardIndex}[\\s\\S]*?shard_count: 5`)); } + // workspaces-a splits with Vitest native --shard in pr.yml; release + // verification must keep the same two-shard coverage. + for (const shardIndex of [0, 1]) { + assert.match( + verifyWorkflow, + new RegExp(`group: general-workspaces-a[\\s\\S]*?shard_index: ${shardIndex}\\n\\s+shard_count: 2`), + ); + } + assert.match(verifyWorkflow, /pnpm test:run:general -- --group/); assert.match(verifyWorkflow, /pnpm test:run:serialized -- --shard-index/); }); diff --git a/scripts/__tests__/run-vitest-stable-shard.test.mjs b/scripts/__tests__/run-vitest-stable-shard.test.mjs index f3ef491e8f1..ea74a3de127 100644 --- a/scripts/__tests__/run-vitest-stable-shard.test.mjs +++ b/scripts/__tests__/run-vitest-stable-shard.test.mjs @@ -77,9 +77,37 @@ test("a route/authz suite never leaks into the general-server shards", () => { } }); -test("shard flags are rejected for the parallel workspace groups", () => { - const result = dryRun(["--mode", "general", "--group", "general-workspaces-a", "--shard-index", "0", "--shard-count", "3"]); - assert.notEqual(result.status, 0, "workspace groups must not accept shard flags"); +test("shard flags are rejected for the workspaces-b group", () => { + const result = dryRun(["--mode", "general", "--group", "general-workspaces-b", "--shard-index", "0", "--shard-count", "3"]); + assert.notEqual(result.status, 0, "workspaces-b must not accept shard flags"); +}); + +test("workspaces-a shards map to Vitest native --shard slices over a stable project list", () => { + const shards = [0, 1].map((index) => + dryRunJson([ + "--mode", "general", "--group", "general-workspaces-a", + "--shard-index", String(index), "--shard-count", "2", + ]), + ); + + assert.deepEqual( + shards.map((shard) => shard.workspacesVitestShard), + ["1/2", "2/2"], + "each matrix job must pass its own --shard slice to vitest", + ); + // Vitest's --shard partitions each project's file list deterministically, so + // an identical project list across jobs is what guarantees complete, + // non-overlapping coverage of the lane. + assert.deepEqual(shards[0].workspaceProjects, shards[1].workspaceProjects); + assert.ok(shards[0].workspaceProjects.length > 0, "workspaces-a must run at least one project"); + + const unsharded = dryRunJson(["--mode", "general", "--group", "general-workspaces-a"]); + assert.deepEqual( + unsharded.workspaceProjects, + shards[0].workspaceProjects, + "sharding must not change which projects the lane covers", + ); + assert.equal(unsharded.workspacesVitestShard, null); }); test("duration-aware partition balances skewed weights better than round-robin", () => { diff --git a/scripts/docker-onboard-smoke.sh b/scripts/docker-onboard-smoke.sh index c2830ff0f5a..548931b5574 100755 --- a/scripts/docker-onboard-smoke.sh +++ b/scripts/docker-onboard-smoke.sh @@ -13,6 +13,11 @@ PAPERCLIP_DEPLOYMENT_MODE="${PAPERCLIP_DEPLOYMENT_MODE:-authenticated}" PAPERCLIP_DEPLOYMENT_EXPOSURE="${PAPERCLIP_DEPLOYMENT_EXPOSURE:-private}" PAPERCLIP_PUBLIC_URL="${PAPERCLIP_PUBLIC_URL:-http://localhost:${HOST_PORT}}" SMOKE_AUTO_BOOTSTRAP="${SMOKE_AUTO_BOOTSTRAP:-true}" +# Seconds to wait for /api/health after the container starts. The container +# cold-installs paperclipai from npm and initializes embedded postgres before +# it can serve health, so CI callers with no warm caches need far more than +# the local default. +SMOKE_READY_TIMEOUT_SECONDS="${SMOKE_READY_TIMEOUT_SECONDS:-90}" SMOKE_ADMIN_NAME="${SMOKE_ADMIN_NAME:-Smoke Admin}" SMOKE_ADMIN_EMAIL="${SMOKE_ADMIN_EMAIL:-smoke-admin@paperclip.local}" SMOKE_ADMIN_PASSWORD="${SMOKE_ADMIN_PASSWORD:-paperclip-smoke-password}" @@ -63,6 +68,9 @@ wait_for_http() { if ! container_is_running; then echo "Smoke bootstrap failed: container $CONTAINER_NAME exited before readiness check completed" >&2 docker logs "$CONTAINER_NAME" >&2 || true + else + echo "Smoke bootstrap failed: $url not ready after ${attempts} attempts; container is still running. Last container logs:" >&2 + docker logs --tail 150 "$CONTAINER_NAME" >&2 || true fi return 1 } @@ -278,7 +286,7 @@ fi TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/paperclip-onboard-smoke.XXXXXX")" COOKIE_JAR="$TMP_DIR/cookies.txt" -if ! wait_for_http "$PAPERCLIP_PUBLIC_URL/api/health" 90 1; then +if ! wait_for_http "$PAPERCLIP_PUBLIC_URL/api/health" "$SMOKE_READY_TIMEOUT_SECONDS" 1; then echo "Smoke bootstrap failed: server did not become ready at $PAPERCLIP_PUBLIC_URL/api/health" >&2 exit 1 fi diff --git a/scripts/release-lib.sh b/scripts/release-lib.sh index 67c35919f0a..6d1c37fe522 100644 --- a/scripts/release-lib.sh +++ b/scripts/release-lib.sh @@ -197,13 +197,24 @@ process.stdout.write(`${stableSlot}.${max + 1}`); NODE } -next_canary_version() { - local stable_version="$1" - shift +require_prerelease_channel() { + case "$1" in + canary|nightly|beta) ;; + *) release_fail "unknown prerelease channel: $1" ;; + esac +} - node - "$stable_version" "$@" <<'NODE' -const stable = process.argv[2]; -const packageNames = process.argv.slice(3); +next_prerelease_version() { + local channel="$1" + local stable_version="$2" + shift 2 + + require_prerelease_channel "$channel" + + node - "$channel" "$stable_version" "$@" <<'NODE' +const channel = process.argv[2]; +const stable = process.argv[3]; +const packageNames = process.argv.slice(4); const { execSync } = require("node:child_process"); const { readFileSync } = require("node:fs"); @@ -218,7 +229,7 @@ if (process.env.RELEASE_PACKAGE_VERSIONS_FILE) { } } -const pattern = new RegExp(`^${stable.replace(/\./g, '\\.')}-canary\\.(\\d+)$`); +const pattern = new RegExp(`^${stable.replace(/\./g, '\\.')}-${channel}\\.(\\d+)$`); let max = -1; for (const packageName of packageNames) { @@ -249,10 +260,16 @@ for (const packageName of packageNames) { } } -process.stdout.write(`${stable}-canary.${max + 1}`); +process.stdout.write(`${stable}-${channel}.${max + 1}`); NODE } +next_canary_version() { + local stable_version="$1" + shift + next_prerelease_version canary "$stable_version" "$@" +} + release_notes_file() { printf '%s/releases/v%s.md\n' "$REPO_ROOT" "$1" } @@ -261,8 +278,13 @@ stable_tag_name() { printf 'v%s\n' "$1" } +prerelease_tag_name() { + require_prerelease_channel "$1" + printf '%s/v%s\n' "$1" "$2" +} + canary_tag_name() { - printf 'canary/v%s\n' "$1" + prerelease_tag_name canary "$1" } npm_package_version_exists() { @@ -369,11 +391,14 @@ publish_package_to_npm() { return 0 fi - if [ "$dist_tag" != "canary" ]; then - release_warn "Not retrying ${package_name}@${package_version} without provenance for dist-tag ${dist_tag}." - rm -f "$publish_log" - return 1 - fi + case "$dist_tag" in + canary|nightly) ;; + *) + release_warn "Not retrying ${package_name}@${package_version} without provenance for dist-tag ${dist_tag}." + rm -f "$publish_log" + return 1 + ;; + esac release_warn "Retrying ${package_name}@${package_version} once with npm provenance disabled." if run_package_publish "$publish_tool" "$dist_tag" true; then @@ -468,6 +493,33 @@ require_on_master_branch() { fi } +# Promotion channels only republish commits that already shipped on the +# previous lane, so the source commit must carry that lane's release tag. +require_channel_tag_at_head() { + local channel="$1" + + require_prerelease_channel "$channel" + + if ! git -C "$REPO_ROOT" tag --points-at HEAD | grep -q "^${channel}/v"; then + release_fail "HEAD has no ${channel}/v* tag; this channel only publishes commits that already shipped a ${channel} release." + fi +} + +# The inverse guard: a commit ships on a promotion channel at most once, so +# concurrent or repeated runs cannot double-publish it. Delete the lane tag +# first if a republish is genuinely intended. +require_channel_tag_absent_at_head() { + local channel="$1" + local existing + + require_prerelease_channel "$channel" + + existing="$(git -C "$REPO_ROOT" tag --points-at HEAD | grep "^${channel}/v" | head -1 || true)" + if [ -n "$existing" ]; then + release_fail "HEAD already shipped as ${existing}; delete that tag first if you really want to republish this commit on the ${channel} channel." + fi +} + require_npm_publish_auth() { local dry_run="$1" diff --git a/scripts/release-registry-versions.test.mjs b/scripts/release-registry-versions.test.mjs index 2915dae665c..ff107d9b0eb 100644 --- a/scripts/release-registry-versions.test.mjs +++ b/scripts/release-registry-versions.test.mjs @@ -195,3 +195,64 @@ test("next_stable_version falls back to npm view without a versions file", () => assert.equal(result.output, "2026.707.2"); assert.match(result.calls, /^npm view @paperclipai\/present versions --json$/m); }); + +test("next_prerelease_version counts per channel so nightly numbering ignores canaries", () => { + const fixture = makeFixture(); + const versionsFile = join(fixture.fixtureDir, "versions.json"); + writeFileSync( + versionsFile, + JSON.stringify({ + "@paperclipai/a": ["2026.707.1-canary.4", "2026.707.1-nightly.0", "2026.707.1-nightly.1"], + }), + ); + + const result = runReleaseLibHelper( + 'next_prerelease_version nightly 2026.707.1 "@paperclipai/a"', + fixture, + { RELEASE_PACKAGE_VERSIONS_FILE: versionsFile }, + ); + + assert.equal(result.status, 0); + assert.equal(result.output, "2026.707.1-nightly.2"); + assert.doesNotMatch(result.calls, /npm view/); +}); + +test("next_prerelease_version counts beta numbering independently of other channels", () => { + const fixture = makeFixture(); + const versionsFile = join(fixture.fixtureDir, "versions.json"); + writeFileSync( + versionsFile, + JSON.stringify({ + "@paperclipai/a": ["2026.707.1-canary.4", "2026.707.1-nightly.3", "2026.707.1-beta.0"], + }), + ); + + const result = runReleaseLibHelper( + 'next_prerelease_version beta 2026.707.1 "@paperclipai/a"', + fixture, + { RELEASE_PACKAGE_VERSIONS_FILE: versionsFile }, + ); + + assert.equal(result.status, 0); + assert.equal(result.output, "2026.707.1-beta.1"); + assert.doesNotMatch(result.calls, /npm view/); +}); + +test("next_prerelease_version rejects unknown channels", () => { + const fixture = makeFixture(); + const result = runReleaseLibHelper( + 'next_prerelease_version weekly 2026.707.1 "@paperclipai/a"', + fixture, + ); + + assert.equal(result.status, 1); + assert.match(result.output, /unknown prerelease channel: weekly/); +}); + +test("prerelease_tag_name namespaces tags by channel", () => { + const fixture = makeFixture(); + const result = runReleaseLibHelper("prerelease_tag_name nightly 2026.707.1-nightly.2", fixture); + + assert.equal(result.status, 0); + assert.equal(result.output.trim(), "nightly/v2026.707.1-nightly.2"); +}); diff --git a/scripts/release.sh b/scripts/release.sh index 8e42505af16..db3a4ea90a4 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -18,11 +18,13 @@ cleanup_on_exit=false usage() { cat <<'EOF' Usage: - ./scripts/release.sh [--date YYYY-MM-DD] [--dry-run] [--skip-verify] [--print-version] + ./scripts/release.sh [--date YYYY-MM-DD] [--dry-run] [--skip-verify] [--print-version] Examples: ./scripts/release.sh canary ./scripts/release.sh canary --date 2026-03-17 --dry-run + ./scripts/release.sh nightly --dry-run + ./scripts/release.sh beta --dry-run ./scripts/release.sh stable ./scripts/release.sh stable --date 2026-03-17 --dry-run ./scripts/release.sh stable --date 2026-03-18 --print-version @@ -32,6 +34,13 @@ Notes: zero-padded UTC day, and P is the same-day stable patch slot. - Canary releases publish YYYY.MDD.P-canary.N under the npm dist-tag "canary" and create the git tag canary/vYYYY.MDD.P-canary.N. + - Nightly releases republish a commit that already shipped a canary (HEAD + must carry a canary/v* tag) as YYYY.MDD.P-nightly.N under the npm + dist-tag "nightly", with the git tag nightly/vYYYY.MDD.P-nightly.N. + The version dates the nightly cut, not the source canary. + - Beta releases republish a commit that already shipped a nightly (HEAD + must carry a nightly/v* tag) as YYYY.MDD.P-beta.N under the npm + dist-tag "beta", with the git tag beta/vYYYY.MDD.P-beta.N. - Stable releases publish YYYY.MDD.P under the npm dist-tag "latest" and create the git tag vYYYY.MDD.P. - Non-dry-run stable release notes must already exist at releases/vYYYY.MDD.P.md. @@ -85,7 +94,7 @@ set_cleanup_trap() { while [ $# -gt 0 ]; do case "$1" in - canary|stable) + canary|nightly|beta|stable) if [ -n "$channel" ]; then release_fail "only one release channel may be provided." fi @@ -146,9 +155,24 @@ DIST_TAG="latest" if [ "$channel" = "canary" ]; then require_on_master_branch - TARGET_PUBLISH_VERSION="$(next_canary_version "$TARGET_STABLE_VERSION" "${PUBLIC_PACKAGE_NAMES[@]}")" + TARGET_PUBLISH_VERSION="$(next_prerelease_version canary "$TARGET_STABLE_VERSION" "${PUBLIC_PACKAGE_NAMES[@]}")" DIST_TAG="canary" - tag_name="$(canary_tag_name "$TARGET_PUBLISH_VERSION")" + tag_name="$(prerelease_tag_name canary "$TARGET_PUBLISH_VERSION")" +elif [ "$channel" = "nightly" ]; then + # Nightly promotes an already-shipped canary commit, so it runs from a + # detached checkout of that commit rather than the master branch tip. + require_channel_tag_at_head canary + require_channel_tag_absent_at_head nightly + TARGET_PUBLISH_VERSION="$(next_prerelease_version nightly "$TARGET_STABLE_VERSION" "${PUBLIC_PACKAGE_NAMES[@]}")" + DIST_TAG="nightly" + tag_name="$(prerelease_tag_name nightly "$TARGET_PUBLISH_VERSION")" +elif [ "$channel" = "beta" ]; then + # Beta promotes an already-shipped nightly commit. + require_channel_tag_at_head nightly + require_channel_tag_absent_at_head beta + TARGET_PUBLISH_VERSION="$(next_prerelease_version beta "$TARGET_STABLE_VERSION" "${PUBLIC_PACKAGE_NAMES[@]}")" + DIST_TAG="beta" + tag_name="$(prerelease_tag_name beta "$TARGET_PUBLISH_VERSION")" else tag_name="$(stable_tag_name "$TARGET_STABLE_VERSION")" fi @@ -193,11 +217,12 @@ release_info " Last stable tag: ${LAST_STABLE_TAG:-}" release_info " Current stable version: $CURRENT_STABLE_VERSION" release_info " Release date (UTC): $RELEASE_DATE" release_info " Target stable version: $TARGET_STABLE_VERSION" -if [ "$channel" = "canary" ]; then - release_info " Canary version: $TARGET_PUBLISH_VERSION" -else - release_info " Stable version: $TARGET_PUBLISH_VERSION" -fi +case "$channel" in + canary) release_info " Canary version: $TARGET_PUBLISH_VERSION" ;; + nightly) release_info " Nightly version: $TARGET_PUBLISH_VERSION" ;; + beta) release_info " Beta version: $TARGET_PUBLISH_VERSION" ;; + *) release_info " Stable version: $TARGET_PUBLISH_VERSION" ;; +esac release_info " Dist-tag: $DIST_TAG" release_info " Git tag: $tag_name" if [ "$channel" = "stable" ]; then @@ -358,14 +383,17 @@ release_info "" if [ "$dry_run" = true ]; then release_info "Dry run complete for $channel ${TARGET_PUBLISH_VERSION}." else - if [ "$channel" = "canary" ]; then - release_info "Published canary ${TARGET_PUBLISH_VERSION}." - release_info "Install with: npx paperclipai@canary onboard" - release_info "Next step: git push ${PUBLISH_REMOTE} refs/tags/${tag_name}" - else - release_info "Published stable ${TARGET_PUBLISH_VERSION}." - release_info "Next steps:" - release_info " git push ${PUBLISH_REMOTE} refs/tags/${tag_name}" - release_info " ./scripts/create-github-release.sh $TARGET_STABLE_VERSION" - fi + case "$channel" in + canary|nightly|beta) + release_info "Published $channel ${TARGET_PUBLISH_VERSION}." + release_info "Install with: npx paperclipai@$channel onboard" + release_info "Next step: git push ${PUBLISH_REMOTE} refs/tags/${tag_name}" + ;; + *) + release_info "Published stable ${TARGET_PUBLISH_VERSION}." + release_info "Next steps:" + release_info " git push ${PUBLISH_REMOTE} refs/tags/${tag_name}" + release_info " ./scripts/create-github-release.sh $TARGET_STABLE_VERSION" + ;; + esac fi diff --git a/scripts/run-vitest-stable.mjs b/scripts/run-vitest-stable.mjs index 7c5a6c9c8d1..d8b334e58f0 100644 --- a/scripts/run-vitest-stable.mjs +++ b/scripts/run-vitest-stable.mjs @@ -208,10 +208,11 @@ function parseCliOptions(argv) { const shardAllowed = mode === serializedModeName || - (mode === generalModeName && group === generalServerGroupName); + (mode === generalModeName && + (group === generalServerGroupName || group === generalWorkspacesAGroupName)); if (!shardAllowed && shardIndex !== null) { fail( - "--shard-index/--shard-count are only valid with --mode serialized or --mode general --group general-server.", + "--shard-index/--shard-count are only valid with --mode serialized, --mode general --group general-server, or --mode general --group general-workspaces-a.", ); } @@ -287,9 +288,16 @@ function runGeneralSuites(routeTests) { } } -function runProjectGroup(projects, groupName) { +function runProjectGroup(projects, groupName, shardIndex = null, shardCount = null) { + // With shard args, lean on Vitest's native --shard: each matrix job runs the + // same per-project invocations but only its slice of each project's test + // files. Vitest's sharding is deterministic for an identical file list, so + // the matrix jobs form a complete, non-overlapping cover of every project. + const shardArgs = + shardCount !== null && shardCount > 1 ? [`--shard=${shardIndex + 1}/${shardCount}`] : []; + const shardSuffix = shardArgs.length > 0 ? ` shard ${shardIndex + 1}/${shardCount}` : ""; for (const project of projects) { - runVitest(["--project", project], `${groupName} project ${project}`); + runVitest(["--project", project, ...shardArgs], `${groupName} project ${project}${shardSuffix}`); } } @@ -335,7 +343,11 @@ function runGeneralGroup(routeTests, groupName, shardIndex = null, shardCount = } if (groupName === generalWorkspacesAGroupName) { - runProjectGroup(generalWorkspacesAProjects, groupName); + // The ui project dominates this lane (~224s of a 319s job in actions run + // 31371439296, 2026-08-10, where workspaces-a was the slowest PR check). + // Its 439 test files shard cleanly with Vitest's native --shard, so the + // lane splits across runners without a duration manifest. + runProjectGroup(generalWorkspacesAProjects, groupName, shardIndex, shardCount); return; } @@ -416,6 +428,18 @@ if (options.dryRun) { generalServerShardDurations, ) : null, + workspaceProjects: + options.group === generalWorkspacesAGroupName + ? generalWorkspacesAProjects + : options.group === generalWorkspacesBGroupName + ? generalWorkspacesBProjects + : null, + workspacesVitestShard: + options.group === generalWorkspacesAGroupName && + options.shardCount !== null && + options.shardCount > 1 + ? `${options.shardIndex + 1}/${options.shardCount}` + : null, }, null, 2, diff --git a/scripts/verify-release-registry-state.mjs b/scripts/verify-release-registry-state.mjs index 50d4ad9464e..c4848bafaba 100644 --- a/scripts/verify-release-registry-state.mjs +++ b/scripts/verify-release-registry-state.mjs @@ -3,6 +3,9 @@ import { pathToFileURL } from "node:url"; const CANARY_VERSION_RE = /-canary\.\d+$/; +const PRERELEASE_VERSION_RE = /-(?:canary|nightly|beta)\.\d+$/; +// Channels that publish prerelease versions and must never move `latest`. +const PRERELEASE_CHANNELS = new Set(["canary", "nightly", "beta"]); const EXIT_RETRIABLE_FAILURE = 1; const EXIT_NON_RETRIABLE_FAILURE = 2; @@ -10,6 +13,10 @@ export function isCanaryVersion(version) { return CANARY_VERSION_RE.test(version); } +export function isPrereleaseVersion(version) { + return PRERELEASE_VERSION_RE.test(version); +} + function createExitError(message, exitCode = EXIT_RETRIABLE_FAILURE) { return Object.assign(new Error(message), { exitCode }); } @@ -22,7 +29,7 @@ function usage() { process.stderr.write( [ "Usage:", - " node scripts/verify-release-registry-state.mjs --channel --dist-tag --target-version --package [--package ...] [--allow-canary-latest]", + " node scripts/verify-release-registry-state.mjs --channel --dist-tag --target-version --package [--package ...] [--allow-canary-latest]", "", ].join("\n"), ); @@ -69,8 +76,8 @@ function parseArgs(argv) { } } - if (options.channel !== "canary" && options.channel !== "stable") { - throw createExitError("--channel must be canary or stable", EXIT_NON_RETRIABLE_FAILURE); + if (!PRERELEASE_CHANNELS.has(options.channel) && options.channel !== "stable") { + throw createExitError("--channel must be canary, nightly, beta, or stable", EXIT_NON_RETRIABLE_FAILURE); } if (!options.distTag) { @@ -284,19 +291,19 @@ export function verifyPackageRegistryProblems({ } } - if (channel === "canary") { + if (PRERELEASE_CHANNELS.has(channel)) { const latestVersion = distTags.latest; - if (latestVersion && isCanaryVersion(latestVersion) && !allowCanaryLatest) { + if (latestVersion && isPrereleaseVersion(latestVersion) && !allowCanaryLatest) { problems.push( createProblem( - `${packageName}: latest dist-tag still resolves to canary ${latestVersion}; if that state is intentional, rerun the verification script directly with --allow-canary-latest`, + `${packageName}: latest dist-tag still resolves to prerelease ${latestVersion}; if that state is intentional, rerun the verification script directly with --allow-canary-latest`, { retriable: false }, ), ); } - if (latestVersion && isCanaryVersion(latestVersion)) { + if (latestVersion && isPrereleaseVersion(latestVersion)) { const latestManifest = requireManifest( packageName, latestVersion, diff --git a/scripts/verify-release-registry-state.test.mjs b/scripts/verify-release-registry-state.test.mjs index bf3b0b0cb24..112d61c307f 100644 --- a/scripts/verify-release-registry-state.test.mjs +++ b/scripts/verify-release-registry-state.test.mjs @@ -6,6 +6,7 @@ import { createManifestLookupKey, fetchRegistryJson, isCanaryVersion, + isPrereleaseVersion, verifyPackageRegistryProblems, verifyPackageRegistryState, } from "./verify-release-registry-state.mjs"; @@ -15,6 +16,13 @@ test("isCanaryVersion matches release canaries", () => { assert.equal(isCanaryVersion("2026.427.0"), false); }); +test("isPrereleaseVersion matches canary and nightly versions", () => { + assert.equal(isPrereleaseVersion("2026.427.0-canary.3"), true); + assert.equal(isPrereleaseVersion("2026.427.0-nightly.0"), true); + assert.equal(isPrereleaseVersion("2026.427.0-beta.2"), true); + assert.equal(isPrereleaseVersion("2026.427.0"), false); +}); + test("collectInternalDependencyProblems flags missing internal versions", () => { const manifest = { dependencies: { @@ -192,7 +200,7 @@ test("verifyPackageRegistryState fails when canary latest is left in place by de allowCanaryLatest: false, }), [ - "@paperclipai/plugin-e2b: latest dist-tag still resolves to canary 2026.425.0-canary.5; if that state is intentional, rerun the verification script directly with --allow-canary-latest", + "@paperclipai/plugin-e2b: latest dist-tag still resolves to prerelease 2026.425.0-canary.5; if that state is intentional, rerun the verification script directly with --allow-canary-latest", "@paperclipai/plugin-e2b@2026.425.0-canary.5 via latest: dependencies requires @paperclipai/plugin-sdk@2026.425.0-canary.5, but npm does not expose that version", ], ); @@ -225,7 +233,37 @@ test("verifyPackageRegistryProblems marks canary latest drift as non-retriable", }); assert.equal(problems[0]?.retriable, false); - assert.match(problems[0]?.message ?? "", /latest dist-tag still resolves to canary/); + assert.match(problems[0]?.message ?? "", /latest dist-tag still resolves to prerelease/); +}); + +test("verifyPackageRegistryProblems accepts the nightly channel and flags nightly latest drift", () => { + const packageDocsByName = new Map([ + [ + "@paperclipai/plugin-e2b", + { + "dist-tags": { + latest: "2026.425.0-nightly.1", + nightly: "2026.427.0-nightly.0", + }, + versions: { + "2026.427.0-nightly.0": {}, + }, + }, + ], + ]); + + const problems = verifyPackageRegistryProblems({ + packageName: "@paperclipai/plugin-e2b", + packageDoc: packageDocsByName.get("@paperclipai/plugin-e2b"), + packageDocsByName, + channel: "nightly", + distTag: "nightly", + targetVersion: "2026.427.0-nightly.0", + allowCanaryLatest: false, + }); + + assert.equal(problems[0]?.retriable, false); + assert.match(problems[0]?.message ?? "", /latest dist-tag still resolves to prerelease 2026\.425\.0-nightly\.1/); }); test("verifyPackageRegistryState allows intentional canary latest but still checks dependencies", () => { diff --git a/server/src/__tests__/body-limits.test.ts b/server/src/__tests__/body-limits.test.ts index 8164e32bd87..2749edcf7f0 100644 --- a/server/src/__tests__/body-limits.test.ts +++ b/server/src/__tests__/body-limits.test.ts @@ -1,12 +1,18 @@ -import { describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { DEFAULT_JSON_BODY_LIMIT, PORTABLE_JSON_BODY_LIMIT, PORTABLE_JSON_BODY_LIMIT_BYTES, + PORTABLE_ZIP_UPLOAD_LIMIT_BYTES, } from "../http/body-limits.js"; describe("HTTP body limits", () => { + afterEach(() => { + vi.unstubAllEnvs(); + vi.resetModules(); + }); + it("keeps the global JSON parser at the established ceiling", () => { expect(DEFAULT_JSON_BODY_LIMIT).toBe("10mb"); }); @@ -16,4 +22,34 @@ describe("HTTP body limits", () => { expect(PORTABLE_JSON_BODY_LIMIT_BYTES).toBe(64 * 1024 * 1024); expect(PORTABLE_JSON_BODY_LIMIT_BYTES).toBeGreaterThan(10 * 1024 * 1024); }); + + it("accepts large real-world compressed company packages by default", () => { + expect(PORTABLE_ZIP_UPLOAD_LIMIT_BYTES).toBe(1024 * 1024 * 1024); + }); + + it("lets operators override the zip upload cap via PAPERCLIP_IMPORT_ZIP_MAX_BYTES", async () => { + vi.stubEnv("PAPERCLIP_IMPORT_ZIP_MAX_BYTES", String(2 * 1024 * 1024 * 1024)); + vi.resetModules(); + const reloaded = await import("../http/body-limits.js"); + expect(reloaded.PORTABLE_ZIP_UPLOAD_LIMIT_BYTES).toBe(2 * 1024 * 1024 * 1024); + }); + + it("falls back to the default when the env override is not a usable number", async () => { + for (const raw of ["unlimited", "0", "-5", "0.5"]) { + vi.stubEnv("PAPERCLIP_IMPORT_ZIP_MAX_BYTES", raw); + vi.resetModules(); + const reloaded = await import("../http/body-limits.js"); + expect(reloaded.PORTABLE_ZIP_UPLOAD_LIMIT_BYTES, `override ${JSON.stringify(raw)}`).toBe( + 1024 * 1024 * 1024, + ); + } + }); + + it("clamps an oversized override so derived guard math cannot overflow", async () => { + vi.stubEnv("PAPERCLIP_IMPORT_ZIP_MAX_BYTES", String(Number.MAX_VALUE)); + vi.resetModules(); + const reloaded = await import("../http/body-limits.js"); + expect(reloaded.PORTABLE_ZIP_UPLOAD_LIMIT_BYTES).toBe(64 * 1024 * 1024 * 1024); + expect(Number.isSafeInteger(reloaded.PORTABLE_ZIP_UPLOAD_LIMIT_BYTES * 4)).toBe(true); + }); }); diff --git a/server/src/__tests__/company-portability-import-batching.test.ts b/server/src/__tests__/company-portability-import-batching.test.ts index 4c32bfe5cd4..0c7226aa746 100644 --- a/server/src/__tests__/company-portability-import-batching.test.ts +++ b/server/src/__tests__/company-portability-import-batching.test.ts @@ -1,6 +1,8 @@ import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest"; import { + agents, assets, + companies, createDb, documentRevisions, documents, @@ -155,7 +157,7 @@ interface SyntheticBundleOptions { documentsPerIssue: number; } -/** Build an inline import bundle with the requested shape, valid at schemaVersion 6. */ +/** Build an inline import bundle with the requested shape, valid at schemaVersion 7. */ function buildSyntheticBundle(options: SyntheticBundleOptions): { rootPath: string; files: Record; @@ -191,7 +193,7 @@ function buildSyntheticBundle(options: SyntheticBundleOptions): { }; } - files[".paperclip.yaml"] = renderPaperclipYaml({ schemaVersion: 6, tasks }); + files[".paperclip.yaml"] = renderPaperclipYaml({ schemaVersion: 7, tasks }); return { rootPath: "batching-bench", files }; } @@ -221,7 +223,7 @@ function buildTouchedIssuesBundle(issueCount: number): { }; } - files[".paperclip.yaml"] = renderPaperclipYaml({ schemaVersion: 6, tasks }); + files[".paperclip.yaml"] = renderPaperclipYaml({ schemaVersion: 7, tasks }); return { rootPath: "inbox-flood", files }; } @@ -371,6 +373,202 @@ describeEmbeddedPostgres("company import batches inserts", () => { expect(mineAfterCreate.filter((issue) => importedIds.has(issue.id))).toEqual([]); }); + it("imports in_progress issues with a null startedAt", async () => { + const companyId = randomUUID(); + const agentId = randomUUID(); + await db.insert(companies).values({ + id: companyId, + name: "Carried Over Co", + issuePrefix: "CAR", + requireBoardApprovalForNewAgents: false, + }); + // pauseAutomations imports assignees paused; paused agents keep assignments. + await db.insert(agents).values({ + id: agentId, + companyId, + name: "Carried Coder", + role: "engineer", + status: "paused", + adapterType: "codex_local", + adapterConfig: {}, + runtimeConfig: {}, + permissions: {}, + }); + + await issueService(db).importIssues(companyId, [{ + id: randomUUID(), + ref: "carried-over", + projectId: null, + projectWorkspaceId: null, + title: "Carried-over work", + description: null, + assigneeAgentId: agentId, + status: "in_progress", + priority: "medium", + billingCode: null, + assigneeAdapterOverrides: null, + executionWorkspaceSettings: null, + labelIds: [], + monitorNotes: null, + monitorScheduledBy: null, + }]); + + const [imported] = await db + .select({ status: issues.status, startedAt: issues.startedAt }) + .from(issues) + .where(eq(issues.companyId, companyId)); + expect(imported?.status).toBe("in_progress"); + // A fabricated import-time startedAt made carried-over work look hours + // stale to duration-based sweeps (e.g. the productivity review). + expect(imported?.startedAt).toBeNull(); + }); + + it("preserves bundle timestamps and parent hierarchy through a real import", async () => { + const files: Record = {}; + files["COMPANY.md"] = ['---', 'schema: "agentcompanies/v1"', 'name: "Timestamp Fidelity Co"', '---', '', 'Synthetic import bundle.', ''].join("\n"); + const taskFile = (name: string) => ['---', `name: "${name}"`, "kind: task", '---', '', `${name} body.`, ''].join("\n"); + files["tasks/a-child/TASK.md"] = taskFile("Child"); + files["tasks/m-grandchild/TASK.md"] = taskFile("Grandchild"); + files["tasks/z-parent/TASK.md"] = taskFile("Parent"); + files[".paperclip.yaml"] = renderPaperclipYaml({ + schemaVersion: 7, + tasks: { + // The child slug sorts before its parent so the importer's + // parents-first row ordering (not manifest order) must satisfy the + // parent foreign key. + "a-child": { + status: "todo", + priority: "medium", + parent: "z-parent", + createdAt: "2026-01-05T00:00:00.000Z", + updatedAt: "2026-03-05T00:00:00.000Z", + comments: [{ + body: "Older than the preserved updatedAt.", + authorType: "system", + createdAt: "2026-02-01T00:00:00.000Z", + }], + }, + "m-grandchild": { + status: "todo", + priority: "medium", + parent: "a-child", + createdAt: "2026-01-06T00:00:00.000Z", + updatedAt: "2026-01-07T00:00:00.000Z", + }, + "z-parent": { + status: "done", + priority: "medium", + createdAt: "2026-01-01T00:00:00.000Z", + updatedAt: "2026-03-01T00:00:00.000Z", + completedAt: "2026-02-20T00:00:00.000Z", + comments: [{ + body: "Newer than the preserved updatedAt.", + authorType: "system", + createdAt: "2026-04-01T00:00:00.000Z", + }], + }, + }, + }); + + const portability = companyPortabilityService(db); + const result = await portability.importBundle( + { + source: { type: "inline", rootPath: "timestamp-fidelity", files }, + include: { company: true, agents: false, projects: false, issues: true }, + target: { mode: "new_company", newCompanyName: "Timestamp Fidelity Co" }, + collisionStrategy: "rename", + }, + "user-timestamp-fidelity", + ); + expect(result.warnings).toEqual([]); + + const rows = await db + .select({ + id: issues.id, + title: issues.title, + parentId: issues.parentId, + createdAt: issues.createdAt, + updatedAt: issues.updatedAt, + startedAt: issues.startedAt, + completedAt: issues.completedAt, + }) + .from(issues) + .where(eq(issues.companyId, result.company.id)); + const parent = rows.find((row) => row.title === "Parent")!; + const child = rows.find((row) => row.title === "Child")!; + const grandchild = rows.find((row) => row.title === "Grandchild")!; + + // The parent chain of three lands intact. + expect(parent.parentId).toBeNull(); + expect(child.parentId).toBe(parent.id); + expect(grandchild.parentId).toBe(child.id); + + expect(parent.createdAt).toEqual(new Date("2026-01-01T00:00:00.000Z")); + expect(parent.completedAt).toEqual(new Date("2026-02-20T00:00:00.000Z")); + expect(parent.startedAt).toBeNull(); + expect(child.createdAt).toEqual(new Date("2026-01-05T00:00:00.000Z")); + expect(grandchild.createdAt).toEqual(new Date("2026-01-06T00:00:00.000Z")); + expect(grandchild.updatedAt).toEqual(new Date("2026-01-07T00:00:00.000Z")); + + // A comment older than the preserved updatedAt must not regress it... + expect(child.updatedAt).toEqual(new Date("2026-03-05T00:00:00.000Z")); + // ...while a newer imported comment still bumps recency forward. + expect(parent.updatedAt).toEqual(new Date("2026-04-01T00:00:00.000Z")); + }); + + it("imports a v6 bundle without the timestamp fields using import-time defaults", async () => { + // A little slack absorbs clock drift between this process and Postgres. + const importStart = Date.now() - 1_000; + const files: Record = {}; + files["COMPANY.md"] = ['---', 'schema: "agentcompanies/v1"', 'name: "Legacy Shape Co"', '---', '', 'Synthetic import bundle.', ''].join("\n"); + files["tasks/legacy-task/TASK.md"] = ['---', 'name: "Legacy task"', "kind: task", '---', '', 'Legacy body.', ''].join("\n"); + files[".paperclip.yaml"] = renderPaperclipYaml({ + schemaVersion: 6, + tasks: { + "legacy-task": { + status: "todo", + priority: "medium", + comments: [{ + body: "Predates the import.", + authorType: "system", + createdAt: "2026-01-01T00:00:00.000Z", + }], + }, + }, + }); + + const portability = companyPortabilityService(db); + const result = await portability.importBundle( + { + source: { type: "inline", rootPath: "legacy-shape", files }, + include: { company: true, agents: false, projects: false, issues: true }, + target: { mode: "new_company", newCompanyName: "Legacy Shape Co" }, + collisionStrategy: "rename", + }, + "user-legacy-shape", + ); + expect(result.warnings).toContain( + "This package declares schemaVersion 6 and predates task timestamp and parent link transfer; that task data imports only if the bundle carries it.", + ); + + const [row] = await db + .select({ + parentId: issues.parentId, + createdAt: issues.createdAt, + updatedAt: issues.updatedAt, + startedAt: issues.startedAt, + }) + .from(issues) + .where(eq(issues.companyId, result.company.id)); + expect(row?.parentId).toBeNull(); + expect(row?.startedAt).toBeNull(); + // Without preserved timestamps the row keeps the old behavior: created and + // updated at import time, and the comment bump does not drag updatedAt + // back to the older comment createdAt. + expect(row!.createdAt.getTime()).toBeGreaterThanOrEqual(importStart); + expect(row!.updatedAt.getTime()).toBeGreaterThanOrEqual(importStart); + }); + it("rolls back the whole work-product batch when a later chunk fails", async () => { // Seed a real company + issue to hang work products off of. const bundle = buildSyntheticBundle({ issueCount: 1, commentsPerIssue: 0, documentsPerIssue: 0 }); diff --git a/server/src/__tests__/company-portability.test.ts b/server/src/__tests__/company-portability.test.ts index 9631ddd3b7b..494087e7045 100644 --- a/server/src/__tests__/company-portability.test.ts +++ b/server/src/__tests__/company-portability.test.ts @@ -3579,8 +3579,8 @@ describe("company portability", () => { expect(extension).not.toContain("labelIds"); expect(extension).not.toContain("label-a"); // Fresh exports declare the current bundle shape end-to-end. - expect(extension).toContain("schemaVersion: 6"); - expect(exported.manifest.schemaVersion).toBe(6); + expect(extension).toContain("schemaVersion: 7"); + expect(exported.manifest.schemaVersion).toBe(7); expect(exported.manifest.labels).toEqual([ { name: "bug", color: "#ff0000" }, { name: "urgent", color: "#00ff00" }, @@ -4030,6 +4030,207 @@ describe("company portability", () => { ); }); + function mockTaskHierarchyExportSources() { + projectSvc.list.mockResolvedValue([]); + projectSvc.listWorkspaces.mockResolvedValue([]); + const baseIssue = { + description: null, + projectId: null, + projectWorkspaceId: null, + assigneeAgentId: null, + priority: "medium", + labelIds: [], + billingCode: null, + executionWorkspaceSettings: null, + assigneeAdapterOverrides: null, + }; + issueSvc.list.mockResolvedValue([ + { + ...baseIssue, + id: "issue-1", + identifier: "PAP-1", + title: "Alpha task", + status: "done", + parentId: null, + createdAt: new Date("2026-01-01T00:00:00.000Z"), + updatedAt: new Date("2026-03-01T00:00:00.000Z"), + startedAt: new Date("2026-01-02T00:00:00.000Z"), + completedAt: new Date("2026-02-01T00:00:00.000Z"), + }, + { + ...baseIssue, + id: "issue-2", + identifier: "PAP-2", + title: "Beta task", + status: "todo", + parentId: "issue-1", + createdAt: new Date("2026-01-05T00:00:00.000Z"), + updatedAt: new Date("2026-01-06T00:00:00.000Z"), + }, + { + ...baseIssue, + id: "issue-3", + identifier: "PAP-3", + title: "Gamma task", + status: "todo", + parentId: "issue-2", + createdAt: new Date("2026-01-07T00:00:00.000Z"), + updatedAt: new Date("2026-01-08T00:00:00.000Z"), + }, + { + ...baseIssue, + id: "issue-4", + identifier: "PAP-4", + title: "Delta task", + status: "todo", + parentId: "issue-outside", + }, + ]); + } + + it("carries task timestamps and parent links through export and import", async () => { + const { db } = fakeImportDb(); + const portability = companyPortabilityService(db); + mockTaskHierarchyExportSources(); + + const exported = await portability.exportBundle("company-1", { + include: { company: true, agents: false, projects: false, issues: true }, + }); + + const extension = asTextFile(exported.files[".paperclip.yaml"]); + expect(extension).toContain("schemaVersion: 7"); + expect(extension).toContain('parent: "pap-1"'); + expect(extension).toContain('createdAt: "2026-01-01T00:00:00.000Z"'); + expect(exported.warnings).toContain( + "1 parent relation references a task outside this export and was not included.", + ); + + const alphaEntry = exported.manifest.issues.find((issue) => issue.slug === "pap-1"); + const betaEntry = exported.manifest.issues.find((issue) => issue.slug === "pap-2"); + const gammaEntry = exported.manifest.issues.find((issue) => issue.slug === "pap-3"); + const deltaEntry = exported.manifest.issues.find((issue) => issue.slug === "pap-4"); + expect(alphaEntry).toEqual(expect.objectContaining({ + parentSlug: null, + createdAt: "2026-01-01T00:00:00.000Z", + updatedAt: "2026-03-01T00:00:00.000Z", + startedAt: "2026-01-02T00:00:00.000Z", + completedAt: "2026-02-01T00:00:00.000Z", + cancelledAt: null, + })); + expect(betaEntry?.parentSlug).toBe("pap-1"); + expect(gammaEntry?.parentSlug).toBe("pap-2"); + // The unexported parent edge is dropped, not carried by raw id. + expect(deltaEntry?.parentSlug).toBeNull(); + expect(extension).not.toContain("issue-outside"); + + companySvc.create.mockResolvedValue({ id: "company-imported", name: "Imported" }); + accessSvc.ensureMembership.mockResolvedValue(undefined); + agentSvc.list.mockResolvedValue([]); + + await portability.importBundle({ + source: { type: "inline", rootPath: exported.rootPath, files: exported.files }, + include: { company: true, agents: false, projects: false, issues: true }, + target: { mode: "new_company", newCompanyName: "Imported" }, + agents: "all", + collisionStrategy: "rename", + }, "user-1"); + + const importedRows = issueSvc.importIssues.mock.calls[0]![1] as Array<{ + id: string; + title: string; + parentId: string | null; + createdAt: Date | null; + updatedAt: Date | null; + startedAt: Date | null; + completedAt: Date | null; + cancelledAt: Date | null; + }>; + const alphaRow = importedRows.find((row) => row.title === "Alpha task")!; + const betaRow = importedRows.find((row) => row.title === "Beta task")!; + const gammaRow = importedRows.find((row) => row.title === "Gamma task")!; + const deltaRow = importedRows.find((row) => row.title === "Delta task")!; + + // Timestamps survive the round trip as concrete dates. + expect(alphaRow.createdAt).toEqual(new Date("2026-01-01T00:00:00.000Z")); + expect(alphaRow.updatedAt).toEqual(new Date("2026-03-01T00:00:00.000Z")); + expect(alphaRow.startedAt).toEqual(new Date("2026-01-02T00:00:00.000Z")); + expect(alphaRow.completedAt).toEqual(new Date("2026-02-01T00:00:00.000Z")); + expect(alphaRow.cancelledAt).toBeNull(); + expect(betaRow.createdAt).toEqual(new Date("2026-01-05T00:00:00.000Z")); + + // The parent chain of three is intact against the pre-generated ids. + expect(alphaRow.parentId).toBeNull(); + expect(betaRow.parentId).toBe(alphaRow.id); + expect(gammaRow.parentId).toBe(betaRow.id); + expect(deltaRow.parentId).toBeNull(); + + // Rows arrive parents-first so chunked inserts satisfy the parent FK. + expect(importedRows.indexOf(alphaRow)).toBeLessThan(importedRows.indexOf(betaRow)); + expect(importedRows.indexOf(betaRow)).toBeLessThan(importedRows.indexOf(gammaRow)); + }); + + it("drops self-referencing and cyclic parent links from a hand-built bundle with warnings", async () => { + const { db } = fakeImportDb(); + const portability = companyPortabilityService(db); + + const taskFile = (name: string) => ["---", `name: "${name}"`, "kind: task", "---", "", `${name} body.`, ""].join("\n"); + const files = { + "COMPANY.md": ["---", 'schema: "agentcompanies/v1"', 'name: "Tampered Import"', "---", ""].join("\n"), + "tasks/task-a/TASK.md": taskFile("Task A"), + "tasks/task-b/TASK.md": taskFile("Task B"), + "tasks/task-c/TASK.md": taskFile("Task C"), + ".paperclip.yaml": [ + 'schema: "paperclip/v1"', + "schemaVersion: 7", + "tasks:", + " task-a:", + ' status: "todo"', + ' parent: "task-b"', + " task-b:", + ' status: "todo"', + ' parent: "task-a"', + " task-c:", + ' status: "todo"', + ' parent: "task-c"', + ' createdAt: "not-a-timestamp"', + "", + ].join("\n"), + }; + + companySvc.create.mockResolvedValue({ id: "company-imported", name: "Tampered Import" }); + accessSvc.ensureMembership.mockResolvedValue(undefined); + agentSvc.list.mockResolvedValue([]); + + const result = await portability.importBundle({ + source: { type: "inline", rootPath: "tampered-package", files }, + include: { company: true, agents: false, projects: false, issues: true }, + target: { mode: "new_company", newCompanyName: "Tampered Import" }, + agents: "all", + collisionStrategy: "rename", + }, "user-1"); + + const importedRows = issueSvc.importIssues.mock.calls[0]![1] as Array<{ + title: string; + parentId: string | null; + createdAt: Date | null; + }>; + const rowA = importedRows.find((row) => row.title === "Task A")!; + const rowB = importedRows.find((row) => row.title === "Task B")!; + const rowC = importedRows.find((row) => row.title === "Task C")!; + + // One direction of the two-task cycle survives; the closing edge drops. + expect([rowA.parentId, rowB.parentId].filter((parentId) => parentId !== null)).toHaveLength(1); + expect(rowC.parentId).toBeNull(); + expect(rowC.createdAt).toBeNull(); + expect(result.warnings.filter((warning) => warning.includes("would create a parent cycle"))).toHaveLength(2); + expect(result.warnings).toContain( + "Task task-c parent task-c was skipped because it would create a parent cycle.", + ); + expect(result.warnings).toContain( + "Task task-c createdAt was ignored because it is not a valid timestamp.", + ); + }); + const attachmentBytesByObjectKey: Record = { "issues/issue-1/notes.bin": "png-bytes", "issues/issue-1/screenshot.png": "png-bytes", @@ -4625,7 +4826,7 @@ describe("company portability", () => { it("imports unstamped v5 packages with an info warning about task data they predate", async () => { const portability = companyPortabilityService({} as any); const v5Warning = - "This package declares schemaVersion 5 and predates label, blocker, document, work product, monitor, attachment, and embedded image transfer; that task data imports only if the bundle carries it."; + "This package declares schemaVersion 5 and predates label, blocker, document, work product, monitor, attachment, embedded image, task timestamp, and parent link transfer; that task data imports only if the bundle carries it."; companySvc.create.mockResolvedValue({ id: "company-imported", name: "Legacy Import" }); accessSvc.ensureMembership.mockResolvedValue(undefined); @@ -4674,7 +4875,7 @@ describe("company portability", () => { const portability = companyPortabilityService({} as any); await expect(portability.importBundle({ - source: { type: "inline", rootPath: "future-package", files: legacyPackageFiles(["schemaVersion: 7"]) }, + source: { type: "inline", rootPath: "future-package", files: legacyPackageFiles(["schemaVersion: 8"]) }, include: { company: true, agents: false, projects: false, issues: true }, target: { mode: "new_company", newCompanyName: "Future Import" }, agents: "all", diff --git a/server/src/__tests__/productivity-review-service.test.ts b/server/src/__tests__/productivity-review-service.test.ts index 12b3bdd18c3..a45e4502ed7 100644 --- a/server/src/__tests__/productivity-review-service.test.ts +++ b/server/src/__tests__/productivity-review-service.test.ts @@ -505,6 +505,28 @@ describeEmbeddedPostgres("productivity review service", () => { expect(hold.held).toBe(false); }); + it("skips a long-active candidate while its assignee is paused and reviews it once unpaused", async () => { + const now = new Date("2026-04-28T12:00:00.000Z"); + const seeded = await seedAssignedIssue({ + status: "in_progress", + startedAt: new Date(now.getTime() - 7 * 60 * 60 * 1000), + }); + await db.update(agents).set({ status: "paused" }).where(eq(agents.id, seeded.coderId)); + const service = productivityReviewService(db); + + const pausedResult = await service.reconcileProductivityReviews({ now, companyId: seeded.companyId }); + + expect(pausedResult.created).toBe(0); + expect(pausedResult.skipped).toBe(1); + expect(await listProductivityReviews(seeded.companyId)).toHaveLength(0); + + await db.update(agents).set({ status: "idle" }).where(eq(agents.id, seeded.coderId)); + const unpausedResult = await service.reconcileProductivityReviews({ now, companyId: seeded.companyId }); + + expect(unpausedResult.created).toBe(1); + expect(await listProductivityReviews(seeded.companyId)).toHaveLength(1); + }); + it("creates a high-churn review even when every sampled run has a progress comment", async () => { const now = new Date("2026-04-28T12:00:00.000Z"); const seeded = await seedAssignedIssue(); diff --git a/server/src/__tests__/summary-slots.test.ts b/server/src/__tests__/summary-slots.test.ts index 2bbb1fd5178..ac9fba3bff5 100644 --- a/server/src/__tests__/summary-slots.test.ts +++ b/server/src/__tests__/summary-slots.test.ts @@ -8,6 +8,7 @@ import { createDb, documentRevisions, documents, + executionWorkspaces, heartbeatRuns, issues, projectWorkspaces, @@ -50,6 +51,7 @@ describeEmbeddedPostgres("summary slot service", () => { await db.delete(documents); await db.delete(issues); await db.delete(heartbeatRuns); + await db.delete(executionWorkspaces); await db.delete(projectWorkspaces); await db.delete(projects); await db.delete(activityLog); @@ -78,6 +80,35 @@ describeEmbeddedPostgres("summary slot service", () => { return projectId; } + async function seedProjectWorkspace(companyId: string, projectId: string) { + const projectWorkspaceId = randomUUID(); + await db.insert(projectWorkspaces).values({ + id: projectWorkspaceId, + companyId, + projectId, + name: "Primary workspace", + }); + return projectWorkspaceId; + } + + async function seedExecutionWorkspace( + companyId: string, + projectId: string, + projectWorkspaceId: string | null = null, + ) { + const executionWorkspaceId = randomUUID(); + await db.insert(executionWorkspaces).values({ + id: executionWorkspaceId, + companyId, + projectId, + projectWorkspaceId, + mode: "isolated_workspace", + strategyType: "git_worktree", + name: `Execution workspace ${executionWorkspaceId}`, + }); + return executionWorkspaceId; + } + async function seedSummarizer(companyId: string, ready = true) { const agentId = randomUUID(); await db.insert(agents).values({ @@ -117,6 +148,15 @@ describeEmbeddedPostgres("summary slot service", () => { return { companyId, scopeKind: "project", slotKey: "header", scopeId: projectId }; } + function executionWorkspaceSelector(companyId: string, executionWorkspaceId: string) { + return { + companyId, + scopeKind: "execution_workspace", + slotKey: "header", + scopeId: executionWorkspaceId, + }; + } + describe("reads and target visibility", () => { it("returns an empty slot state before any generation", async () => { const companyId = await seedCompany(); @@ -144,6 +184,18 @@ describeEmbeddedPostgres("summary slot service", () => { }); }); + it("rejects an execution workspace owned by another company", async () => { + const companyId = await seedCompany(); + const otherCompanyId = await seedCompany(); + const otherProjectId = await seedProject(otherCompanyId); + const foreignExecutionWorkspaceId = await seedExecutionWorkspace(otherCompanyId, otherProjectId); + const svc = summarySlotService(db); + + await expect( + svc.getSlot(executionWorkspaceSelector(companyId, foreignExecutionWorkspaceId)), + ).rejects.toMatchObject({ status: 404 }); + }); + it("rejects a workspaces_overview selector that carries a scopeId", async () => { const companyId = await seedCompany(); const svc = summarySlotService(db); @@ -268,6 +320,65 @@ describeEmbeddedPostgres("summary slot service", () => { expect(issueRow.description).not.toContain("Other project issue"); }); + it("keeps summaries and snapshots isolated between execution workspaces", async () => { + const companyId = await seedCompany(); + const projectId = await seedProject(companyId); + const projectWorkspaceId = await seedProjectWorkspace(companyId, projectId); + const firstExecutionWorkspaceId = await seedExecutionWorkspace(companyId, projectId, projectWorkspaceId); + const secondExecutionWorkspaceId = await seedExecutionWorkspace(companyId, projectId, projectWorkspaceId); + await seedSummarizer(companyId); + const svc = summarySlotService(db); + + await db.insert(issues).values([ + { + companyId, + projectId, + projectWorkspaceId, + executionWorkspaceId: firstExecutionWorkspaceId, + identifier: `${issuePrefix(companyId)}-201`, + issueNumber: 201, + title: "First workspace task", + status: "in_progress", + priority: "medium", + }, + { + companyId, + projectId, + projectWorkspaceId, + executionWorkspaceId: secondExecutionWorkspaceId, + identifier: `${issuePrefix(companyId)}-202`, + issueNumber: 202, + title: "Second workspace task", + status: "blocked", + priority: "high", + }, + ]); + + const firstSelector = executionWorkspaceSelector(companyId, firstExecutionWorkspaceId); + const secondSelector = executionWorkspaceSelector(companyId, secondExecutionWorkspaceId); + const generated = await svc.generate(firstSelector, { userId: "board-user" }); + + expect(generated.slot).toMatchObject({ + scopeKind: "execution_workspace", + scopeId: firstExecutionWorkspaceId, + status: "generating", + }); + const generationIssue = await db + .select() + .from(issues) + .where(eq(issues.id, generated.generatingIssue.id)) + .then((rows) => rows[0]!); + expect(generationIssue.description).toContain("First workspace task"); + expect(generationIssue.description).not.toContain("Second workspace task"); + expect(generationIssue.description).toContain('"scopeKind": "execution_workspace"'); + expect(generationIssue.description).toContain(`"scopeId": "${firstExecutionWorkspaceId}"`); + await expect(svc.getSlot(secondSelector)).resolves.toEqual({ + slot: null, + document: null, + generatingIssue: null, + }); + }); + it("dedupes concurrent generate clicks without creating an orphan task", async () => { const companyId = await seedCompany(); const projectId = await seedProject(companyId); diff --git a/server/src/built-ins/agents/summarizer/AGENTS.md b/server/src/built-ins/agents/summarizer/AGENTS.md index abc7f4b696e..60c9a3ea7fa 100644 --- a/server/src/built-ins/agents/summarizer/AGENTS.md +++ b/server/src/built-ins/agents/summarizer/AGENTS.md @@ -2,11 +2,11 @@ You are Summarizer, a built-in reporting agent at Paperclip. When you wake up, follow the Paperclip heartbeat procedure. Work only on issues assigned to you. Always leave a task comment before exiting a heartbeat. -Your job is to turn the current state of a Paperclip scope — a project, the workspaces overview, or a single project workspace — into a short, honest, human-readable Markdown summary and write it back to that scope's summary slot as a new revision. When an issue asks you to generate or refresh a summary, use the `summarize-status` skill as your operating procedure and start with its API quick reference instead of discovering routes. +Your job is to turn the current state of a Paperclip scope — a project, the workspaces overview, a project workspace, or a specific execution workspace — into a short, honest, human-readable Markdown summary and write it back to that scope's summary slot as a new revision. When an issue asks you to generate or refresh a summary, use the `summarize-status` skill as your operating procedure and start with its API quick reference instead of discovering routes. ## Core responsibilities -- Read the scope named by the generation issue (`scopeKind` = `project` | `workspaces_overview` | `project_workspace`, plus `scopeId` and `slotKey`). +- Read the scope named by the generation issue (`scopeKind` = `project` | `workspaces_overview` | `project_workspace` | `execution_workspace`, plus `scopeId` and `slotKey`). - Read the summary slot's most recent revision first, so you lead with what's new instead of repeating a headline the reader already saw. - Triage, don't enumerate: from everything in the scope, work out the 1–3 specific, concrete actions the reader should take right now to unblock the work, and leave everything else off the page. Read whatever issues, comments, or blocker chains you need to genuinely understand where things are. - Open every summary with those 1–3 actionable items — each saying what to do and why it's the thing holding up progress, with an inline link. If genuinely nothing needs the reader, say so plainly in one line and name the next thing worth watching. diff --git a/server/src/built-ins/agents/summarizer/routines/refresh-stale-summaries.md b/server/src/built-ins/agents/summarizer/routines/refresh-stale-summaries.md index 4c051a831ba..8920f0ae037 100644 --- a/server/src/built-ins/agents/summarizer/routines/refresh-stale-summaries.md +++ b/server/src/built-ins/agents/summarizer/routines/refresh-stale-summaries.md @@ -32,6 +32,7 @@ variables: - project - workspaces_overview - project_workspace + - execution_workspace triggers: - kind: schedule label: Daily stale-summary refresh diff --git a/server/src/http/body-limits.ts b/server/src/http/body-limits.ts index 29c92b2baf9..6601dadb772 100644 --- a/server/src/http/body-limits.ts +++ b/server/src/http/body-limits.ts @@ -6,4 +6,18 @@ export const PORTABLE_JSON_BODY_LIMIT_BYTES = 64 * 1024 * 1024; // application/zip) instead of an inflated inline JSON body. The compressed zip // is roughly a third of the inline size, but the limit is kept generous so a // large company package uploads in one request rather than truncating in transit. -export const PORTABLE_ZIP_UPLOAD_LIMIT_BYTES = 128 * 1024 * 1024; +// The whole upload is buffered in memory and unzipped in one pass, so this cap +// bounds peak per-import memory (roughly the compressed size plus the inflated +// package). Operators who need more (or less) headroom can override it with +// PAPERCLIP_IMPORT_ZIP_MAX_BYTES; the import route scales its decompression-bomb +// guards from whatever value is in effect. +// The override is clamped to [1 byte, 64 GiB]: a fractional value would floor +// to a zero-byte limit that rejects every upload, and an astronomically large +// one would overflow the derived 4x decompression guard. 64 GiB is far beyond +// what the in-memory import pipeline can serve anyway. +const MAX_ZIP_UPLOAD_LIMIT_OVERRIDE_BYTES = 64 * 1024 * 1024 * 1024; +const zipUploadLimitOverride = Math.floor(Number(process.env.PAPERCLIP_IMPORT_ZIP_MAX_BYTES)); +export const PORTABLE_ZIP_UPLOAD_LIMIT_BYTES = + Number.isFinite(zipUploadLimitOverride) && zipUploadLimitOverride >= 1 + ? Math.min(zipUploadLimitOverride, MAX_ZIP_UPLOAD_LIMIT_OVERRIDE_BYTES) + : 1024 * 1024 * 1024; diff --git a/server/src/routes/companies.ts b/server/src/routes/companies.ts index 19686def29b..186e05db563 100644 --- a/server/src/routes/companies.ts +++ b/server/src/routes/companies.ts @@ -6,7 +6,11 @@ import { z } from "zod"; import type { Db } from "@paperclipai/db"; import { agents as agentsTable } from "@paperclipai/db"; import type { CompanyPortabilityImportResult } from "@paperclipai/shared"; -import { readZipArchive } from "@paperclipai/shared/portability-zip"; +import { + MAX_ZIP_ENTRY_DECOMPRESSED_BYTES, + MAX_ZIP_TOTAL_DECOMPRESSED_BYTES, + readZipArchive, +} from "@paperclipai/shared/portability-zip"; import { DEFAULT_FEEDBACK_DATA_SHARING_TERMS_VERSION, companyArtifactsQuerySchema, @@ -131,7 +135,9 @@ async function resolveImportPayload(req: Request, res: Response): Promise>; try { - archive = await readZipArchive(zipBytes); + // Scale the bomb guards from the configured upload cap so a legitimately + // compressible package under the cap never trips them (~4x covers dense + // text; the per-entry ceiling is bounded by V8's string limit regardless). + archive = await readZipArchive(zipBytes, { + maxEntryDecompressedBytes: MAX_ZIP_ENTRY_DECOMPRESSED_BYTES, + maxTotalDecompressedBytes: Math.max( + MAX_ZIP_TOTAL_DECOMPRESSED_BYTES, + PORTABLE_ZIP_UPLOAD_LIMIT_BYTES * 4, + ), + }); } catch (error) { throw badRequest(`Import package could not be read: ${errorMessage(error)}`); } diff --git a/server/src/services/built-in-agents.ts b/server/src/services/built-in-agents.ts index b8a25460c72..40c46efe72c 100644 --- a/server/src/services/built-in-agents.ts +++ b/server/src/services/built-in-agents.ts @@ -181,7 +181,7 @@ const FALLBACK_REFLECTION_COACH_SKILL = [ const FALLBACK_SUMMARIZER_INSTRUCTIONS = [ "You are Summarizer, a built-in reporting agent at Paperclip.", "", - "Turn the current state of a Paperclip scope (project, workspaces overview, or a single project workspace) into a short, honest, human-readable Markdown summary and write it back to that scope's summary slot as a new revision. Use the `summarize-status` skill as your operating procedure.", + "Turn the current state of a Paperclip scope (project, workspaces overview, project workspace, or execution workspace) into a short, honest, human-readable Markdown summary and write it back to that scope's summary slot as a new revision. Use the `summarize-status` skill as your operating procedure.", "", "Read-and-report only: never change issues, workspaces, or code. Cite issue identifiers, never fabricate status, keep every read company-scoped, and run on the low-cost model profile lane by default.", "", @@ -401,7 +401,7 @@ const DEFINITIONS = validateBuiltInAgentDefinitions([ displayName: "Summarizer", featureKeys: ["summarizer"], shortPurpose: - "Writes short, human-readable Markdown status summaries into project, workspaces-overview, and project-workspace summary slots on demand.", + "Writes short, human-readable Markdown status summaries into project, workspaces-overview, project-workspace, and execution-workspace summary slots on demand.", defaultInstructions: SUMMARIZER_INSTRUCTIONS, defaultRole: "general", defaultTitle: "Summarizer", @@ -419,7 +419,7 @@ const DEFINITIONS = validateBuiltInAgentDefinitions([ }, defaultBudgetMonthlyCents: 0, bundle: { - stockVersion: "2026-07-15", + stockVersion: "2026-08-02", instructions: { entryFile: "AGENTS.md", files: { @@ -452,7 +452,7 @@ const DEFINITIONS = validateBuiltInAgentDefinitions([ type: "select", defaultValue: "all", required: true, - options: ["all", "project", "workspaces_overview", "project_workspace"], + options: ["all", "project", "workspaces_overview", "project_workspace", "execution_workspace"], }, ], triggers: [ diff --git a/server/src/services/company-portability.ts b/server/src/services/company-portability.ts index cdf4cb56303..c86c2bbbefd 100644 --- a/server/src/services/company-portability.ts +++ b/server/src/services/company-portability.ts @@ -161,8 +161,10 @@ const DEFAULT_INCLUDE: CompanyPortabilityInclude = { const DEFAULT_COLLISION_STRATEGY: CompanyPortabilityCollisionStrategy = "rename"; // The bundle shape this build reads and writes. Bundles began declaring // their schemaVersion in the .paperclip.yaml extension at 6; undeclared -// bundles are read as 5, the last unstamped shape. -const BUNDLE_SCHEMA_VERSION = 6; +// bundles are read as 5, the last unstamped shape. 7 adds preserved task +// timestamps and parent links; 5/6 bundles still import, with those fields +// falling back to import-time defaults. +const BUNDLE_SCHEMA_VERSION = 7; const UNSTAMPED_BUNDLE_SCHEMA_VERSION = 5; const DEFAULT_IMPORTED_LABEL_COLOR = "#6366f1"; // Blob entries are content-addressed by sha256; the store itself is @@ -899,6 +901,38 @@ function readPortableIssueLabelNames(value: unknown): string[] { return names; } +/** + * Read a preserved issue timestamp from the extension, validated the same way + * comment createdAt is (Date.parse). Invalid values are ignored with a + * warning — never a hard failure — so a hand-edited bundle still imports. + */ +function readPortableIssueTimestamp( + value: unknown, + warnings: string[], + sourceLabel: string, + fieldLabel: string, +): string | null { + if (value === undefined || value === null) return null; + const raw = asString(value); + if (raw && !Number.isNaN(Date.parse(raw))) return raw; + warnings.push(`${sourceLabel} ${fieldLabel} was ignored because it is not a valid timestamp.`); + return null; +} + +/** Render a stored timestamp as a portable ISO string, or omit it when unset/invalid. */ +function toPortableTimestamp(value: Date | string | null | undefined): string | undefined { + if (value == null) return undefined; + const date = value instanceof Date ? value : new Date(value); + return Number.isNaN(date.getTime()) ? undefined : date.toISOString(); +} + +/** Convert a manifest timestamp (already validated at parse time) to a Date, or null. */ +function portableManifestDate(value: string | null | undefined): Date | null { + if (!value) return null; + const date = new Date(value); + return Number.isNaN(date.getTime()) ? null : date; +} + function readPortableIssueBlockedBy(value: unknown): string[] { if (!Array.isArray(value)) return []; const slugs: string[] = []; @@ -3353,6 +3387,12 @@ function buildManifestFromPackageFiles( workProducts: normalizePortableIssueWorkProducts(extension.workProducts), monitor: normalizePortableIssueMonitor(extension.monitor), attachments: normalizePortableIssueAttachments(extension.attachments, warnings, `Task ${slug}`), + parentSlug: asString(extension.parent), + createdAt: readPortableIssueTimestamp(extension.createdAt, warnings, `Task ${slug}`, "createdAt"), + updatedAt: readPortableIssueTimestamp(extension.updatedAt, warnings, `Task ${slug}`, "updatedAt"), + startedAt: readPortableIssueTimestamp(extension.startedAt, warnings, `Task ${slug}`, "startedAt"), + completedAt: readPortableIssueTimestamp(extension.completedAt, warnings, `Task ${slug}`, "completedAt"), + cancelledAt: readPortableIssueTimestamp(extension.cancelledAt, warnings, `Task ${slug}`, "cancelledAt"), metadata: isPlainRecord(extension.metadata) ? extension.metadata : null, }); if (frontmatter.kind && frontmatter.kind !== "task") { @@ -3361,7 +3401,10 @@ function buildManifestFromPackageFiles( } if (bundleSchemaVersion < BUNDLE_SCHEMA_VERSION && manifest.issues.length > 0) { - warnings.push(`This package declares schemaVersion ${bundleSchemaVersion} and predates label, blocker, document, work product, monitor, attachment, and embedded image transfer; that task data imports only if the bundle carries it.`); + const predated = bundleSchemaVersion < 6 + ? "label, blocker, document, work product, monitor, attachment, embedded image, task timestamp, and parent link transfer" + : "task timestamp and parent link transfer"; + warnings.push(`This package declares schemaVersion ${bundleSchemaVersion} and predates ${predated}; that task data imports only if the bundle carries it.`); } manifest.envInputs = dedupeEnvInputs(manifest.envInputs); @@ -4216,6 +4259,7 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { } let unexportedBlockerEdgeCount = 0; + let unexportedParentEdgeCount = 0; let unportableWorkProductRefCount = 0; const exportedBlobs = new Map(); for (const issue of selectedIssueRows) { @@ -4256,6 +4300,13 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { unexportedBlockerEdgeCount += relationSummaries.blocks .filter((blocked) => !taskSlugByIssueId.has(blocked.id)) .length; + // Parent links travel by task slug like blockers; a parent outside the + // export selection drops the edge and is counted for one warning. + let parentTaskSlug: string | null = null; + if (issue.parentId) { + parentTaskSlug = taskSlugByIssueId.get(issue.parentId) ?? null; + if (!parentTaskSlug) unexportedParentEdgeCount += 1; + } const issueDocumentRows = await documentsSvc.listIssueDocuments(issue.id, { includeSystem: true }); const documentEntries = issueDocumentRows.map((document) => { const documentPath = `tasks/${taskSlug}/documents/${document.key}.md`; @@ -4339,6 +4390,12 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { identifier: issue.identifier, status: issue.status, priority: issue.priority, + parent: parentTaskSlug ?? undefined, + createdAt: toPortableTimestamp(issue.createdAt), + updatedAt: toPortableTimestamp(issue.updatedAt), + startedAt: toPortableTimestamp(issue.startedAt), + completedAt: toPortableTimestamp(issue.completedAt), + cancelledAt: toPortableTimestamp(issue.cancelledAt), // Labels travel by name (their natural key); the bundle-level labels // section carries the matching color definitions. labels: (issue.labelIds ?? []) @@ -4380,6 +4437,9 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { if (unexportedBlockerEdgeCount > 0) { warnings.push(`${unexportedBlockerEdgeCount} blocker relation${unexportedBlockerEdgeCount === 1 ? " references a task" : "s reference tasks"} outside this export and ${unexportedBlockerEdgeCount === 1 ? "was" : "were"} not included.`); } + if (unexportedParentEdgeCount > 0) { + warnings.push(`${unexportedParentEdgeCount} parent relation${unexportedParentEdgeCount === 1 ? " references a task" : "s reference tasks"} outside this export and ${unexportedParentEdgeCount === 1 ? "was" : "were"} not included.`); + } if (unportableWorkProductRefCount > 0) { warnings.push(`${unportableWorkProductRefCount} work product${unportableWorkProductRefCount === 1 ? " references" : "s reference"} execution workspaces or runs that are not portable; those references were omitted from the export.`); } @@ -5721,6 +5781,7 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { const importedIssueIdBySlug = new Map(); const blockedByBySlug = new Map(); + const parentSlugBySlug = new Map(); let unarmedMonitorCount = 0; let attachmentsSkippedNoStorage = 0; const attachmentMaxBytes = normalizeIssueAttachmentMaxBytes(targetCompany.attachmentMaxBytes ?? null); @@ -5924,6 +5985,9 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { if ((manifestIssue.blockedBy ?? []).length > 0) { blockedByBySlug.set(manifestIssue.slug, manifestIssue.blockedBy ?? []); } + if (manifestIssue.parentSlug) { + parentSlugBySlug.set(manifestIssue.slug, manifestIssue.parentSlug); + } for (const documentEntry of manifestIssue.documents ?? []) { const documentBody = readPortableTextFile(plan.source.files, documentEntry.path); if (documentBody === null) { @@ -6050,9 +6114,70 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { labelIds: resolvedLabelIds, monitorNotes, monitorScheduledBy, + parentId: null, + createdAt: portableManifestDate(manifestIssue.createdAt), + updatedAt: portableManifestDate(manifestIssue.updatedAt), + startedAt: portableManifestDate(manifestIssue.startedAt), + completedAt: portableManifestDate(manifestIssue.completedAt), + cancelledAt: portableManifestDate(manifestIssue.cancelledAt), }); } + // Parent links resolve against the pre-generated ids before the flush + // so each issue row carries its parentId on insert. Edges whose parent + // was not imported, self-references, and cycles (possible only in a + // hand-edited bundle) drop with a warning, mirroring blocker handling. + if (parentSlugBySlug.size > 0) { + const rowBySlug = new Map(issueRows.map((row) => [row.ref, row] as const)); + const acceptedParentById = new Map(); + const wouldCreateParentCycle = (childId: string, parentId: string) => { + let current: string | undefined = parentId; + const visited = new Set(); + while (current) { + if (current === childId) return true; + if (visited.has(current)) return false; + visited.add(current); + current = acceptedParentById.get(current); + } + return false; + }; + for (const [slug, parentSlug] of parentSlugBySlug) { + const row = rowBySlug.get(slug); + if (!row) continue; + const parentId = importedIssueIdBySlug.get(parentSlug); + if (!parentId) { + warnings.push(`Task ${slug} parent ${parentSlug} was skipped because that task was not imported.`); + continue; + } + if (parentId === row.id || wouldCreateParentCycle(row.id, parentId)) { + warnings.push(`Task ${slug} parent ${parentSlug} was skipped because it would create a parent cycle.`); + continue; + } + acceptedParentById.set(row.id, parentId); + row.parentId = parentId; + } + // The parent foreign key is checked per insert statement, so order + // rows parents-first: a child must never land in an earlier chunk + // than its parent. + if (acceptedParentById.size > 0) { + const rowById = new Map(issueRows.map((row) => [row.id, row] as const)); + const orderedRows: typeof issueRows = []; + const emitted = new Set(); + for (const row of issueRows) { + const chain: typeof issueRows = []; + let current: (typeof issueRows)[number] | undefined = row; + while (current && !emitted.has(current.id)) { + emitted.add(current.id); + chain.push(current); + current = current.parentId ? rowById.get(current.parentId) : undefined; + } + for (const entry of chain.reverse()) orderedRows.push(entry); + } + issueRows.length = 0; + issueRows.push(...orderedRows); + } + } + // Flush the buffered rows in dependency order: issues first (parents of // every other row), then comments (attachments may reference them), then // the remaining children. Each writer inserts in chunked multi-row diff --git a/server/src/services/import-write-types.ts b/server/src/services/import-write-types.ts index 264f80f97ad..f5d57449849 100644 --- a/server/src/services/import-write-types.ts +++ b/server/src/services/import-write-types.ts @@ -35,6 +35,18 @@ export interface ImportIssueRow { /** Imported monitors land un-armed; only notes/provenance are restored. */ monitorNotes: string | null; monitorScheduledBy: string | null; + /** Parent issue id resolved within the same import batch (pre-generated), or absent for roots. */ + parentId?: string | null; + /** + * Preserved source timestamps carried by schemaVersion >= 7 bundles. + * Absent/null createdAt and updatedAt fall back to the insert time; + * absent startedAt stays null — the writer never fabricates it. + */ + createdAt?: Date | null; + updatedAt?: Date | null; + startedAt?: Date | null; + completedAt?: Date | null; + cancelledAt?: Date | null; } /** A resolved comment row with a pre-generated id, ready for batch insert. */ diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index 347168911b7..b87156d913e 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -94,7 +94,7 @@ import { resolveIssueGoalId, resolveNextIssueGoalId } from "./issue-goal-fallbac import { getRunLogStore } from "./run-log-store.js"; import { getDefaultCompanyGoal } from "./goals.js"; import { assertAssignableAgent } from "./agent-assignability.js"; -import { insertRowsInChunks } from "./batch-insert.js"; +import { DEFAULT_INSERT_CHUNK_ROWS, insertRowsInChunks } from "./batch-insert.js"; import type { ImportIssueRow, ImportIssueCommentRow, @@ -7352,9 +7352,19 @@ export function issueService(db: Db) { responsibleUserId: null, requestDepth: clampIssueRequestDepth(undefined), originKind: "manual", - startedAt: row.status === "in_progress" ? new Date() : null, - completedAt: row.status === "done" ? new Date() : null, - cancelledAt: row.status === "cancelled" ? new Date() : null, + // The caller resolves parentId against ids in this same batch, so a + // parent always lands in the same insert (rows arrive parents-first). + parentId: row.parentId ?? null, + // Preserved bundle timestamps win; without them createdAt/updatedAt + // fall back to the insert time (the old defaultNow() behavior). + createdAt: row.createdAt ?? new Date(), + updatedAt: row.updatedAt ?? new Date(), + // Imported in-progress work did not start at import time; fabricating + // startedAt here trips duration-based sweeps (e.g. productivity + // review). Only a bundle-carried startedAt is written. + startedAt: row.startedAt ?? null, + completedAt: row.completedAt ?? (row.status === "done" ? new Date() : null), + cancelledAt: row.cancelledAt ?? (row.status === "cancelled" ? new Date() : null), monitorNotes: row.monitorNotes ?? null, monitorScheduledBy: row.monitorScheduledBy ?? null, }); @@ -7394,10 +7404,28 @@ export function issueService(db: Db) { }; }); await insertRowsInChunks(tx, issueComments, commentRows); - // Mirror addComment's recency bump, once per affected issue. - const issueIds = [...new Set(rows.map((row) => row.issueId))]; - if (issueIds.length > 0) { - await tx.update(issues).set({ updatedAt: new Date() }).where(inArray(issues.id, issueIds)); + // Mirror addComment's recency bump, once per affected issue — but never + // backwards. An issue imported with a preserved updatedAt keeps it + // unless a newer imported comment outdates it; issues without preserved + // timestamps carry an insert-time updatedAt, so GREATEST reproduces the + // old "bump to now" behavior for them. + const bumpAtByIssueId = new Map(); + for (const row of commentRows) { + const existing = bumpAtByIssueId.get(row.issueId); + if (!existing || row.createdAt > existing) bumpAtByIssueId.set(row.issueId, row.createdAt); + } + const bumpEntries = [...bumpAtByIssueId.entries()]; + for (let start = 0; start < bumpEntries.length; start += DEFAULT_INSERT_CHUNK_ROWS) { + const chunk = bumpEntries.slice(start, start + DEFAULT_INSERT_CHUNK_ROWS); + await tx.execute(sql` + update ${issues} + set updated_at = greatest(${issues.updatedAt}, bumps.bump_at) + from (values ${sql.join( + chunk.map(([issueId, bumpAt]) => sql`(${issueId}::uuid, ${bumpAt.toISOString()}::timestamptz)`), + sql`, `, + )}) as bumps(issue_id, bump_at) + where ${issues.id} = bumps.issue_id + `); } }); }, diff --git a/server/src/services/productivity-review.ts b/server/src/services/productivity-review.ts index 199b122421c..909682d6758 100644 --- a/server/src/services/productivity-review.ts +++ b/server/src/services/productivity-review.ts @@ -896,6 +896,11 @@ export function productivityReviewService(db: Db, deps?: { enqueueWakeup?: Enque result.skipped += 1; continue; } + // A paused assignee cannot act on a review, so raising one only creates noise. + if (sourceAgent.status === "paused") { + result.skipped += 1; + continue; + } const evidence = await collectEvidence(candidate, sourceAgent, thresholds, now); if (!evidence) { result.skipped += 1; diff --git a/server/src/services/summary-slots.ts b/server/src/services/summary-slots.ts index bcd6f0c5436..6f07b83872c 100644 --- a/server/src/services/summary-slots.ts +++ b/server/src/services/summary-slots.ts @@ -3,6 +3,7 @@ import type { Db } from "@paperclipai/db"; import { documentRevisions, documents, + executionWorkspaces, issues, projectWorkspaces, projects, @@ -123,7 +124,9 @@ function scopeLabel(scopeKind: SummarySlotScopeKind): string { case "project": return "project"; case "project_workspace": - return "workspace"; + return "project workspace"; + case "execution_workspace": + return "execution workspace"; case "workspaces_overview": return "workspaces overview"; default: @@ -175,6 +178,15 @@ export function summarySlotService(db: Db) { .where(and(eq(projectWorkspaces.id, sel.scopeId), eq(projectWorkspaces.companyId, sel.companyId))) .then((rows) => rows[0] ?? null); if (!row) throw notFound("Summary target not found"); + return; + } + if (sel.scopeKind === "execution_workspace") { + const row = await db + .select({ id: executionWorkspaces.id }) + .from(executionWorkspaces) + .where(and(eq(executionWorkspaces.id, sel.scopeId), eq(executionWorkspaces.companyId, sel.companyId))) + .then((rows) => rows[0] ?? null); + if (!row) throw notFound("Summary target not found"); } } @@ -299,9 +311,10 @@ export function summarySlotService(db: Db) { async function resolveGenerationTargetProject(sel: ResolvedSelector): Promise<{ projectId: string | null; projectWorkspaceId: string | null; + executionWorkspaceId: string | null; }> { if (sel.scopeKind === "project") { - return { projectId: sel.scopeId, projectWorkspaceId: null }; + return { projectId: sel.scopeId, projectWorkspaceId: null, executionWorkspaceId: null }; } if (sel.scopeKind === "project_workspace" && sel.scopeId) { const row = await db @@ -309,14 +322,34 @@ export function summarySlotService(db: Db) { .from(projectWorkspaces) .where(and(eq(projectWorkspaces.id, sel.scopeId), eq(projectWorkspaces.companyId, sel.companyId))) .then((rows) => rows[0] ?? null); - return { projectId: row?.projectId ?? null, projectWorkspaceId: sel.scopeId }; + return { + projectId: row?.projectId ?? null, + projectWorkspaceId: sel.scopeId, + executionWorkspaceId: null, + }; + } + if (sel.scopeKind === "execution_workspace" && sel.scopeId) { + const row = await db + .select({ + projectId: executionWorkspaces.projectId, + projectWorkspaceId: executionWorkspaces.projectWorkspaceId, + }) + .from(executionWorkspaces) + .where(and(eq(executionWorkspaces.id, sel.scopeId), eq(executionWorkspaces.companyId, sel.companyId))) + .then((rows) => rows[0] ?? null); + return { + projectId: row?.projectId ?? null, + projectWorkspaceId: row?.projectWorkspaceId ?? null, + executionWorkspaceId: sel.scopeId, + }; } - return { projectId: null, projectWorkspaceId: null }; + return { projectId: null, projectWorkspaceId: null, executionWorkspaceId: null }; } function scopeIssueConditions(sel: ResolvedSelector) { if (sel.scopeKind === "project") return [eq(issues.projectId, sel.scopeId!)]; if (sel.scopeKind === "project_workspace") return [eq(issues.projectWorkspaceId, sel.scopeId!)]; + if (sel.scopeKind === "execution_workspace") return [eq(issues.executionWorkspaceId, sel.scopeId!)]; return []; } @@ -464,7 +497,7 @@ export function summarySlotService(db: Db) { } } - const { projectId, projectWorkspaceId } = await resolveGenerationTargetProject(sel); + const { projectId, projectWorkspaceId, executionWorkspaceId } = await resolveGenerationTargetProject(sel); const scopeSnapshot = await buildScopeSnapshot(sel, existing?.lastGeneratedAt ?? null); const createdAt = new Date(); const generationVersion = existing?.generatingIssueId ?? existing?.updatedAt.toISOString() ?? "initial"; @@ -472,6 +505,7 @@ export function summarySlotService(db: Db) { const created = await issuesSvc.create(sel.companyId, { projectId, projectWorkspaceId, + executionWorkspaceId, title: generationIssueTitle(sel, createdAt), description: generationIssueDescription(sel, scopeSnapshot), status: "todo", diff --git a/tests/release-smoke/docker-auth-onboarding.spec.ts b/tests/release-smoke/docker-auth-onboarding.spec.ts index d4494599e37..f094e9e537f 100644 --- a/tests/release-smoke/docker-auth-onboarding.spec.ts +++ b/tests/release-smoke/docker-auth-onboarding.spec.ts @@ -10,8 +10,11 @@ const ADMIN_PASSWORD = "paperclip-smoke-password"; const COMPANY_NAME = `Release-Smoke-${Date.now()}`; +const MISSION = "Ship a reliable release smoke suite for Paperclip."; const AGENT_NAME = "CEO"; -const TASK_TITLE = "Release smoke task"; +// Seeded by the wizard's launch step (DEFAULT_TASK_TITLE in +// ui/src/components/OnboardingWizard.tsx). +const FIRST_TASK_TITLE = "Hire your first engineer and create a hiring plan"; async function signIn(page: Page) { await page.goto("/"); @@ -38,39 +41,50 @@ async function openOnboarding(page: Page) { } test.describe("Docker authenticated onboarding smoke", () => { - test("logs in, completes onboarding, and triggers the first CEO run", async ({ + test("logs in, completes onboarding, and hires the lead agent", async ({ page, }) => { await signIn(page); await openOnboarding(page); + // Step 1: name the company. await page.locator('input[placeholder="Acme Corp"]').fill(COMPANY_NAME); await page.getByRole("button", { name: "Next" }).click(); + // Step 2: define the mission directly; confirming creates the company. await expect( - page.locator("h3", { hasText: "Create your first agent" }) - ).toBeVisible({ timeout: 10_000 }); - - await expect(page.locator('input[placeholder="CEO"]')).toHaveValue(AGENT_NAME); - await page.getByRole("button", { name: "Next" }).click(); - - await expect( - page.locator("h3", { hasText: "Give it something to do" }) + page.locator("h3", { hasText: "Define your mission" }) ).toBeVisible({ timeout: 10_000 }); + await page.getByRole("button", { name: "I know my mission" }).click(); await page - .locator('input[placeholder="e.g. Research competitor pricing"]') - .fill(TASK_TITLE); + .locator('textarea[placeholder="What is your team trying to achieve?"]') + .fill(MISSION); + await page.getByRole("button", { name: "Confirm mission" }).click(); + + // Step 3: name the team lead. + const leadNameInput = page.locator('input[placeholder="Chief of staff"]'); + await expect(leadNameInput).toBeVisible({ timeout: 20_000 }); + await leadNameInput.fill(AGENT_NAME); await page.getByRole("button", { name: "Next" }).click(); - await expect( - page.locator("h3", { hasText: "Ready to launch" }) - ).toBeVisible({ timeout: 10_000 }); - await expect(page.getByText(COMPANY_NAME)).toBeVisible(); - await expect(page.getByText(AGENT_NAME)).toBeVisible(); - await expect(page.getByText(TASK_TITLE)).toBeVisible(); - - await page.getByRole("button", { name: "Create & Open Task" }).click(); - await expect(page).toHaveURL(/\/issues\//, { timeout: 10_000 }); + // Step 4: keep the default adapter and hire the lead. The adapter + // environment test runs inside the smoke container, where no agent CLIs + // are installed; an unhealthy report is expected and must not block the + // hire. Allow generous time for the env probe + hire + auto-approval. + const heartbeatButton = page.getByRole("button", { + name: "Give it a heartbeat", + }); + await expect(heartbeatButton).toBeVisible({ timeout: 10_000 }); + await expect(heartbeatButton).toBeEnabled({ timeout: 30_000 }); + await heartbeatButton.click(); + + // Step 5: review, then launch. "Get started" provisions the onboarding + // goal/project and navigates to the dashboard only on success. + const getStartedButton = page.getByRole("button", { name: "Get started" }); + await expect(getStartedButton).toBeVisible({ timeout: 60_000 }); + await expect(getStartedButton).toBeEnabled({ timeout: 10_000 }); + await getStartedButton.click(); + await expect(page).toHaveURL(/\/dashboard/, { timeout: 30_000 }); const baseUrl = new URL(page.url()).origin; @@ -95,6 +109,26 @@ test.describe("Docker authenticated onboarding smoke", () => { expect(ceoAgent!.role).toBe("ceo"); expect(ceoAgent!.adapterType).not.toBe("process"); + const goalsRes = await page.request.get( + `${baseUrl}/api/companies/${company!.id}/goals` + ); + expect(goalsRes.ok()).toBe(true); + const goals = (await goalsRes.json()) as Array<{ + id: string; + title: string; + level: string; + status: string; + }>; + expect(goals).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + title: MISSION, + level: "company", + status: "active", + }), + ]) + ); + const issuesRes = await page.request.get( `${baseUrl}/api/companies/${company!.id}/issues` ); @@ -104,9 +138,9 @@ test.describe("Docker authenticated onboarding smoke", () => { title: string; assigneeAgentId: string | null; }>; - const issue = issues.find((entry) => entry.title === TASK_TITLE); - expect(issue).toBeTruthy(); - expect(issue!.assigneeAgentId).toBe(ceoAgent!.id); + const seededIssue = issues.find((entry) => entry.title === FIRST_TASK_TITLE); + expect(seededIssue).toBeTruthy(); + expect(seededIssue!.assigneeAgentId).toBe(ceoAgent!.id); await expect.poll( async () => { diff --git a/ui/src/lib/import-preflight.ts b/ui/src/lib/import-preflight.ts index df8c143df3e..fcc9e42bb7f 100644 --- a/ui/src/lib/import-preflight.ts +++ b/ui/src/lib/import-preflight.ts @@ -2,7 +2,8 @@ import type { CompanyPortabilityFileEntry } from "@paperclipai/shared"; // Inline imports post the whole parsed package as one JSON body, so oversized // packages must be blocked before the request is built. Packages past this -// limit go through the CLI folder import today; a blob-store relay is planned. +// limit go through the zip upload path (a far higher, operator-configurable +// server cap); a blob-store relay is planned for arbitrarily large migrations. export const INLINE_IMPORT_MAX_BYTES = 56 * 1024 * 1024; export function isBlobStoreFilePath(filePath: string): boolean { diff --git a/ui/src/lib/inbox.test.ts b/ui/src/lib/inbox.test.ts index 76deef856be..3fdf2ef02a9 100644 --- a/ui/src/lib/inbox.test.ts +++ b/ui/src/lib/inbox.test.ts @@ -35,6 +35,7 @@ import { loadInboxIssueColumns, loadInboxWorkItemGroupBy, loadCollapsedInboxGroupKeys, + loadCollapsedInboxParentIds, loadLastInboxTab, matchesInboxIssueSearch, normalizeInboxIssueColumns, @@ -45,6 +46,7 @@ import { resolveInboxSelectionIndex, saveInboxFilterPreferences, saveCollapsedInboxGroupKeys, + saveCollapsedInboxParentIds, saveInboxIssueColumns, saveInboxWorkItemGroupBy, saveLastInboxTab, @@ -1560,6 +1562,23 @@ describe("inbox helpers", () => { expect(loadCollapsedInboxGroupKeys("company-1")).toEqual(new Set()); }); + it("persists collapsed inbox parents per company", () => { + saveCollapsedInboxParentIds("company-1", new Set(["parent-1", "parent-2"])); + saveCollapsedInboxParentIds("company-2", new Set(["parent-3"])); + + expect(loadCollapsedInboxParentIds("company-1")).toEqual(new Set(["parent-1", "parent-2"])); + expect(loadCollapsedInboxParentIds("company-2")).toEqual(new Set(["parent-3"])); + + saveCollapsedInboxParentIds("company-1", new Set()); + expect(loadCollapsedInboxParentIds("company-1")).toEqual(new Set()); + }); + + it("returns empty collapsed inbox parents for missing or invalid storage", () => { + expect(loadCollapsedInboxParentIds("company-1")).toEqual(new Set()); + localStorage.setItem("paperclip:inbox:collapsed-parents:company-1", JSON.stringify({ nope: true })); + expect(loadCollapsedInboxParentIds("company-1")).toEqual(new Set()); + }); + it("does not reset workspace grouping before experimental settings have loaded", () => { expect(shouldResetInboxWorkspaceGrouping("workspace", false, false)).toBe(false); }); diff --git a/ui/src/lib/inbox.ts b/ui/src/lib/inbox.ts index be0e9145f77..284a1163a24 100644 --- a/ui/src/lib/inbox.ts +++ b/ui/src/lib/inbox.ts @@ -26,6 +26,7 @@ export const INBOX_NESTING_KEY = "paperclip:inbox:nesting"; export const INBOX_GROUP_BY_KEY = "paperclip:inbox:group-by"; export const INBOX_FILTER_PREFERENCES_KEY_PREFIX = "paperclip:inbox:filters"; export const INBOX_COLLAPSED_GROUPS_KEY_PREFIX = "paperclip:inbox:collapsed-groups"; +export const INBOX_COLLAPSED_PARENTS_KEY_PREFIX = "paperclip:inbox:collapsed-parents"; export type InboxTab = "mine" | "recent" | "unread" | "blocked" | "all"; export type InboxCategoryFilter = | "everything" @@ -187,6 +188,11 @@ function getInboxCollapsedGroupsStorageKey(companyId: string | null | undefined) return `${INBOX_COLLAPSED_GROUPS_KEY_PREFIX}:${companyId}`; } +function getInboxCollapsedParentsStorageKey(companyId: string | null | undefined): string | null { + if (!companyId) return null; + return `${INBOX_COLLAPSED_PARENTS_KEY_PREFIX}:${companyId}`; +} + export function loadInboxFilterPreferences( companyId: string | null | undefined, ): InboxFilterPreferences { @@ -271,6 +277,36 @@ export function saveCollapsedInboxGroupKeys( } } +export function loadCollapsedInboxParentIds( + companyId: string | null | undefined, +): Set { + const storageKey = getInboxCollapsedParentsStorageKey(companyId); + if (!storageKey) return new Set(); + + try { + const raw = localStorage.getItem(storageKey); + if (!raw) return new Set(); + const parsed = JSON.parse(raw); + return new Set(Array.isArray(parsed) ? parsed.filter((entry): entry is string => typeof entry === "string") : []); + } catch { + return new Set(); + } +} + +export function saveCollapsedInboxParentIds( + companyId: string | null | undefined, + parentIds: ReadonlySet, +) { + const storageKey = getInboxCollapsedParentsStorageKey(companyId); + if (!storageKey) return; + + try { + localStorage.setItem(storageKey, JSON.stringify([...parentIds])); + } catch { + // Ignore localStorage failures. + } +} + export function loadDismissedInboxAlerts(): Set { try { const raw = localStorage.getItem(DISMISSED_KEY); diff --git a/ui/src/pages/CompanyImport.test.tsx b/ui/src/pages/CompanyImport.test.tsx index dd2fefcb97b..b27910aa2b6 100644 --- a/ui/src/pages/CompanyImport.test.tsx +++ b/ui/src/pages/CompanyImport.test.tsx @@ -22,6 +22,9 @@ const mockAgentsApi = vi.hoisted(() => ({ list: vi.fn(), resume: vi.fn(), })); +const mockAdaptersApi = vi.hoisted(() => ({ + list: vi.fn(), +})); const mockRoutinesApi = vi.hoisted(() => ({ update: vi.fn(), })); @@ -54,6 +57,10 @@ vi.mock("../api/agents", () => ({ agentsApi: mockAgentsApi, })); +vi.mock("../api/adapters", () => ({ + adaptersApi: mockAdaptersApi, +})); + vi.mock("../api/routines", () => ({ routinesApi: mockRoutinesApi, })); @@ -156,6 +163,44 @@ function buildPreviewResult(): CompanyPortabilityPreviewResult { } as unknown as CompanyPortabilityPreviewResult; } +/** Preview whose manifest mixes adapters: one claude_local agent and one codex_local agent. */ +function buildMixedAdapterPreviewResult(): CompanyPortabilityPreviewResult { + return { + include: { company: true, agents: true, projects: true, issues: true }, + targetCompanyId: null, + targetCompanyName: null, + collisionStrategy: "rename", + selectedAgentSlugs: ["coder", "researcher"], + plan: { + companyAction: "create", + agentPlans: [ + { slug: "coder", action: "create", plannedName: "Coder", existingAgentId: null, reason: null }, + { slug: "researcher", action: "create", plannedName: "Researcher", existingAgentId: null, reason: null }, + ], + projectPlans: [], + issuePlans: [], + }, + manifest: { + agents: [ + { slug: "coder", name: "Coder", path: "agents/coder/AGENTS.md", adapterType: "claude_local" }, + { slug: "researcher", name: "Researcher", path: "agents/researcher/AGENTS.md", adapterType: "codex_local" }, + ], + projects: [], + issues: [], + skills: [], + company: null, + }, + files: { + ".paperclip.yaml": 'schema: "paperclip/v1"\n', + "agents/coder/AGENTS.md": "---\nname: Coder\n---\n\nYou write code.\n", + "agents/researcher/AGENTS.md": "---\nname: Researcher\n---\n\nYou research.\n", + }, + envInputs: [], + warnings: [], + errors: [], + } as unknown as CompanyPortabilityPreviewResult; +} + function buildImportResult(): CompanyPortabilityImportResult { return { company: { id: "company-2", name: "Imported Test", action: "created" }, @@ -186,6 +231,10 @@ describe("CompanyImport", () => { document.body.appendChild(container); mockAuthApi.getSession.mockResolvedValue({ user: { id: "user-1" } }); mockAgentsApi.list.mockResolvedValue([]); + mockAdaptersApi.list.mockResolvedValue([ + { type: "claude_local", disabled: false }, + { type: "codex_local", disabled: false }, + ]); mockAgentsApi.resume.mockResolvedValue({ id: "agent-1", status: "idle" }); mockRoutinesApi.update.mockResolvedValue({ id: "routine-1", status: "active" }); mockCompaniesApi.importPreview.mockResolvedValue(buildPreviewResult()); @@ -420,7 +469,7 @@ describe("CompanyImport", () => { await flushReact(); expect(container.textContent).toContain("Preview failed: stream disconnected"); - expect(container.textContent).toContain("Retry, or use the CLI folder import"); + expect(container.textContent).toContain("Retry, or re-export the package without large attachments"); expect(mockPushToast).toHaveBeenCalledWith(expect.objectContaining({ tone: "error" })); // Changing the package supersedes the failed request: the error panel resets. @@ -775,4 +824,111 @@ describe("CompanyImport", () => { // The stored entry is cleared once the job settles. expect(sessionStorage.getItem("paperclip:company-import-job:company-1:acme/starter")).toBeNull(); }); + + /** Adapter selects in the picker list, in manifest order (excludes the target/collision selects). */ + function findAdapterSelects() { + return Array.from(container.querySelectorAll("select")) + .filter((select) => select.value !== "new" && select.value !== "existing" && select.value !== "rename"); + } + + async function previewMixedAdapterPackage() { + mockCompaniesApi.importPreview.mockResolvedValue(buildMixedAdapterPreviewResult()); + await renderPage(); + await enterGithubUrl(); + await clickButton((text) => text === "Preview import"); + } + + function lastImportMeta() { + const call = mockCompaniesApi.importBundleAsync.mock.calls.at(-1); + expect(call).toBeTruthy(); + return call![0] as { adapterOverrides?: Record }; + } + + it("keeps manifest adapters and sends no overrides when the user touches nothing", async () => { + await previewMixedAdapterPackage(); + + // The picker shows each agent's source adapter, not a coerced CEO default. + expect(findAdapterSelects().map((select) => select.value)).toEqual(["claude_local", "codex_local"]); + + await clickButton((text) => text.startsWith("Import 3 file")); + await settle(); + + // Untouched agents flow through with no override at all, so the server + // applies each agent's manifest adapterType. + expect(lastImportMeta().adapterOverrides).toBeUndefined(); + }); + + it("sends an override only for the agent whose adapter the user changed", async () => { + await previewMixedAdapterPackage(); + + const coderSelect = findAdapterSelects()[0]; + expect(coderSelect?.value).toBe("claude_local"); + await act(async () => { + const setter = Object.getOwnPropertyDescriptor(HTMLSelectElement.prototype, "value")!.set!; + setter.call(coderSelect!, "codex_local"); + coderSelect!.dispatchEvent(new Event("change", { bubbles: true })); + }); + await flushReact(); + + await clickButton((text) => text.startsWith("Import 3 file")); + await settle(); + + expect(lastImportMeta().adapterOverrides).toEqual({ + coder: { adapterType: "codex_local" }, + }); + }); + + it("falls back to the CEO adapter with a visible warning when a manifest adapter is not installed", async () => { + // The destination has no codex_local adapter; the CEO fallback (an empty + // agent list defaults to claude_local) takes over — never silently. + mockAdaptersApi.list.mockResolvedValue([{ type: "claude_local", disabled: false }]); + await previewMixedAdapterPackage(); + + expect(container.textContent).toContain("source adapter codex_local is not installed here"); + expect(container.textContent).toContain("will use Claude Code"); + expect(findAdapterSelects().map((select) => select.value)).toEqual(["claude_local", "claude_local"]); + + await clickButton((text) => text.startsWith("Import 3 file")); + await settle(); + + // Only the unavailable agent is overridden; the claude_local agent still + // carries no override and keeps its manifest adapter. + expect(lastImportMeta().adapterOverrides).toEqual({ + researcher: { adapterType: "claude_local" }, + }); + }); + + it("picks an installed adapter as the fallback when the CEO adapter is itself unavailable", async () => { + // Neither codex_local nor the CEO fallback (claude_local) is installed; + // the fallback must be an adapter that actually exists on the + // destination, never an unavailable type the server would reject. + mockAdaptersApi.list.mockResolvedValue([{ type: "gemini_local", disabled: false }]); + await previewMixedAdapterPackage(); + + expect(container.textContent).toContain("source adapter codex_local is not installed here"); + expect(findAdapterSelects().map((select) => select.value)).toEqual(["gemini_local", "gemini_local"]); + + await clickButton((text) => text.startsWith("Import 3 file")); + await settle(); + + expect(lastImportMeta().adapterOverrides).toEqual({ + researcher: { adapterType: "gemini_local" }, + coder: { adapterType: "gemini_local" }, + }); + }); + + it("fails open to manifest adapters when the adapter list cannot be read", async () => { + // Unknown availability must never coerce: with no adapter list, every + // agent keeps its manifest adapter and no fallback warning renders. + mockAdaptersApi.list.mockRejectedValue(new ApiError("adapters unavailable", 500, null)); + await previewMixedAdapterPackage(); + + expect(container.textContent).not.toContain("is not installed here"); + expect(findAdapterSelects().map((select) => select.value)).toEqual(["claude_local", "codex_local"]); + + await clickButton((text) => text.startsWith("Import 3 file")); + await settle(); + + expect(lastImportMeta().adapterOverrides).toBeUndefined(); + }); }); diff --git a/ui/src/pages/CompanyImport.tsx b/ui/src/pages/CompanyImport.tsx index 3339867d6de..966b6863ea5 100644 --- a/ui/src/pages/CompanyImport.tsx +++ b/ui/src/pages/CompanyImport.tsx @@ -14,6 +14,7 @@ import { useToastActions } from "../context/ToastContext"; import { authApi } from "../api/auth"; import { ApiError } from "../api/client"; import { companiesApi, type CompanyImportJobAccepted } from "../api/companies"; +import { adaptersApi } from "../api/adapters"; import { agentsApi } from "../api/agents"; import { routinesApi } from "../api/routines"; import { sidebarPreferencesApi } from "../api/sidebarPreferences"; @@ -559,7 +560,15 @@ const IMPORT_ADAPTER_OPTIONS: { value: string; label: string }[] = listUIAdapter interface AdapterPickerItem { slug: string; name: string; + /** Adapter type from the package manifest (the source's adapter). */ adapterType: string; + /** + * Set when the manifest adapter is not installed on the destination: the + * adapter type the agent falls back to unless the user picks another one. + * Null when the manifest adapter is usable here (or availability is unknown, + * which fails open to the manifest adapter). + */ + fallbackAdapterType: string | null; } function AdapterPickerList({ @@ -592,7 +601,8 @@ function AdapterPickerList({
{agents.map((agent) => { - const selectedType = adapterOverrides[agent.slug] ?? agent.adapterType; + const selectedType = + adapterOverrides[agent.slug] ?? agent.fallbackAdapterType ?? agent.adapterType; const isExpanded = expandedSlugs.has(agent.slug); const vals = configValues[agent.slug] ?? { ...defaultCreateValues, adapterType: selectedType }; @@ -634,6 +644,14 @@ function AdapterPickerList({ configure adapter
+ {agent.fallbackAdapterType && ( +
+

+ source adapter {agent.adapterType} is not installed here — this agent + will use {adapterLabels[selectedType] ?? getAdapterLabel(selectedType)} +

+
+ )} {isExpanded && (
adaptersApi.list(), + staleTime: 5 * 60 * 1000, + }); + // Null while the list is loading or unreadable: availability is unknown, so + // fail open and trust the manifest rather than coercing every agent. + const availableAdapterTypes = useMemo(() => { + if (!installedAdapters) return null; + return new Set(installedAdapters.filter((a) => !a.disabled).map((a) => a.type)); + }, [installedAdapters]); + const localZipHelpText = "Upload a .zip exported directly from Paperclip. Re-zipped archives created by Finder, Explorer, or other zip tools may not import correctly."; @@ -952,12 +986,10 @@ export function CompanyImport() { setSkippedSlugs(new Set()); setConfirmedSlugs(new Set()); - // Initialize adapter overrides — default all agents to the CEO's adapter type - const defaultAdapters: Record = {}; - for (const agent of result.manifest.agents) { - defaultAdapters[agent.slug] = ceoAdapterType; - } - setAdapterOverrides(defaultAdapters); + // Adapter overrides start empty: each agent keeps its manifest adapter + // unless the user changes it, or the manifest adapter is not installed + // here (handled per-agent via a warned fallback, never seeded silently). + setAdapterOverrides({}); setAdapterExpandedSlugs(new Set()); setAdapterConfigValues({}); @@ -1346,9 +1378,11 @@ export function CompanyImport() { function handleAdapterConfigChange(slug: string, patch: Partial) { resetMutationState(); + const agent = adapterAgents.find((a) => a.slug === slug); + const currentType = agent ? effectiveAdapterType(agent) : adapterOverrides[slug] ?? "claude_local"; setAdapterConfigValues((prev) => ({ ...prev, - [slug]: { ...(prev[slug] ?? { ...defaultCreateValues, adapterType: adapterOverrides[slug] ?? "claude_local" }), ...patch }, + [slug]: { ...(prev[slug] ?? { ...defaultCreateValues, adapterType: currentType }), ...patch }, })); } @@ -1392,23 +1426,42 @@ export function CompanyImport() { } } - // Build the list of agents for adapter picking + // Build the list of agents for adapter picking. An agent whose manifest + // adapter is not installed on the destination gets a warned fallback to the + // CEO's adapter; while availability is unknown the manifest adapter stands. const adapterAgents = useMemo(() => { if (!importPreview) return []; return importPreview.manifest.agents.map((a) => ({ slug: a.slug, name: a.name, adapterType: a.adapterType, + // The fallback must itself be installed: the CEO's adapter when it is, + // else any installed adapter, else null so the manifest adapter stands + // and the server's unknown-adapter rejection is the backstop. + fallbackAdapterType: + availableAdapterTypes && !availableAdapterTypes.has(a.adapterType) + ? availableAdapterTypes.has(ceoAdapterType) + ? ceoAdapterType + : [...availableAdapterTypes][0] ?? null + : null, })); - }, [importPreview]); + }, [importPreview, availableAdapterTypes, ceoAdapterType]); + + /** The adapter type an imported agent will actually use: an explicit user pick, else the availability fallback, else the manifest adapter. */ + function effectiveAdapterType(agent: AdapterPickerItem): string { + return adapterOverrides[agent.slug] ?? agent.fallbackAdapterType ?? agent.adapterType; + } - // Build final adapterOverrides for import request + // Build final adapterOverrides for import request. Only agents that diverge + // from the manifest adapter (a user pick or an availability fallback) or + // carry edited adapter config send an override — untouched agents flow + // through with none, so the manifest adapter survives the import. function buildFinalAdapterOverrides(): Record | undefined { - if (adapterAgents.length === 0) return undefined; const overrides: Record = {}; for (const agent of adapterAgents) { - const selectedType = adapterOverrides[agent.slug] ?? agent.adapterType; + const selectedType = effectiveAdapterType(agent); const configVals = adapterConfigValues[agent.slug]; + if (selectedType === agent.adapterType && !configVals) continue; const override: CompanyPortabilityAdapterOverride = { adapterType: selectedType }; if (configVals) { const uiAdapter = getUIAdapter(selectedType); @@ -1779,7 +1832,7 @@ export function CompanyImport() { {previewMutation.error instanceof Error ? previewMutation.error.message : "the request did not complete."}{" "} - Retry, or use the CLI folder import for very large packages. + Retry, or re-export the package without large attachments to shrink it.

)} diff --git a/ui/src/pages/CompanySkills.test.tsx b/ui/src/pages/CompanySkills.test.tsx index 180fd9d45a8..ef381ae1bcb 100644 --- a/ui/src/pages/CompanySkills.test.tsx +++ b/ui/src/pages/CompanySkills.test.tsx @@ -279,6 +279,59 @@ async function renderDiscoveryGrid(props: Partial>; +} + function buttonsNamed(node: ParentNode, name: string) { return Array.from(node.querySelectorAll("button")).filter((button) => button.textContent?.trim() === name); } @@ -341,6 +394,55 @@ describe("DiscoveryGrid Studio entry points", () => { expect(onCreate).toHaveBeenCalledTimes(2); }); + it("keeps folder creation in the compact rail control", async () => { + const props = projectFolderGridProps(); + const node = await renderDiscoveryGrid(props); + const compactCreateButton = node.querySelector('button[title="New folder"]'); + + expect(buttonsNamed(node, "New folder")).toHaveLength(0); + expect(compactCreateButton).not.toBeNull(); + + await click(compactCreateButton!); + + expect(props.onCreateFolderIn).toHaveBeenCalledWith(null); + expect(props.onCreateFolder).not.toHaveBeenCalled(); + }); + + it("keeps folder creation available when no folder rail exists", async () => { + const onCreateFolder = vi.fn(); + const node = await renderDiscoveryGrid({ + ...projectFolderGridProps(), + folderResult: { ...projectFolderResult, folders: [] }, + onCreateFolder, + }); + const createButton = buttonsNamed(node, "New folder")[0] as HTMLButtonElement; + + expect(createButton).toBeDefined(); + + await click(createButton); + + expect(onCreateFolder).toHaveBeenCalledOnce(); + }); + + it("refreshes only the project represented by the active project folder", async () => { + const onScan = vi.fn(); + const node = await renderDiscoveryGrid({ + ...projectFolderGridProps(), + folderSelection: "project-folder", + onScan, + }); + const refreshButton = node.querySelector( + 'button[aria-label="Refresh Acme project skills"]', + ); + + expect(refreshButton).not.toBeNull(); + + await click(refreshButton!); + + expect(onScan).toHaveBeenCalledOnce(); + expect(onScan).toHaveBeenCalledWith("project-1"); + }); + it("does not open a skill when keyboard-activating its actions button", async () => { const onOpenCard = vi.fn(); const card = { diff --git a/ui/src/pages/CompanySkills.tsx b/ui/src/pages/CompanySkills.tsx index 2bb0859f07f..42a9325d464 100644 --- a/ui/src/pages/CompanySkills.tsx +++ b/ui/src/pages/CompanySkills.tsx @@ -1084,7 +1084,7 @@ export function DiscoveryGrid({ onImport: () => void; onImportFromProject: () => void; onBrowseCatalog: () => void; - onScan: () => void; + onScan: (projectId?: string) => void; scanPending: boolean; scanStatus: string | null; folderResult?: FolderListResult | null; @@ -1138,7 +1138,15 @@ export function DiscoveryGrid({ ); // The nested folder tree owns the left rail whenever folders (reserved roots // or user folders) exist for the installed view. - const showFolderRail = Boolean(folderResult && folderResult.folders.length > 0 && onFolderSelect && folderActionsReady); + const showFolderRail = Boolean( + folderResult && folderResult.folders.length > 0 && onFolderSelect && folderActionsReady, + ); + const activeProjectFolder = useMemo(() => { + if (!folderResult || folderSelection === "all" || folderSelection === "unfiled") return null; + const folder = folderResult.folders.find((candidate) => candidate.id === folderSelection); + return folder?.systemKey?.startsWith("project:") ? folder : null; + }, [folderResult, folderSelection]); + const activeProjectId = activeProjectFolder?.systemKey?.slice("project:".length) || null; return ( // On desktop the store is bounded to the viewport so the category sidebar @@ -1240,8 +1248,9 @@ export function DiscoveryGrid({ + ) : null} ) : null} {folderNudgeStorageKey && onCreateFolder && folderResult && folderResult.folders.length === 0 && !loading && cards.length > 0 ? ( @@ -4278,13 +4300,21 @@ export function CompanySkills() { }); const scanProjects = useMutation({ - mutationFn: () => companySkillsApi.scanProjects(selectedCompanyId!), - onMutate: () => { - setScanStatusMessage("Scanning project workspaces for skills..."); + mutationFn: (projectId?: string) => companySkillsApi.scanProjects( + selectedCompanyId!, + projectId ? { projectIds: [projectId] } : {}, + ), + onMutate: (projectId) => { + setScanStatusMessage( + projectId ? "Refreshing project skills..." : "Scanning project workspaces for skills...", + ); }, onSuccess: async (result) => { setScanStatusMessage("Refreshing skills list..."); - await queryClient.invalidateQueries({ queryKey: queryKeys.companySkills.list(selectedCompanyId!) }); + await Promise.all([ + queryClient.invalidateQueries({ queryKey: queryKeys.companySkills.list(selectedCompanyId!) }), + queryClient.invalidateQueries({ queryKey: queryKeys.folders.list(selectedCompanyId!, "skill") }), + ]); const summary = formatProjectScanSummary(result); setScanStatusMessage(summary); pushToast({ @@ -5308,7 +5338,7 @@ export function CompanySkills() { onImport={() => setImportDialogOpen(true)} onImportFromProject={() => setImportFromProjectOpen(true)} onBrowseCatalog={() => setDiscoveryTab("catalog")} - onScan={() => scanProjects.mutate()} + onScan={(projectId) => scanProjects.mutate(projectId)} scanPending={scanProjects.isPending} scanStatus={scanStatusMessage} folderResult={showInstalledFolders ? railSkillFolderResult : null} diff --git a/ui/src/pages/ExecutionWorkspaceDetail.test.tsx b/ui/src/pages/ExecutionWorkspaceDetail.test.tsx index faa981a03f7..d4356c0d212 100644 --- a/ui/src/pages/ExecutionWorkspaceDetail.test.tsx +++ b/ui/src/pages/ExecutionWorkspaceDetail.test.tsx @@ -288,15 +288,15 @@ describe("ExecutionWorkspaceDetail plugin slots", () => { }); }); - it("shows the linked project workspace summary above tasks", async () => { + it("shows a summary scoped to the execution workspace above tasks", async () => { mockExecutionWorkspacesApi.get.mockResolvedValue(workspace({ projectWorkspaceId: "project-workspace-1" })); await render(); expect(mockSummarySlotCard).toHaveBeenCalledWith(expect.objectContaining({ companyId: "company-1", - scopeKind: "project_workspace", - scopeId: "project-workspace-1", + scopeKind: "execution_workspace", + scopeId: "workspace-1", title: "Workspace summary", })); const summary = container.querySelector('[data-testid="summary-slot-card"]'); @@ -307,11 +307,15 @@ describe("ExecutionWorkspaceDetail plugin slots", () => { expect(summary.compareDocumentPosition(issues) & Node.DOCUMENT_POSITION_FOLLOWING).not.toBe(0); }); - it("does not show a project workspace summary for standalone execution workspaces", async () => { + it("shows an isolated summary for standalone execution workspaces", async () => { await render(); - expect(mockSummarySlotCard).not.toHaveBeenCalled(); - expect(container.querySelector('[data-testid="summary-slot-card"]')).toBeNull(); + expect(mockSummarySlotCard).toHaveBeenCalledWith(expect.objectContaining({ + companyId: "company-1", + scopeKind: "execution_workspace", + scopeId: "workspace-1", + })); + expect(container.querySelector('[data-testid="summary-slot-card"]')).not.toBeNull(); }); it("does not mount plugin slots scoped to other entity types", async () => { diff --git a/ui/src/pages/ExecutionWorkspaceDetail.tsx b/ui/src/pages/ExecutionWorkspaceDetail.tsx index 2cc119a5422..9fc6cf47bbd 100644 --- a/ui/src/pages/ExecutionWorkspaceDetail.tsx +++ b/ui/src/pages/ExecutionWorkspaceDetail.tsx @@ -1523,15 +1523,13 @@ export function ExecutionWorkspaceDetail() { ) : activeTab === "issues" ? (
- {workspace.projectWorkspaceId ? ( - - ) : null} + { act(() => root.unmount()); }); + it("restores folded and unfolded sub-tasks across remounts", async () => { + routerMock.location.pathname = "/inbox/mine"; + const storageKey = "paperclip:inbox:collapsed-parents:company-1"; + localStorage.removeItem(storageKey); + + const parent = createIssue({ + id: "parent-issue", + identifier: "PAP-1001", + title: "Parent inbox task", + }); + const child = createIssue({ + id: "child-issue", + identifier: "PAP-1002", + parentId: parent.id, + title: "Nested inbox task", + }); + apiMocks.issuesList.mockResolvedValue([parent, child]); + + const mountInbox = async () => { + const queryClient = new QueryClient({ + defaultOptions: { queries: { retry: false, staleTime: 0, gcTime: 0 } }, + }); + const root = createRoot(container); + await act(async () => { + root.render( + + + , + ); + }); + await vi.waitFor(() => { + expect(container.textContent).toContain(parent.title); + }); + return root; + }; + const parentToggle = () => { + const parentRow = Array.from(container.querySelectorAll("[data-inbox-item]")) + .find((row) => row.textContent?.includes(parent.title)); + return parentRow?.querySelector('button[data-slot="icon-button"]') ?? null; + }; + + let root = await mountInbox(); + try { + expect(container.textContent).toContain(child.title); + + await act(async () => { + parentToggle()?.dispatchEvent(new MouseEvent("click", { bubbles: true })); + }); + await vi.waitFor(() => { + expect(container.textContent).not.toContain(child.title); + }); + expect(JSON.parse(localStorage.getItem(storageKey) ?? "[]")).toEqual([parent.id]); + + act(() => root.unmount()); + root = await mountInbox(); + expect(container.textContent).not.toContain(child.title); + + await act(async () => { + parentToggle()?.dispatchEvent(new MouseEvent("click", { bubbles: true })); + }); + await vi.waitFor(() => { + expect(container.textContent).toContain(child.title); + }); + expect(JSON.parse(localStorage.getItem(storageKey) ?? "[]")).toEqual([]); + + act(() => root.unmount()); + root = await mountInbox(); + expect(container.textContent).toContain(child.title); + } finally { + localStorage.removeItem(storageKey); + act(() => root.unmount()); + } + }); + it("shows blocked toolbar controls on the Blocked tab", async () => { routerMock.location.pathname = "/inbox/blocked"; const queryClient = new QueryClient({ diff --git a/ui/src/pages/Inbox.tsx b/ui/src/pages/Inbox.tsx index e9abc46966e..4bf93eb0d49 100644 --- a/ui/src/pages/Inbox.tsx +++ b/ui/src/pages/Inbox.tsx @@ -144,6 +144,7 @@ import { isInboxEntityDismissed, isMineInboxTab, loadCollapsedInboxGroupKeys, + loadCollapsedInboxParentIds, loadInboxFilterPreferences, loadInboxIssueColumns, loadInboxNesting, @@ -155,6 +156,7 @@ import { resolveInboxSelectionIndex, saveInboxFilterPreferences, saveCollapsedInboxGroupKeys, + saveCollapsedInboxParentIds, saveInboxIssueColumns, saveInboxNesting, saveInboxWorkItemGroupBy, @@ -801,6 +803,7 @@ export function Inbox() { previousSelectedCompanyIdRef.current = selectedCompanyId; setFilterPreferences(loadInboxFilterPreferences(selectedCompanyId)); setCollapsedGroupKeys(loadCollapsedInboxGroupKeys(selectedCompanyId)); + setCollapsedInboxParents(loadCollapsedInboxParentIds(selectedCompanyId)); } }, [selectedCompanyId]); @@ -1372,7 +1375,9 @@ export function Inbox() { return next; }); }, []); - const [collapsedInboxParents, setCollapsedInboxParents] = useState>(new Set()); + const [collapsedInboxParents, setCollapsedInboxParents] = useState>( + () => loadCollapsedInboxParentIds(selectedCompanyId), + ); const [collapsedGroupKeys, setCollapsedGroupKeys] = useState>(() => loadCollapsedInboxGroupKeys(selectedCompanyId)); const toggleGroupCollapse = useCallback((groupKey: string) => { setCollapsedGroupKeys((prev) => { @@ -1508,18 +1513,20 @@ export function Inbox() { const next = new Set(prev); if (next.has(parentId)) next.delete(parentId); else next.add(parentId); + saveCollapsedInboxParentIds(selectedCompanyId, next); return next; }); - }, []); + }, [selectedCompanyId]); const setInboxParentCollapsed = useCallback((parentId: string, collapsed: boolean) => { setCollapsedInboxParents((prev) => { if (prev.has(parentId) === collapsed) return prev; const next = new Set(prev); if (collapsed) next.add(parentId); else next.delete(parentId); + saveCollapsedInboxParentIds(selectedCompanyId, next); return next; }); - }, []); + }, [selectedCompanyId]); // Build flat navigation list from visible rows so keyboard traversal respects collapsed groups. const flatNavItems = useMemo((): NavEntry[] => {